how to create row in model based on data fetched from JSON request from third-party crawling with “best practice”

describe the problem:

I want to crawl data from dataforseo , and save them directly in model database through model.create() method with having multi model with multi relation with models so for instance in model A have

1. ManyToMany relation with model B

2. ManyToMany relation with model C

3. ManyToMany relation with model D

   and model B have relation with model C

so my question is how to save JSON response to all model mentioned above smoothly through model A create Method

response came from request:

[
   {
      "title":"title",
      "url":"url",
      "description":"description",
      "pre_snippet":"pre_snippet",
      "extended_snippet":"extended_snippet",
      "images":"images",
      "amp_version":"amp_version",
      "rating":{
         "rating_type":"rating_type",
         "value":"value",
         "votes_count":"votes_count",
         "rating_max":"rating_max"
      },
      "price":{
         "current":"current",
         "regular":"regular",
         "max_value":"max_value"
      },
      "highlighted":{
         "name_highlighted":"name_highlighted"
      }
   },
   {
      "title":"title",
      "url":"url",
      "description":"description",
      "pre_snippet":"pre_snippet",
      "extended_snippet":"extended_snippet",
      "images":"images",
      "amp_version":"amp_version",
      "rating":{
         "rating_type":"rating_type",
         "value":"value",
         "votes_count":"votes_count",
         "rating_max":"rating_max"
      },
      "price":{
         "current":"current",
         "regular":"regular",
         "max_value":"max_value"
      },
      "highlighted":{
         "name_highlighted":"name_highlighted"
      }
   }
]

Code:

view.py file

@api_view(['POST'])
@parser_classes((JSONParser,))
def crawl_data(request):
    """
    A view that can accept POST requests with JSON content.
    """
    Product.create(
         title=request.data[title]
         url=request.data[url]
         describtion=request.data[describtion]
         ...
         )
    return Response({'received data': request.data})

models.py

class Highlighted(models.Model):
    name_highlighted = models.CharField(max_length=100)

    def __str__(self):
        return str(self.name_highlighted)


class Rating(models.Model):
    rating_type = models.CharField(max_length=500, null=True, blank=True)  # make unique
    value = models.CharField(max_length=500, null=True, blank=True)
    votes_count = models.CharField(max_length=500, null=True, blank=True)
    rating_max = models.CharField(max_length=500, null=True, blank=True)

    def __str__(self):
        return str(self.value)


class Price(models.Model):
    current = models.CharField(max_length=500, null=True, blank=True, default="none")
    regular = models.CharField(max_length=500, null=True, blank=True)
    max_value = models.CharField(max_length=500, null=True, blank=True)

    def __str__(self):
        return str(self.current)


class Product(models.Model):
    title = models.CharField(max_length=500, null=True, blank=True)
    url = models.CharField(max_length=500, null=True, blank=True)
    description = models.CharField(max_length=500, null=True, blank=True)
    pre_snippet = models.CharField(max_length=500, null=True, blank=True)
    extended_snippet = models.CharField(max_length=500, null=True, blank=True)
    images = models.CharField(max_length=500, null=True, blank=True)
    amp_version = models.CharField(max_length=500, null=True, blank=True)
    rating = models.ManyToManyField(Rating, null=True, blank=True)
    price = models.ManyToManyField(Price, null=True, blank=True)
    highlighted = models.ManyToManyField(Highlighted)

    def __str__(self):
        return str(self.url)

 

any help appreciate it

Answer

you can do override create method by do the following in your models.py file :

try this out:

in product model you can override create model like so

class Product(models.Model):
    title = models.CharField(max_length=500, null=True, blank=True)
    url = models.CharField(max_length=500, null=True, blank=True)
    description = models.CharField(max_length=500, null=True, blank=True)
    pre_snippet = models.CharField(max_length=500, null=True, blank=True)
    extended_snippet = models.CharField(max_length=500, null=True, blank=True)
    images = models.CharField(max_length=500, null=True, blank=True)
    amp_version = models.CharField(max_length=500, null=True, blank=True)
    rating = models.ManyToManyField(Rating, null=True, blank=True)
    price = models.ManyToManyField(Price, null=True, blank=True)
    highlighted = models.ManyToManyField(Highlighted)
    extended_people_also_search = models.ManyToManyField(ExtendedPeopleAlsoSearch)

    def __str__(self):
        return str(self.url)

    @classmethod
    def create(cls, **kwargs):
        product = cls.objects.create(
            title=kwargs['title'],
            url=kwargs['url'],
            description=kwargs['description'],
            pre_snippet=kwargs['pre_snippet'],
            extended_snippet=kwargs['extended_snippet'],
            images=kwargs['images'],
            amp_version=kwargs['amp_version'],

        )

        # add price dict

        price, created = Price.objects.get_or_create(current=kwargs['price']["current"],
                                                     regular=kwargs['price']["regular"],
                                                     max_value=kwargs['price']["max_value"],
                                                     )
        product.price.add(price)

        # add rating dict
        rating, created = Rating.objects.get_or_create(rating_type=kwargs['rating']["rating_type"],
                                                       value=kwargs['rating']["value"],
                                                       votes_count=kwargs['rating']["votes_count"],
                                                       rating_max=kwargs['rating']["rating_max"],
                                                       )
        product.rating.add(rating)

        return product

    def __str__(self):
        return str(self.url)

and in your views.py in function crawl you can just iterate through your JSON response and just add them to your model like below:

@api_view(['POST'])
@parser_classes((JSONParser,))
def crawl_data(request):
    """
    A view that can accept POST requests with JSON content.
    """
    for product_data in request.data:
        product = Product.create(**product_data)
    # DO SOMETHING WITH product
    return Response({'received data': request.data})