My css selectors response.css('div.jhfizC')
and ('a[itemprop="url"]')
show 97 items in the web page, but my code is only scraping 35 items. Where is the fault?
Here is my code:
import scrapy class EbayfootSpider(scrapy.Spider): name = 'exebay' start_urls = ['https://www.mercari.com/search/?brandIds=4578&categoryIds=724&facets=2&length=30'] def parse(self, response): big_div = response.css('div.jhfizC') for all_div in big_div: item_url = all_div.css('a[itemprop="url"]').css('::attr(href)')[0].extract() if item_url is not None: print(item_url)
Advertisement
Answer
In the end of the url just put length 90 instead of 30 , length indicate 30 item per page.
In [1]: url = 'https://www.mercari.com/search/?brandIds=4578&categoryIds=724&facets=2&length=90' fetch(url) In [4]: response Out[4]: <200 https://www.mercari.com/search/?brandIds=4578&categoryIds=724&facets=2&length=90> In [6]: big_div = response.css('div.jhfizC') In [7]: item_url = all_div.css('a[itemprop="url"]').css('::attr(href)') In [11]: for i, all_div in enumerate( big_div): ...: item_url = all_div.css('a[itemprop="url"]').css('::attr(href)')[0].extract() ...: print(i,'https://www.mercari.com/'+item_url) ...: 0 https://www.mercari.com//us/item/m27379701884/?ref=search_results 1 https://www.mercari.com//us/item/m87408459178/?ref=search_results 2 https://www.mercari.com//us/item/m31639451091/?ref=search_results 3 https://www.mercari.com//us/item/m55461396239/?ref=search_results 4 https://www.mercari.com//us/item/m61065950216/?ref=search_results 5 https://www.mercari.com//us/item/m59003878460/?ref=search_results 6 https://www.mercari.com//us/item/m42114019532/?ref=search_results 7 https://www.mercari.com//us/item/m60410492125/?ref=search_results 8 https://www.mercari.com//us/item/m61707057858/?ref=search_results 9 https://www.mercari.com//us/item/m77824553207/?ref=search_results 10 https://www.mercari.com//us/item/m72401412407/?ref=search_results 11 https://www.mercari.com//us/item/m83737664911/?ref=search_results 12 https://www.mercari.com//us/item/m31513565581/?ref=search_results 13 https://www.mercari.com//us/item/m53686770401/?ref=search_results 14 https://www.mercari.com//us/item/m31687661458/?ref=search_results 15 https://www.mercari.com//us/item/m73565122351/?ref=search_results 16 https://www.mercari.com//us/item/m54774041977/?ref=search_results 17 https://www.mercari.com//us/item/m63815616689/?ref=search_results 18 https://www.mercari.com//us/item/m72900819243/?ref=search_results 19 https://www.mercari.com//us/item/m38982779224/?ref=search_results 20 https://www.mercari.com//us/item/m16950460012/?ref=search_results 21 https://www.mercari.com//us/item/m96072811718/?ref=search_results 22 https://www.mercari.com//us/item/m98926246504/?ref=search_results 23 https://www.mercari.com//us/item/m22674930214/?ref=search_results 24 https://www.mercari.com//us/item/m76505587116/?ref=search_results 25 https://www.mercari.com//us/item/m81533546144/?ref=search_results 26 https://www.mercari.com//us/item/m71881974396/?ref=search_results 27 https://www.mercari.com//us/item/m57251522596/?ref=search_results 28 https://www.mercari.com//us/item/m85616167131/?ref=search_results 29 https://www.mercari.com//us/item/m66592587980/?ref=search_results 30 https://www.mercari.com//us/item/m16222098730/?ref=search_results 31 https://www.mercari.com//us/item/m30516632346/?ref=search_results 32 https://www.mercari.com//us/item/m82678279409/?ref=search_results 33 https://www.mercari.com//us/item/m22459472107/?ref=search_results 34 https://www.mercari.com//us/item/m49112577494/?ref=search_results 35 https://www.mercari.com//us/item/m89573947301/?ref=search_results 36 https://www.mercari.com//us/item/m15348612036/?ref=search_results 37 https://www.mercari.com//us/item/m11392827625/?ref=search_results 38 https://www.mercari.com//us/item/m60718086586/?ref=search_results 39 https://www.mercari.com//us/item/m57133492284/?ref=search_results 40 https://www.mercari.com//us/item/m73858839710/?ref=search_results 41 https://www.mercari.com//us/item/m77945239665/?ref=search_results 42 https://www.mercari.com//us/item/m13356901817/?ref=search_results 43 https://www.mercari.com//us/item/m63603936227/?ref=search_results 44 https://www.mercari.com//us/item/m49178440576/?ref=search_results 45 https://www.mercari.com//us/item/m14806568451/?ref=search_results 46 https://www.mercari.com//us/item/m67151874325/?ref=search_results 47 https://www.mercari.com//us/item/m90580799274/?ref=search_results 48 https://www.mercari.com//us/item/m66942995364/?ref=search_results 49 https://www.mercari.com//us/item/m90049596871/?ref=search_results 50 https://www.mercari.com//us/item/m25892617355/?ref=search_results 51 https://www.mercari.com//us/item/m11787855892/?ref=search_results 52 https://www.mercari.com//us/item/m10379012995/?ref=search_results 53 https://www.mercari.com//us/item/m28508818974/?ref=search_results 54 https://www.mercari.com//us/item/m23474551312/?ref=search_results 55 https://www.mercari.com//us/item/m99385515312/?ref=search_results 56 https://www.mercari.com//us/item/m13303803841/?ref=search_results 57 https://www.mercari.com//us/item/m36605382938/?ref=search_results 58 https://www.mercari.com//us/item/m27122502813/?ref=search_results 59 https://www.mercari.com//us/item/m58324967907/?ref=search_results 60 https://www.mercari.com//us/item/m88943761510/?ref=search_results 61 https://www.mercari.com//us/item/m73769298655/?ref=search_results 62 https://www.mercari.com//us/item/m60815399306/?ref=search_results 63 https://www.mercari.com//us/item/m48872595825/?ref=search_results 64 https://www.mercari.com//us/item/m53519878176/?ref=search_results 65 https://www.mercari.com//us/item/m59524853672/?ref=search_results 66 https://www.mercari.com//us/item/m72346338496/?ref=search_results 67 https://www.mercari.com//us/item/m86496319979/?ref=search_results 68 https://www.mercari.com//us/item/m54138309188/?ref=search_results 69 https://www.mercari.com//us/item/m61559200881/?ref=search_results 70 https://www.mercari.com//us/item/m42596652475/?ref=search_results 71 https://www.mercari.com//us/item/m54065649678/?ref=search_results 72 https://www.mercari.com//us/item/m87125447888/?ref=search_results 73 https://www.mercari.com//us/item/m26421150316/?ref=search_results 74 https://www.mercari.com//us/item/m72992367701/?ref=search_results 75 https://www.mercari.com//us/item/m46688464169/?ref=search_results 76 https://www.mercari.com//us/item/m34842804531/?ref=search_results 77 https://www.mercari.com//us/item/m80806378583/?ref=search_results 78 https://www.mercari.com//us/item/m39280119583/?ref=search_results 79 https://www.mercari.com//us/item/m48130903632/?ref=search_results 80 https://www.mercari.com//us/item/m57955016021/?ref=search_results 81 https://www.mercari.com//us/item/m54360880136/?ref=search_results 82 https://www.mercari.com//us/item/m29432458124/?ref=search_results 83 https://www.mercari.com//us/item/m81313993828/?ref=search_results 84 https://www.mercari.com//us/item/m34469644752/?ref=search_results 85 https://www.mercari.com//us/item/m26953235217/?ref=search_results 86 https://www.mercari.com//us/item/m29197336718/?ref=search_results 87 https://www.mercari.com//us/item/m24460351728/?ref=search_results 88 https://www.mercari.com//us/item/m95427865100/?ref=search_results 89 https://www.mercari.com//us/item/m29184483225/?ref=search_results 90 https://www.mercari.com//us/item/m47761827365/?ref=search_results 91 https://www.mercari.com//us/item/m333400415/?ref=search_results 92 https://www.mercari.com//us/item/m222574675/?ref=search_results 93 https://www.mercari.com//us/item/m916567471/?ref=search_results 94 https://www.mercari.com//us/item/m516644163/?ref=search_results 95 https://www.mercari.com//us/item/m30663599997/?ref=search_results 96 https://www.mercari.com//us/item/m935881167/?ref=search_results 97 https://www.mercari.com//us/item/m645603485/?ref=search_results