As to say this code works but problem that i am facing that only one url it scrape the data afterward it through an error as show below in figure help me out from this . it print only one link after it through session not created error
JavaScript
x
54
54
1
import requests
2
from bs4 import BeautifulSoup
3
from selenium import webdriver
4
from selenium.webdriver.chrome.options import Options
5
from selenium.webdriver.common.keys import Keys
6
import pandas as pd
7
import time
8
9
url = ["https://www.skechers.com/women/shoes/ultra-flex---twilight-twinkle/149173.html",
10
"https://www.skechers.com/women/shoes/ultra-flex---statements/12841.html",
11
"https://www.skechers.com/women/shoes/ultra-flex/12843.html",
12
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst---extreme-outlook/124580.html",
13
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst---lunar-mars/124577.html",
14
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst/124575.html",
15
"https://www.skechers.com/women/shoes/skechers-gowalk-stability---coco-jazz/124603.html",
16
"https://www.skechers.com/women/shoes/skechers-gowalk-stability---magnificent-glow/124602.html",
17
"https://www.skechers.com/women/shoes/skechers-gowalk-joy/124191.html",
18
"https://www.skechers.com/women/shoes/skechers-gowalk-joy---paradise/15601.html",
19
"https://www.skechers.com/women/shoes/skech-air-element-2.0---vivid-blush/149408.html",
20
"https://www.skechers.com/women/shoes/skech-air-element-2.0---kaleidoscope-splash/149134.html",
21
"https://www.skechers.com/women/shoes/skech-air-element-2.0/149062.html",
22
"https://www.skechers.com/women/shoes/skech-air-element-2.0---rising-gaze/149611.html",
23
"https://www.skechers.com/women/shoes/skechers-gowalk-air---twirl/124073.html",
24
"https://www.skechers.com/women/shoes/gowalk-air---whirl/124074.html",
25
"https://www.skechers.com/women/shoes/skechers-arch-fit-refine---her-best/104092.html",
26
"https://www.skechers.com/women/shoes/skechers-arch-fit-refine/104090.html",
27
"https://www.skechers.com/women/shoes/glide-step---stepping-up/104086.html",
28
"https://www.skechers.com/women/shoes/relaxed-fit-empire-dlux---sharp-witted/149007.html",
29
"https://www.skechers.com/women/shoes/relaxed-fit-empire-dlux---paradise-sky/149274.html",
30
"https://www.skechers.com/women/shoes/skechers-gorun-smart/128230.html"]
31
for endpoint in url:
32
options = webdriver.ChromeOptions()
33
options.add_experimental_option("excludeSwitches", ['enable-automation'])
34
options.add_argument('--disable-blink-features=AutomationControlled')
35
options.add_argument(
36
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36")
37
options.add_argument("--remote-debugging-port=9222")
38
39
driver = webdriver.Chrome('F:/chromedriver.exe',options=options)
40
driver.get(endpoint)
41
time.sleep(5)
42
image= None
43
page_Source = driver.page_source
44
soup = BeautifulSoup(page_Source, 'html.parser')
45
content= soup.find_all('div',class_='c-product-description-features s-product-description-features row')
46
47
pid=soup.find('span',class_='product-id').text
48
links = soup.find_all('img', srcset=True)
49
for link in links:
50
img= link['srcset']
51
alt=link['alt']
52
53
print(pid ,img,alt)
54
Advertisement
Answer
JavaScript
1
46
46
1
import requests
2
from bs4 import BeautifulSoup
3
from selenium import webdriver
4
from selenium.webdriver.chrome.options import Options
5
from selenium.webdriver.common.keys import Keys
6
7
import time
8
9
url = ["https://www.skechers.com/women/shoes/ultra-flex/12843.html",
10
"https://www.skechers.com/women/shoes/ultra-flex---twilight-twinkle/149173.html",
11
"https://www.skechers.com/women/shoes/ultra-flex---statements/12841.html",
12
"https://www.skechers.com/women/shoes/ultra-flex/12843.html",
13
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst---extreme-outlook/124580.html",
14
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst---lunar-mars/124577.html",
15
"https://www.skechers.com/women/shoes/skechers-gowalk-hyper-burst/124575.html",
16
"https://www.skechers.com/women/shoes/skechers-gowalk-stability---coco-jazz/124603.html",
17
"https://www.skechers.com/women/shoes/skechers-gowalk-stability---magnificent-glow/124602.html",
18
"https://www.skechers.com/women/shoes/skechers-gowalk-joy/124191.html",
19
"https://www.skechers.com/women/shoes/skechers-gowalk-joy---paradise/15601.html",
20
"https://www.skechers.com/women/shoes/skech-air-element-2.0---vivid-blush/149408.html",
21
"https://www.skechers.com/women/shoes/skech-air-element-2.0---kaleidoscope-splash/149134.html",
22
"https://www.skechers.com/women/shoes/skech-air-element-2.0/149062.html",
23
"https://www.skechers.com/women/shoes/skech-air-element-2.0---rising-gaze/149611.html",
24
"https://www.skechers.com/women/shoes/skechers-gowalk-air---twirl/124073.html",
25
"https://www.skechers.com/women/shoes/gowalk-air---whirl/124074.html",
26
"https://www.skechers.com/women/shoes/skechers-arch-fit-refine---her-best/104092.html",
27
"https://www.skechers.com/women/shoes/skechers-arch-fit-refine/104090.html",
28
"https://www.skechers.com/women/shoes/glide-step---stepping-up/104086.html",
29
"https://www.skechers.com/women/shoes/relaxed-fit-empire-dlux---sharp-witted/149007.html",
30
"https://www.skechers.com/women/shoes/relaxed-fit-empire-dlux---paradise-sky/149274.html",
31
"https://www.skechers.com/women/shoes/skechers-gorun-smart/128230.html"]
32
count=0
33
for endpoint in url:
34
print(count)
35
count+=1
36
options = webdriver.ChromeOptions()
37
options.add_experimental_option("excludeSwitches", ['enable-automation'])
38
options.add_argument('--disable-blink-features=AutomationControlled')
39
options.add_argument(
40
"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36")
41
#options.add_argument("--remote-debugging-port=9222")
42
43
driver = webdriver.Chrome('./chromedriver.exe', options=options)
44
driver.get(endpoint)
45
46
ISSUE:
The issue is with the remote debugging port, you already used it with one instance ,
Solution:
remove that