JavaScript
x
37
37
1
from selenium import webdriver
2
from selenium.webdriver.chrome.service import Service as ChromeService
3
from selenium.webdriver.common.by import By
4
import configparser
5
from datetime import datetime
6
7
parser = configparser.RawConfigParser()
8
parser.read('config.ini')
9
10
url= parser['PROPERTIES']['URL']
11
END_DATE = datetime.strptime(parser['DATE']['END'], '%Y-%m-%d')
12
START_DATE=datetime.strptime(parser['DATE']['START'],'%Y-%m-%d')
13
# Setting up driver options
14
options = webdriver.ChromeOptions()
15
# Setting up Path to chromedriver executable file
16
CHROMEDRIVER_PATH =r'C:UsersHPDesktopINTERNSHIPinfluensterchromedriver.exe'
17
# Adding options
18
options.add_experimental_option("excludeSwitches", ["enable-automation"])
19
options.add_experimental_option("useAutomationExtension", False)
20
# Setting up chrome service
21
service = ChromeService(executable_path=CHROMEDRIVER_PATH)
22
# Establishing Chrom web driver using set services and options
23
driver = webdriver.Chrome(service=service, options=options)
24
25
driver.get(url)
26
27
reviews=driver.find_elements_by_xpath('//*[@id="app-base"]/div[1]/div[4]/div[1]/div[1]/div[3]')
28
count=0
29
item_list = []
30
31
for review in reviews:
32
item={
33
'username': review.find_element_by_xpath(".//a[contains(@class,'name')]").text,
34
}
35
item_list.append(item)
36
print(item_list)
37
OUTPUT IS JUST ONE NAME AND NOT ALL I need to scrape all the reviews from https://www.influenster.com/reviews/loreal-paris-elvive-extraordinary-oil-deep-nourishing-shampoo-and-conditioner-set-126-fl-oz. Even I am running a loop I am getting only one username. Please help me out
Advertisement
Answer
You getting only 1 review because XPath locator you are using //*[@id="app-base"]/div[1]/div[4]/div[1]/div[1]/div[3]
returns only 1 element, so your for
loop is performed only once.
You can improve this code by improving that XPath locator.
Also you need to close the cookies banner there.
Also you need to add a wait to waif for elements to be clickable / visible before you accessing them, as following.
This should work better:
JavaScript
1
39
39
1
from selenium.webdriver.support.ui import WebDriverWait
2
from selenium.webdriver.common.by import By
3
from selenium.webdriver.support import expected_conditions as EC
4
import configparser
5
from datetime import datetime
6
7
parser = configparser.RawConfigParser()
8
parser.read('config.ini')
9
10
url= parser['PROPERTIES']['URL']
11
END_DATE = datetime.strptime(parser['DATE']['END'], '%Y-%m-%d')
12
START_DATE=datetime.strptime(parser['DATE']['START'],'%Y-%m-%d')
13
# Setting up driver options
14
options = webdriver.ChromeOptions()
15
# Setting up Path to chromedriver executable file
16
CHROMEDRIVER_PATH =r'C:UsersHPDesktopINTERNSHIPinfluensterchromedriver.exe'
17
# Adding options
18
options.add_experimental_option("excludeSwitches", ["enable-automation"])
19
options.add_experimental_option("useAutomationExtension", False)
20
# Setting up chrome service
21
service = ChromeService(executable_path=CHROMEDRIVER_PATH)
22
# Establishing Chrom web driver using set services and options
23
driver = webdriver.Chrome(service=service, options=options)
24
wait = WebDriverWait(driver, 20)
25
driver.get(url)
26
# The 2 lines below is what I actually added here + necessary imports
27
# and `wait` object initialization
28
wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#onetrust-accept-btn-handler"))).click()
29
reviews = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".conversations-left .item")))
30
count=0
31
item_list = []
32
33
for review in reviews:
34
item={
35
'username': review.find_element_by_xpath(".//a[contains(@class,'name')]").text,
36
}
37
item_list.append(item)
38
print(item_list)
39