Skip to content
Advertisement

Scroll in Selenium Webdriver (Python)

Prerequisites.
You need an account at Instagram to use this script.
Setup a test environment:

Log in, open the needed list(works correctly):

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep

driver = webdriver.Chrome(
# driver = webdriver.Firefox(    
# driver = webdriver.PhantomJS(
    service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])

driver.get("https://instagram.com/accounts/login")
username = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")

username1 = 'instagram' # change it!
password1 = 'instagrampassword1' # change it!

username.send_keys(username1)
password.send_keys(password1)

submit_button = driver.find_element_by_css_selector(
    '#react-root > div > article > div > div:nth-child(1) > div > form > span > button')
submit_button.click()

sleep(2)

link = 'https://www.instagram.com/youtube/'
driver.get(link)

driver.implicitly_wait(2)
driver.find_elements_by_class_name("_218yx")[2].click()

Wrong scroll. How to fix this block?

How to focus and scroll correctly on this page?

My attempts:

driver.find_element_by_class_name("_cx1ua").send_keys(Keys.NULL) # focus
      #The element has been deleted entirely or
      #The element is no longer attached to the DOM.

driver.find_element_by_class_name("_q44m8").send_keys(Keys.NULL)
# cannot focus element

driver.find_element_by_class_name("_qjr85").send_keys(Keys.NULL) 
# cannot focus element


for i in range(5):
    driver.find_element_by_class_name("_cx1ua").send_keys(Keys.END)

=============================================================
to @Moshisho :

We need to focus on some element to activate it.
The question is what the element we need to choose to focus and how?
This is not a “body”:
something like that, but not this:

background = driver.find_element_by_css_selector("body")
# background = driver.find_element_by_css_selector("div._2uju6")

for i in range(5):
    background.send_keys(Keys.SPACE)
    time.sleep(1)

Without it this command do not work.

to @Naveen :

print(driver.find_element_by_css_selector("div._a1rcs").location_once_scrolled_into_view) # {'x': 0, 'y': 0}
print(driver.find_element_by_class_name("_cx1ua").location_once_scrolled_into_view) # {'x': 376, 'y': 229}
print(driver.find_element_by_class_name("_q44m8").location_once_scrolled_into_view) # {'x': 376, 'y': 180}
print(driver.find_element_by_class_name("_qjr85").location_once_scrolled_into_view) # {'x': 376, 'y': 180}

And what’s next?

driver.execute_script("window.scrollTo(0, 3000);") # do not working

https://www.instagram.com/youtube/following/

Advertisement

Answer

Try the following code:

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from time import sleep
from selenium.webdriver.support.ui import Select

driver = webdriver.Chrome(
# driver = webdriver.Firefox(    
# driver = webdriver.PhantomJS(
    service_args=['--ignore-ssl-errors=true', '--ssl-protocol=any'])

driver.maximize_window()

driver.get("https://instagram.com/accounts/login")
username = driver.find_element_by_name("username")
password = driver.find_element_by_name("password")

username1 = 'instagramlogin1'    # change it!
password1 = 'instagrampassword1' # change it!

username.send_keys(username1)
password.send_keys(password1)

submit_button = driver.find_element_by_css_selector(
    '#react-root > div > article > div > div:nth-child(1) > div > form > span > button')
submit_button.click()

sleep(2)

link = 'https://www.instagram.com/youtube/'
driver.get(link)

driver.implicitly_wait(2)
following = driver.find_element_by_xpath("//a[@href='/youtube/following/']/span")
total_following = int(following.text)
print "total no. of users following: ", total_following
# click on 239 following, displays 10 users
following.click()

loaded_following = driver.find_elements_by_xpath("//ul[@class='_539vh _4j13h']/li")
loaded_till_now = len(loaded_following)

while(loaded_till_now<total_following):
    print "following users loaded till now: ", loaded_till_now
    print loaded_following[loaded_till_now-1]
    loaded_following[loaded_till_now-1].location_once_scrolled_into_view
    # driver.execute_script("arguments[0].focus();", loaded_following[loaded_till_now-1])
    driver.find_element_by_tag_name('body').send_keys(Keys.END) # triggers AJAX request to load more users. observed that loading 10 users at a time.
    sleep(1) # tried wihtout sleep but throws StaleElementReferenceException. As it takes time to get the resposne and update the DOM
    loaded_following = driver.find_elements_by_xpath("//ul[@class='_539vh _4j13h']/li")
    loaded_till_now = len(loaded_following)

# All 239 users are loaded. 
driver.quit()

Observed that browser is sending AJAX request to load more users. this action is triggered when you scroll using mouse or enter Space or End keys

Advertisement