Put the entire code into a question, thank you to all that have replied but this issue is super annoying either way help is appreciated!
Context: This code is meant to go onto the top reddit post of the day/week, then screenshot it and once that’s done it goes to the comments and screenshots the top comments of said post, the former works but the latter does not.
JavaScript
x
80
80
1
import time,utils,string
2
from selenium.webdriver.common.by import By
3
from selenium.webdriver.support.ui import WebDriverWait
4
from selenium.webdriver.support import expected_conditions as EC
5
from utils import config
6
7
def scrape(post_url):
8
bot = utils.create_bot(headless=True)
9
data = {}
10
11
try:
12
# Load cookies to prevent cookie overlay & other issues
13
bot.get('https://www.reddit.com')
14
for cookie in config['reddit_cookies'].split('; '):
15
cookie_data = cookie.split('=')
16
bot.add_cookie({'name':cookie_data[0],'value':cookie_data[1],'domain':'reddit.com'})
17
bot.get(post_url)
18
19
# Fetching the post itself, text & screenshot
20
post = WebDriverWait(bot, 20).until(EC.presence_of_element_located((By.CSS_SELECTOR, '.Post')))
21
post_text = post.find_element(By.CSS_SELECTOR, 'h1').text
22
data['post'] = post_text
23
post.screenshot('output/post.png')
24
25
# Let comments load
26
bot.execute_script("window.scrollTo(0, document.body.scrollHeight);")
27
time.sleep(3)
28
29
# Fetching comments & top level comment determinator
30
comments = WebDriverWait(bot, 20).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, 'div[id^=t1_][tabindex]')))
31
allowed_style = comments[0].get_attribute("style")
32
33
# Filter for top only comments
34
NUMBER_OF_COMMENTS = 10
35
comments = [comment for comment in comments if comment.get_attribute("style") == allowed_style][:NUMBER_OF_COMMENTS]
36
37
print('💬 Scraping comments...',end="",flush=True)
38
# Save time & resources by only fetching X content
39
for i in range(len(comments)):
40
try:
41
print('.',end="",flush=True)
42
# Filter out locked comments (AutoMod)
43
try:
44
comments[i].find_elements(By.CSS_SELECTOR, '.icon.icon-lock_fill')
45
continue
46
except:
47
pass
48
49
# Scrolling to the comment ensures that the profile picture loads
50
# Credits: https://stackoverflow.com/a/57630350
51
desired_y = (comments[i].size['height'] / 2) + comments[i].location['y']
52
window_h = bot.execute_script('return window.innerHeight')
53
window_y = bot.execute_script('return window.pageYOffset')
54
current_y = (window_h / 2) + window_y
55
scroll_y_by = desired_y - current_y
56
57
bot.execute_script("window.scrollBy(0, arguments[0]);", scroll_y_by)
58
time.sleep(0.2)
59
60
# Getting comment into string
61
text = "n".join([element.text for element in comments[i].find_elements_by_css_selector('.RichTextJSON-root')])
62
63
# Screenshot & save text
64
comments[i].screenshot(f'output/{i}.png')
65
data[str(i)] = ''.join(filter(lambda c: c in string.printable, text))
66
except Exception as e:
67
if config['debug']:
68
raise e
69
pass
70
71
if bot.session_id:
72
bot.quit()
73
return data
74
except Exception as e:
75
if bot.session_id:
76
bot.quit()
77
if config['debug']:
78
raise e
79
return False
80
Advertisement
Answer
Code was fixed by removing code which filters locked comments