I’m trying to create a script using python that separate 2 kind of websites , the one with SPF included and the others with SPF , and classify them using python, so in the beginning it worked perfectly but these daysit gives me a message error that I don’t find a clue about it
JavaScript
x
85
85
1
from selenium import webdriver
2
from selenium.webdriver.chrome.options import Options
3
from concurrent.futures import ThreadPoolExecutor
4
import re
5
import requests
6
import json
7
from datetime import datetime
8
from colorama import Fore, Back, Style
9
import colorama
10
from webdriver_manager.chrome import ChromeDriverManager
11
12
colorama.init()
13
def checkCaptchaPresent(driver):
14
captchaFound = True
15
while captchaFound:
16
try:
17
driver.find_element_by_id("captcha-form")
18
driver.set_window_position(0, 0)
19
except:
20
driver.set_window_position(20000, 0)
21
captchaFound = False
22
return 0
23
24
def requestSPF(url):
25
response = requests.get("https://api.sparkpost.com/api/v1/messaging-tools/spf/query?domain={}".format(url)).json()
26
27
for error in response['results']['errors']:
28
if "does not have an SPF record" in error['message']:
29
print(Fore.RED + "{} does not have an SPF record".format(url))
30
return [url]
31
32
print(Fore.GREEN + "{} have an SPF record".format(url))
33
return []
34
35
chrome_options = Options()
36
PATH = "webdriver/chromedriver.exe"
37
chrome_options.add_argument('--no-sandbox')
38
chrome_options.add_argument('--window-size=1000,1000')
39
chrome_options.add_argument('log-level=3')
40
# chrome_options.add_argument("--user-data-dir=SFPInspector")
41
42
while True:
43
driver = webdriver.Chrome(ChromeDriverManager().install(),options=chrome_options)
44
driver.set_window_position(20000, 0)
45
search_term = input("Enter search term: ")
46
number_results = int(input("Max number of url to scrape: "))
47
language_code = "en"
48
49
driver.get('https://www.google.com/search?q={}&num={}&hl={}'.format(search_term, number_results+1, language_code))
50
print('https://www.google.com/search?q={}&num={}&hl={}'.format(search_term, number_results+1, language_code))
51
checkCaptchaPresent(driver)
52
urls = driver.find_elements_by_xpath("//div[@id='search']/div/div/div[@class='g']//div[@class='yuRUbf']/a")
53
websiteLink = []
54
for url in urls:
55
scrappedURL = url.get_attribute('href')
56
print(scrappedURL)
57
websiteLink.append(scrappedURL)
58
59
filteredURL = []
60
for i, url in enumerate(websiteLink):
61
match = re.compile("^http.*com[/]")
62
matchedURL = match.findall(url)
63
64
filteredURL += matchedURL
65
66
filteredURL = [url.replace('https:', '').replace('http:', '').replace('/', '') for url in filteredURL]
67
68
noSPFURL = []
69
with ThreadPoolExecutor(max_workers=int(10)) as pool:
70
for res in pool.map(requestSPF, filteredURL):
71
noSPFURL += res
72
73
print(Style.RESET_ALL)
74
driver.close()
75
fileName = datetime.now().strftime("%d%m%Y-%H%M")
76
print("Saving reports: report/{}_AllSite_{}.txt".format(''.join(e for e in search_term if e.isalnum()), fileName))
77
with open('report/{}_AllSite_{}.txt'.format(''.join(e for e in search_term if e.isalnum()), fileName), 'w') as filehandle:
78
for link in websiteLink:
79
filehandle.write("{}n".format(link))
80
81
print("Saving reports: report/{}_NoSPF_{}.txt".format(''.join(e for e in search_term if e.isalnum()), fileName))
82
with open('report/{}_NoSPF_{}.txt'.format(''.join(e for e in search_term if e.isalnum()), fileName), 'w') as filehandle:
83
for link in noSPFURL:
84
filehandle.write("{}n".format(link))
85
The output message is as follows:
JavaScript
1
8
1
====== WebDriver manager ======
2
Could not get version for google-chrome with the command: powershell "$ErrorActionPreference='silentlycontinue' ; (Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion ; if (-not $? -or $? -match $error) { (Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { (Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version } if (-not $? -or $? -match $error) { reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version }"
3
Current google-chrome version is UNKNOWN
4
Get LATEST chromedriver version for UNKNOWN google-chrome
5
Trying to download new driver from https://chromedriver.storage.googleapis.com/98.0.4758.102/chromedriver_win32.zip
6
Driver has been saved in cache [C:Usersdell.wdmdriverschromedriverwin3298.0.4758.102]
7
Enter search term:
8
Advertisement
Answer
This error message…
JavaScript
1
5
1
====== WebDriver manager ======
2
Could not get version for google-chrome with the command: powershell "$ErrorActionPreference='silentlycontinue' ; (Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion ; if (-not $? -or $? -match $error) { (Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { (Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version } if (-not $? -or $? -match $error) { reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version }"
3
Current google-chrome version is UNKNOWN
4
Get LATEST chromedriver version for UNKNOWN google-chrome
5
…implies that the Webdriver Manager was unable to retrieve the version of the installed google-chrome browser within the system through any of the below powershell commands and registry query:
Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version
reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version
Deep Dive
This issue was discussed in details in the following discussions:
- google-chrome version is UNKNOWN with webdriver_manager 3.5.3 (Win)
- base64 encode powershell commands
and was finally fixed through fix powershell determination and str concatenation
Solution
Upgrade webdriver-manager to the latest version of webdriver-manager 3.5.4