I’m trying to create a script using python that separate 2 kind of websites , the one with SPF included and the others with SPF , and classify them using python, so in the beginning it worked perfectly but these daysit gives me a message error that I don’t find a clue about it
from selenium import webdriver from selenium.webdriver.chrome.options import Options from concurrent.futures import ThreadPoolExecutor import re import requests import json from datetime import datetime from colorama import Fore, Back, Style import colorama from webdriver_manager.chrome import ChromeDriverManager colorama.init() def checkCaptchaPresent(driver): captchaFound = True while captchaFound: try: driver.find_element_by_id("captcha-form") driver.set_window_position(0, 0) except: driver.set_window_position(20000, 0) captchaFound = False return 0 def requestSPF(url): response = requests.get("https://api.sparkpost.com/api/v1/messaging-tools/spf/query?domain={}".format(url)).json() for error in response['results']['errors']: if "does not have an SPF record" in error['message']: print(Fore.RED + "{} does not have an SPF record".format(url)) return [url] print(Fore.GREEN + "{} have an SPF record".format(url)) return [] chrome_options = Options() PATH = "webdriver/chromedriver.exe" chrome_options.add_argument('--no-sandbox') chrome_options.add_argument('--window-size=1000,1000') chrome_options.add_argument('log-level=3') # chrome_options.add_argument("--user-data-dir=SFPInspector") while True: driver = webdriver.Chrome(ChromeDriverManager().install(),options=chrome_options) driver.set_window_position(20000, 0) search_term = input("Enter search term: ") number_results = int(input("Max number of url to scrape: ")) language_code = "en" driver.get('https://www.google.com/search?q={}&num={}&hl={}'.format(search_term, number_results+1, language_code)) print('https://www.google.com/search?q={}&num={}&hl={}'.format(search_term, number_results+1, language_code)) checkCaptchaPresent(driver) urls = driver.find_elements_by_xpath("//div[@id='search']/div/div/div[@class='g']//div[@class='yuRUbf']/a") websiteLink = [] for url in urls: scrappedURL = url.get_attribute('href') print(scrappedURL) websiteLink.append(scrappedURL) filteredURL = [] for i, url in enumerate(websiteLink): match = re.compile("^http.*com[/]") matchedURL = match.findall(url) filteredURL += matchedURL filteredURL = [url.replace('https:', '').replace('http:', '').replace('/', '') for url in filteredURL] noSPFURL = [] with ThreadPoolExecutor(max_workers=int(10)) as pool: for res in pool.map(requestSPF, filteredURL): noSPFURL += res print(Style.RESET_ALL) driver.close() fileName = datetime.now().strftime("%d%m%Y-%H%M") print("Saving reports: report/{}_AllSite_{}.txt".format(''.join(e for e in search_term if e.isalnum()), fileName)) with open('report/{}_AllSite_{}.txt'.format(''.join(e for e in search_term if e.isalnum()), fileName), 'w') as filehandle: for link in websiteLink: filehandle.write("{}n".format(link)) print("Saving reports: report/{}_NoSPF_{}.txt".format(''.join(e for e in search_term if e.isalnum()), fileName)) with open('report/{}_NoSPF_{}.txt'.format(''.join(e for e in search_term if e.isalnum()), fileName), 'w') as filehandle: for link in noSPFURL: filehandle.write("{}n".format(link))
The output message is as follows:
====== WebDriver manager ====== Could not get version for google-chrome with the command: powershell "$ErrorActionPreference='silentlycontinue' ; (Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion ; if (-not $? -or $? -match $error) { (Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { (Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version } if (-not $? -or $? -match $error) { reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version }" Current google-chrome version is UNKNOWN Get LATEST chromedriver version for UNKNOWN google-chrome Trying to download new driver from https://chromedriver.storage.googleapis.com/98.0.4758.102/chromedriver_win32.zip Driver has been saved in cache [C:Usersdell.wdmdriverschromedriverwin3298.0.4758.102] Enter search term:
Advertisement
Answer
This error message…
====== WebDriver manager ====== Could not get version for google-chrome with the command: powershell "$ErrorActionPreference='silentlycontinue' ; (Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion ; if (-not $? -or $? -match $error) { (Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { (Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion } if (-not $? -or $? -match $error) { reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version } if (-not $? -or $? -match $error) { reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version }" Current google-chrome version is UNKNOWN Get LATEST chromedriver version for UNKNOWN google-chrome
…implies that the Webdriver Manager was unable to retrieve the version of the installed google-chrome browser within the system through any of the below powershell commands and registry query:
Get-Item -Path "$env:PROGRAMFILESGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
Get-Item -Path "$env:PROGRAMFILES(x86)GoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
Get-Item -Path "$env:LOCALAPPDATAGoogleChromeApplicationchrome.exe").VersionInfo.FileVersion
reg query "HKCUSOFTWAREGoogleChromeBLBeacon" /v version
reg query "HKLMSOFTWAREWow6432NodeMicrosoftWindowsCurrentVersionUninstallGoogle Chrome" /v version
Deep Dive
This issue was discussed in details in the following discussions:
- google-chrome version is UNKNOWN with webdriver_manager 3.5.3 (Win)
- base64 encode powershell commands
and was finally fixed through fix powershell determination and str concatenation
Solution
Upgrade webdriver-manager to the latest version of webdriver-manager 3.5.4