I made some pretty simple script which pulls data from clicky.com api but for some reason it does not work as expected from time to time.
Sometimes it gets results but another time I am getting the following errors which I cant debug. I am fairly new to asyncio and aiohttp
Traceback (most recent call last):
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/1_stable/asy/usable/baza_goals.py", line 118, in <module>
goals_results_last_week = asyncio.run(goals_clicky_results_last_week())
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 641, in run_until_complete
return future.result()
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/1_stable/asy/usable/baza_goals.py", line 82, in goals_clicky_results_last_week
responses_clicky_goals = await asyncio.gather(*tasks_goals)
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/client.py", line 1122, in send
return self._coro.send(arg)
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/client.py", line 535, in _request
conn = await self._connector.connect(
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 542, in connect
proto = await self._create_connection(req, traces, timeout)
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 907, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 1175, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 986, in _wrap_create_connection
return await self._loop.create_connection(*args, **kwargs) # type: ignore[return-value] # noqa
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 1040, in create_connection
sock = await self._connect_sock(
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 954, in _connect_sock
await self.sock_connect(sock, address)
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/selector_events.py", line 502, in sock_connect
return await fut
RuntimeError: await wasn't used with future
How to debug this? Whats the problem here?
edited: Here is my code for you to test:
import asyncio
import datetime
import aiohttp
import requests
start_operacji = datetime.datetime.now()
print('start', start_operacji)
date_filename = datetime.datetime.now().strftime('%d-%m_%H-%M')
def goals_clicky_tasks_last_week(session):
tasks_clicky_goals = []
# todo to mozna by jeszcze dać do asyncio
clicky_auth = requests.get(
'https://api.clicky.com/api/account/sites?username=meeffe&password=hAs!23$5cy&output=json')
auth_jsonised = clicky_auth.json()
list_site_id_sitekey_dict = []
for k in auth_jsonised:
site_id_sitekey_dict = {'site_id': k['site_id'], 'sitekey': k['sitekey']}
list_site_id_sitekey_dict.append(site_id_sitekey_dict)
for auth_item in list_site_id_sitekey_dict:
goal_url = f"https://api.clicky.com/api/stats/4?site_id={auth_item['site_id']}&sitekey={auth_item['sitekey']}&type=goals&date=today&limit=1000&output=json"
tasks_clicky_goals.append(asyncio.ensure_future(session.get(goal_url, ssl=False)))
return tasks_clicky_goals
async def goals_clicky_results_last_week():
list_final_goals = []
async with aiohttp.ClientSession() as session:
tasks_goals = goals_clicky_tasks_last_week(session)
responses_clicky_goals = await asyncio.gather(*tasks_goals)
for response_clicky_goal in responses_clicky_goals:
clicky_data = await response_clicky_goal.json(content_type=None)
goals_list = []
for url_item_goal in clicky_data[0]['dates'][0]['items']:
if url_item_goal['conversion'] != '':
if url_item_goal['title'].startswith(
'http'): # nie bierze pod uwagę goalsów które zawierają U - https:// etc
goals_dict = {'url': url_item_goal['title'].replace('http://', 'https://'),
'goals': url_item_goal['value'],
'ad_ctr': url_item_goal['conversion']
}
goals_list.append(goals_dict)
else:
continue
else:
continue
list_final_goals.append(goals_list)
flattened_list_final_goals = [val for sublist in list_final_goals for val in sublist]
return flattened_list_final_goals
print(asyncio.run(goals_clicky_results_last_week()), 'goals_clicky_results_last_week')
goals_results_last_week = asyncio.run(goals_clicky_results_last_week())
######################################################################
end = datetime.datetime.now() - start_operacji
print('Ready:)!')
print('It took: ', end)
Advertisement
Answer
I actually found a solution by myself.
- Instead of aiohttp I used httpx
- I used timeout with every request
- I removed unnecessary await
Changes in an original code below. Now the script run 100% stable. To be frank I am not sure which of these changes had the biggest impact but it works as expected.
timeout = httpx.Timeout(60.0, connect=60.0)
async with httpx.AsyncClient(verify=False, timeout=timeout) as session:
tasks_goals = goals_clicky_tasks_last_week(session)
responses_clicky_goals = await asyncio.gather(*tasks_goals)
for response_clicky_goal in responses_clicky_goals:
clicky_data = response_clicky_goal.json()
...