I made some pretty simple script which pulls data from clicky.com api but for some reason it does not work as expected from time to time.
Sometimes it gets results but another time I am getting the following errors which I cant debug. I am fairly new to asyncio and aiohttp
JavaScript
x
29
29
1
Traceback (most recent call last):
2
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/1_stable/asy/usable/baza_goals.py", line 118, in <module>
3
goals_results_last_week = asyncio.run(goals_clicky_results_last_week())
4
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/runners.py", line 44, in run
5
return loop.run_until_complete(main)
6
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 641, in run_until_complete
7
return future.result()
8
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/1_stable/asy/usable/baza_goals.py", line 82, in goals_clicky_results_last_week
9
responses_clicky_goals = await asyncio.gather(*tasks_goals)
10
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/client.py", line 1122, in send
11
return self._coro.send(arg)
12
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/client.py", line 535, in _request
13
conn = await self._connector.connect(
14
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 542, in connect
15
proto = await self._create_connection(req, traces, timeout)
16
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 907, in _create_connection
17
_, proto = await self._create_direct_connection(req, traces, timeout)
18
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 1175, in _create_direct_connection
19
transp, proto = await self._wrap_create_connection(
20
File "/Users/almeco/Downloads/python/_projekty/projekt_baza_review/venv/lib/python3.10/site-packages/aiohttp/connector.py", line 986, in _wrap_create_connection
21
return await self._loop.create_connection(*args, **kwargs) # type: ignore[return-value] # noqa
22
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 1040, in create_connection
23
sock = await self._connect_sock(
24
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/base_events.py", line 954, in _connect_sock
25
await self.sock_connect(sock, address)
26
File "/Library/Frameworks/Python.framework/Versions/3.10/lib/python3.10/asyncio/selector_events.py", line 502, in sock_connect
27
return await fut
28
RuntimeError: await wasn't used with future
29
How to debug this? Whats the problem here?
edited: Here is my code for you to test:
JavaScript
1
83
83
1
import asyncio
2
import datetime
3
4
import aiohttp
5
import requests
6
7
8
start_operacji = datetime.datetime.now()
9
print('start', start_operacji)
10
11
12
date_filename = datetime.datetime.now().strftime('%d-%m_%H-%M')
13
14
def goals_clicky_tasks_last_week(session):
15
tasks_clicky_goals = []
16
# todo to mozna by jeszcze dać do asyncio
17
clicky_auth = requests.get(
18
'https://api.clicky.com/api/account/sites?username=meeffe&password=hAs!23$5cy&output=json')
19
auth_jsonised = clicky_auth.json()
20
21
list_site_id_sitekey_dict = []
22
23
for k in auth_jsonised:
24
site_id_sitekey_dict = {'site_id': k['site_id'], 'sitekey': k['sitekey']}
25
list_site_id_sitekey_dict.append(site_id_sitekey_dict)
26
27
for auth_item in list_site_id_sitekey_dict:
28
goal_url = f"https://api.clicky.com/api/stats/4?site_id={auth_item['site_id']}&sitekey={auth_item['sitekey']}&type=goals&date=today&limit=1000&output=json"
29
tasks_clicky_goals.append(asyncio.ensure_future(session.get(goal_url, ssl=False)))
30
31
return tasks_clicky_goals
32
33
34
async def goals_clicky_results_last_week():
35
list_final_goals = []
36
37
async with aiohttp.ClientSession() as session:
38
39
tasks_goals = goals_clicky_tasks_last_week(session)
40
41
responses_clicky_goals = await asyncio.gather(*tasks_goals)
42
43
for response_clicky_goal in responses_clicky_goals:
44
45
clicky_data = await response_clicky_goal.json(content_type=None)
46
47
goals_list = []
48
for url_item_goal in clicky_data[0]['dates'][0]['items']:
49
if url_item_goal['conversion'] != '':
50
if url_item_goal['title'].startswith(
51
'http'): # nie bierze pod uwagę goalsów które zawierają U - https:// etc
52
goals_dict = {'url': url_item_goal['title'].replace('http://', 'https://'),
53
'goals': url_item_goal['value'],
54
'ad_ctr': url_item_goal['conversion']
55
}
56
goals_list.append(goals_dict)
57
else:
58
continue
59
else:
60
continue
61
62
list_final_goals.append(goals_list)
63
64
flattened_list_final_goals = [val for sublist in list_final_goals for val in sublist]
65
66
return flattened_list_final_goals
67
68
print(asyncio.run(goals_clicky_results_last_week()), 'goals_clicky_results_last_week')
69
goals_results_last_week = asyncio.run(goals_clicky_results_last_week())
70
71
72
######################################################################
73
74
75
76
end = datetime.datetime.now() - start_operacji
77
78
print('Ready:)!')
79
print('It took: ', end)
80
81
82
83
Advertisement
Answer
I actually found a solution by myself.
- Instead of aiohttp I used httpx
- I used timeout with every request
- I removed unnecessary await
Changes in an original code below. Now the script run 100% stable. To be frank I am not sure which of these changes had the biggest impact but it works as expected.
JavaScript
1
14
14
1
timeout = httpx.Timeout(60.0, connect=60.0)
2
3
async with httpx.AsyncClient(verify=False, timeout=timeout) as session:
4
5
tasks_goals = goals_clicky_tasks_last_week(session)
6
7
responses_clicky_goals = await asyncio.gather(*tasks_goals)
8
9
for response_clicky_goal in responses_clicky_goals:
10
11
clicky_data = response_clicky_goal.json()
12
13
14