What is the best way to retry async API calls using aiohttp? I want to retry the request for the standard socket error, timeout error, etc as well as for certain status codes 500, 501
. I have tried using aiohttp_async, but can’t get it to work:
import asyncio from aiohttp import ClientSession from aiohttp_retry import RetryClient # Async single retry fetch async def async_retry_fetch(url, retry_client): async with retry_client.get(url, retry_attempts=3, retry_for_status=[500, 501]) as response: try: data = await response.json() except Exception as e: raise Exception("Could not convert json") return data async def main(): urls = [ "https://httpstat.us/200", "https://httpstat.us/500" ] api_calls = [] async with ClientSession() as session: retry_client = RetryClient(session) for url in urls: api_calls.append(async_retry_fetch(url, retry_client)) res = await asyncio.gather(*api_calls, return_exceptions=True) print("RESULT", res) asyncio.run(main())
Output:
RESULT [AttributeError("'ClientSession' object has no attribute 'debug'"), AttributeError("'ClientSession' object has no attribute 'debug'")]
Advertisement
Answer
It looks like you have installed aiohttp_retry
version 2.x
but you use arguments for version 1.2 and this gives AttributeError
.
Old version could use
get(..., retry_attempts=3)
but new version needs
get(..., retry_options=ExponentialRetry(attempts=3)
or
RetryClient(..., retry_options=ExponentialRetry(attempts=3))
But attempts
in RetryClient
has default value 3
so you could skip it.
Other problem is that RetryClient()
can’t get session
as parameter.
And it doesn’t need it because it creates own ClientSession()
in __init__
See source code
This works for me:
import asyncio from aiohttp_retry import RetryClient, ExponentialRetry class MyLogger(): def debug(self, *args, **kwargs): print('[debug]:', *args, **kwargs) async def async_retry_fetch(url, retry_client): retry_options = ExponentialRetry(attempts=3) #async with retry_client.get(url) as response: # OR async with retry_client.get(url, retry_options=ExponentialRetry(attempts=3), raise_for_status=[500, 501]) as response: try: data = await response.json() except Exception as e: raise Exception("Could not convert json") return data async def main(): urls = [ "https://httpstat.us/200", "https://httpstat.us/500", "https://httpstat.us/501", "https://httpbin.org/status/500", "https://httpbin.org/status/501", "https://httpbin.org/json" ] #async with RetryClient(logger=MyLogger(), retry_options=ExponentialRetry(attempts=3), raise_for_status=[500, 501]) as retry_client: # OR async with RetryClient(logger=MyLogger()) as retry_client: api_calls = [] for url in urls: api_calls.append(async_retry_fetch(url, retry_client)) res = await asyncio.gather(*api_calls, return_exceptions=True) for item in res: print(item) print('---') # --- start --- asyncio.run(main())
Result:
[debug]: Attempt 0 out of 3 [debug]: Attempt 0 out of 3 [debug]: Attempt 0 out of 3 [debug]: Attempt 0 out of 3 [debug]: Attempt 0 out of 3 [debug]: Attempt 0 out of 3 [debug]: Attempt 1 out of 3 [debug]: Attempt 1 out of 3 [debug]: Attempt 1 out of 3 [debug]: Attempt 1 out of 3 [debug]: Attempt 2 out of 3 [debug]: Attempt 2 out of 3 [debug]: Attempt 2 out of 3 [debug]: Attempt 2 out of 3 Could not convert json --- 500, message='Internal Server Error', url=URL('https://httpstat.us/500') --- 501, message='Not Implemented', url=URL('https://httpstat.us/501') --- 500, message='INTERNAL SERVER ERROR', url=URL('https://httpbin.org/status/500') --- 501, message='NOT IMPLEMENTED', url=URL('https://httpbin.org/status/501') --- {'slideshow': {'author': 'Yours Truly', 'date': 'date of publication', 'slides': [{'title': 'Wake up to WonderWidgets!', 'type': 'all'}, {'items': ['Why <em>WonderWidgets</em> are great', 'Who <em>buys</em> WonderWidgets'], 'title': 'Overview', 'type': 'all'}], 'title': 'Sample Slide Show'}}