I wrote this decorator to be used by two functions:
JavaScript
x
53
53
1
def next_page(func) -> List[Dict]:
2
"""
3
Decorator to re-run api call if number of objects returned is 100.
4
SmartSurvey API can only return objects for 1 "page" at each call.
5
One page stores 100 objects. By passing the argument page += 1 you can get
6
the next 100 records till the call return less than 100.
7
8
:param func:
9
:return: list of dictionaries
10
"""
11
12
@functools.wraps(func)
13
def wrapper(ASO, page, **kwargs):
14
list_of_records = []
15
partial_list = func(ASO, page, **kwargs)
16
list_of_records.extend(partial_list)
17
while len(partial_list) == 100:
18
page += 1
19
partial_list = func(ASO, page, **kwargs)
20
list_of_records.extend(partial_list)
21
return list_of_records
22
23
return wrapper
24
25
26
@next_page
27
def _get_responses(ASO, page, survey_id, since) -> List[Dict]:
28
"""
29
Get responses from SmartSurvey API based on survey id.
30
:param ASO: API object
31
:param survey_id: INTEGER
32
:param page: INTEGER
33
:param since: INTEGER (unix epoch)
34
:return: list of dictionaries
35
"""
36
responses = ASO.get_responses(page=page, survey_id=survey_id,
37
since=since, page_size=100)
38
39
return responses
40
41
42
@next_page
43
def _get_surveys(ASO, page) -> List[Dict]:
44
"""
45
Get surveys from SmartSurvey API.
46
:param ASO: API object
47
:param page: INTEGER
48
:return: list of dictionaries
49
"""
50
surveys = ASO.get_surveys(page=page, page_size=100)
51
52
return surveys
53
I keep getting the error:
JavaScript
1
9
1
Traceback (most recent call last):
2
File "/Users/asportelli/gitRepos/smart-survey/scripts/api_to_s3_local.py", line 206, in <module>
3
main()
4
File "/Users/asportelli/gitRepos/smart-survey/scripts/api_to_s3_local.py", line 202, in main
5
responses_to_s3(s3, API, survey_meta_data, _last_n_hours(1))
6
File "/Users/asportelli/gitRepos/smart-survey/scripts/api_to_s3_local.py", line 160, in responses_to_s3
7
since=since)
8
TypeError: wrapper() got multiple values for argument 'page'
9
How can I pass the argument page recursively without having it multiple times?
Thanks!
EDIT:
functions are called as in:
JavaScript
1
45
45
1
def surveys_metadata(ASO) -> pd.DataFrame:
2
"""
3
get id and last updated ts of all surveys
4
5
:param ASO: Api Scraper Object
6
:param from_ts: unix epoch timestamp
7
:return: pandas df
8
"""
9
dfs = []
10
surveys = _get_surveys(ASO, page=1)
11
for survey in surveys:
12
df = pd.DataFrame(
13
{'id': [survey['id']],
14
'date_modified': [survey['date_modified']]})
15
dfs.append(df)
16
output = pd.concat(dfs)
17
logger.info(f"surveys_found: {len(output)}")
18
19
return output
20
21
22
def responses_to_s3(s3, ASO, df, since=_last_n_hours(24 * 1)):
23
"""
24
Write SmartSurvey responses to AWS S3 bucket.
25
:param s3: boto3 client
26
:param ASO: API object
27
:param df: smart survey metadata (pd.DataFrame)
28
:param since: INTEGER (unix epoch)
29
30
:return:
31
"""
32
for survey_id in df['id']:
33
responses = _get_responses(ASO, survey_id, page=1,
34
since=since)
35
36
logger.info(f"Extracted {len(responses)} from survey: {survey_id}")
37
for response in responses:
38
response_id = response['id']
39
response_ts = response['date_modified']
40
date = str(response['date_modified'])[:10]
41
_dict_to_s3(s3, 'dundata-nonprod-lake',
42
f'to_process/smart-survey/responses/{date}/'
43
f'{survey_id}-{response_id}-{response_ts}.json',
44
response)
45
Advertisement
Answer
Positional arguments in python must be passed in order of definition in the function prototype.
This line passes the value of survey_id
to the argument page
as it’s the second argument passed.
JavaScript
1
3
1
responses = _get_responses(ASO, survey_id, page=1,
2
since=since)
3
Call the function as:
JavaScript
1
3
1
responses = _get_responses(ASO, 1, survey_id=survey_id,
2
since=since)
3