when I am passing manually json file as key to load in python shell then its working fine. code below
import os import json import boto3 import io import requests import botocore bucket_name = 'dev-data' folder_name = 'raw/test/' key_source = 'raw/test/extract_api_20200719.json' s3_client = boto3.client('s3') json_obj = s3_client.get_object(Bucket=bucket_name, Key=key_source) json_data = json_obj["Body"].read().decode('utf-8') print("############################json_data####################### :", json_data ) print("############################json_data_type################## :", type(json_data)) json_dict = json.loads(json_data) print("############################json_dict####################### :", json_dict ) print("############################json_dict_type ################# :", type(json_dict))
However when using for loop to read JSON object from s3 bucket, then I am getting the error
import os import json import boto3 import io import requests import botocore bucket_name = 'dev-data' folder_name = 'raw/test/' s3_resource = boto3.resource('s3') bucket = s3_resource.Bucket(bucket_name) for obj in bucket.objects.filter(Prefix=folder_name): print('Object to extract :', obj) print('obj key: ', obj.key) s3_client = boto3.client('s3') json_obj = s3_client.get_object(Bucket=bucket_name, Key=obj.key) json_data = json_obj["Body"].read().decode('utf-8') json_dict = json.loads(json_data) error: json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Advertisement
Answer
Some of the entries in bucket.objects
don’t have any JSON in the data, so check for that and skip them.
for obj in bucket.objects.filter(Prefix=folder_name): print('Object to extract :', obj) print('obj key: ', obj.key) s3_client = boto3.client('s3') json_obj = s3_client.get_object(Bucket=bucket_name, Key=obj.key) json_data = json_obj["Body"].read().decode('utf-8') if not json_data: print("Skipping empty", obj.key) continue json_dict = json.loads(json_data)