forerad

commit 85101ea4d7294416de40d37fc371b34772432f92
parent de837e5227079b4adb8fa408e6e24772d1906574
Author: Steve Gattuso <steve@stevegattuso.me>
Date:   Mon,  6 Nov 2023 15:44:24 +0100

fix s3 client

Diffstat:
M
forerad/scrapers/historical.py
 | 
13
+++++++------

1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/forerad/scrapers/historical.py b/forerad/scrapers/historical.py
@@ -19,10 +19,6 @@ if not CACHE_DIR.exists():
     CACHE_DIR.mkdir()
     utils.logger.debug('Initializing .cache dir')
 
-def __get_s3_client():
-    config = bclient.Config(signature_version=UNSIGNED)
-    return boto3.client('s3', config=config)
-
 
 class HistoricalTripArchive():
     object_key: str
@@ -33,6 +29,11 @@ class HistoricalTripArchive():
         self.date = date
 
     @classmethod
+    def __get_s3_client(cls):
+        config = bclient.Config(signature_version=UNSIGNED)
+        return boto3.client('s3', config=config)
+
+    @classmethod
     def from_s3(cls, obj: dict):
         """
         Converts an S3 object dictionary into a HistoricalTripArchive record, returning
@@ -69,7 +70,7 @@ class HistoricalTripArchive():
         Returns a list of HistoricalTripArchive objects that are available on
         S3 but may or may not be cached locally.
         """
-        s3 = __get_s3_client()
+        s3 = cls.__get_s3_client()
         resp = s3.list_objects_v2(Bucket=TRIP_BUCKET)
 
         archives = [HistoricalTripArchive.from_s3(o) for o in resp['Contents']]
@@ -109,7 +110,7 @@ class HistoricalTripArchive():
         blob = self.__fetch_cached_blob()
         if blob is None:
             utils.logger.info(f"Fetching {self.csv_name} from S3")
-            s3 = __get_s3_client()
+            s3 = self.__get_s3_client()
             resp = s3.get_object(Bucket=TRIP_BUCKET, Key=self.object_key)
             blob = io.BytesIO(resp['Body'].read())
             self.__store_blob(blob)