in glam/api/management/commands/import_user_counts.py [0:0]
def handle(self, bucket, *args, **options):
self.gcs_client = storage.Client()
blobs = self.gcs_client.list_blobs(bucket)
blobs = list(
filter(lambda b: b.name.startswith("glam-extract-firefox-counts"), blobs)
)
for blob in blobs:
# Create temp table for data.
tmp_table = "tmp_import_desktop_counts"
log(f"Creating temp table for import: {tmp_table}.")
with connection.cursor() as cursor:
cursor.execute(f"DROP TABLE IF EXISTS {tmp_table}")
cursor.execute(f"CREATE TABLE {tmp_table} (LIKE glam_firefox_counts)")
cursor.execute(f"ALTER TABLE {tmp_table} DROP COLUMN id")
# Download CSV file to local filesystem.
fp = tempfile.NamedTemporaryFile()
log(f"Copying GCS file {blob.name} to local file {fp.name}.")
blob.download_to_filename(fp.name)
# Load CSV into temp table & insert data from temp table into
# aggregation tables, using upserts.
self.import_file(tmp_table, fp)
# Drop temp table and remove file.
log("Dropping temp table.")
with connection.cursor() as cursor:
cursor.execute(f"DROP TABLE {tmp_table}")
log(f"Deleting local file: {fp.name}.")
fp.close()