def build_bundles()

in cronjobs/src/commands/build_bundles.py [0:0]

• 94 lines of code
• 31 McCabe index (conditional complexity)

def build_bundles(event, context):
    """
    Main command entry point that:
    - fetches all collections changesets
    - builds a `changesets.json.mozlz4`
    - builds a `startup.json.mozlz4`
    - fetches attachments of all collections with bundle flag
    - builds `{bid}--{cid}.zip` for each of them
    - send the bundles to the Cloud storage bucket
    """
    rs_server = event.get("server") or SERVER

    client = KintoClient(server_url=rs_server)

    base_url = client.server_info()["capabilities"]["attachments"]["base_url"]

    all_changesets = fetch_all_changesets(client)

    # Build all archives in temp directory.
    tmp_dir = tempfile.mkdtemp()
    os.chdir(tmp_dir)

    bundles_to_upload = []
    bundles_to_delete = []

    # Build attachments bundle for collections which have the option set.
    for changeset in all_changesets:
        bid = changeset["metadata"]["bucket"]
        cid = changeset["metadata"]["id"]
        should_bundle = changeset["metadata"].get("attachment", {}).get("bundle", False)
        attachments_bundle_filename = f"{bid}--{cid}.zip"

        if not should_bundle:
            bundles_to_delete.append(attachments_bundle_filename)
            if not BUILD_ALL:
                continue
        else:
            print(f"{bid}/{cid} has attachments bundles enabled")

        existing_bundle_timestamp = get_modified_timestamp(
            f"{base_url}{DESTINATION_FOLDER}/{bid}--{cid}.zip"
        )
        print(f"'{bid}--{cid}.zip' was modified at {existing_bundle_timestamp}")
        print(f"Latest change on {bid}/{cid} was at {changeset['timestamp']}")
        if not BUILD_ALL and changeset["timestamp"] < existing_bundle_timestamp:
            # Collection hasn't changed since last bundling.
            print(f"{bid}/{cid} hasn't changed since last bundle.")
            continue

        # Skip bundle if no attachments found.
        records = [r for r in changeset["changes"] if "attachment" in r]
        if not records:
            print(f"{bid}/{cid} has no attachments")
            bundles_to_delete.append(attachments_bundle_filename)
            continue

        print(f"{bid}/{cid} {len(records)} records with attachments")

        # Skip bundle if total size is too big.
        total_size_bytes = sum(r["attachment"]["size"] for r in records)
        total_size_mb = total_size_bytes / 1024 / 1024
        if total_size_bytes > BUNDLE_MAX_SIZE_BYTES:
            print(f"Bundle would be too big ({total_size_mb:.2f}MB). Skip.")
            continue
        print(f"Attachments total size {total_size_mb:.2f}MB")

        # Fetch all attachments and build "{bid}--{cid}.zip"
        args_list = [(f"{base_url}{r['attachment']['location']}",) for r in records]
        all_attachments = call_parallel(fetch_attachment, args_list)
        write_zip(
            attachments_bundle_filename,
            [(f"{record['id']}.meta.json", json.dumps(record)) for record in records]
            + [
                (record["id"], attachment)
                for record, attachment in zip(records, all_attachments)
            ],
        )
        bundles_to_upload.append(attachments_bundle_filename)

    highest_timestamp = max(c["timestamp"] for c in all_changesets)
    print(f"Latest server change was at {highest_timestamp}")

    existing_bundle_timestamp = get_modified_timestamp(
        f"{base_url}{DESTINATION_FOLDER}/changesets.json.mozlz4"
    )
    print(f"'changesets.json.mozlz4' was published at {existing_bundle_timestamp}")
    if BUILD_ALL or (existing_bundle_timestamp < highest_timestamp):
        write_json_mozlz4(
            "changesets.json.mozlz4",
            [
                changeset
                for changeset in all_changesets
                if "preview" not in changeset["metadata"]["bucket"]
            ],
        )
        bundles_to_upload.append("changesets.json.mozlz4")
    else:
        print("Existing 'changesets.json.mozlz4' bundle up-to-date. Nothing to do.")

    # Build a bundle for collections that are marked with "startup" flag.
    startup_file = "startup.json.mozlz4"
    existing_bundle_timestamp = get_modified_timestamp(
        f"{base_url}{DESTINATION_FOLDER}/{startup_file}"
    )
    print(f"{startup_file!r} was published at {existing_bundle_timestamp}")
    if BUILD_ALL or existing_bundle_timestamp < highest_timestamp:
        write_json_mozlz4(
            startup_file,
            [
                changeset
                for changeset in all_changesets
                if "startup" in changeset["metadata"].get("flags", [])
                and "preview" not in changeset["metadata"]["bucket"]
            ],
        )
        bundles_to_upload.append(startup_file)
    else:
        print(f"Existing {startup_file!r} bundle up-to-date. Nothing to do.")

    if not SKIP_UPLOAD:
        sync_cloud_storage(
            STORAGE_BUCKET_NAME,
            DESTINATION_FOLDER,
            bundles_to_upload,
            bundles_to_delete,
        )