def organize_data()

in pageload-summary/summarize_testing.py [0:0]

• 58 lines of code
• 20 McCabe index (conditional complexity)

def organize_data(data, platforms):
    """Organizes the data into a format that is easier to handle.

    Ex: data = {
        "platform1": {
            "test1": {
                "extra_options": set(),
                "tags": set(),
                "values": {
                    "time": val,
                    ...
                }
            },
            ...
        },
        ...
    }
    """
    platform_ind = get_data_ind(data, "platform")
    test_ind = get_data_ind(data, "suite")
    extra_ind = get_data_ind(data, "extra_options")
    tag_ind = get_data_ind(data, "tags")
    val_ind = get_data_ind(data, "value")
    time_ind = get_data_ind(data, "push_timestamp")
    app_ind = get_data_ind(data, "application")

    org_data = {}
    for entry in data[1:]:
        platform = entry[platform_ind]
        if platforms and platform not in platforms:
            continue

        test = entry[test_ind]
        app = entry[app_ind]
        extras = entry[extra_ind].split()
        tags = entry[tag_ind].split()
        variants = "None"
        pl_type = "cold"

        if "warm" not in extras and "cold" not in extras:
            continue

        if "live" in extras:
            continue

        if "warm" in extras:
            pl_type = "warm"

        if "fission" in extras:
            variants += "fission-"
        if "webrender" in extras:
            variants += "webrender"

        if "nocondprof" in extras:
            extras.remove("nocondprof")
        # if "nocondprof" in tags:
        #     tags.remove("nocondprof")
        if "visual" not in extras:
            extras.append("visual")
        # if "visual" not in tags:
        #     tags.append("visual")

        # if test not in ("amazon", "google-mail", "google-slides", "imgur", "tumblr", "twitch", "twitter"):
        #     continue

        if variants != "None":
            print("here")
            variants = variants.replace("None", "")

        mod_test_name = f"{test}-{app}" + "-".join(sorted(extras))
        test_data = (
            org_data.setdefault(platform, {})
            .setdefault(app, {})
            .setdefault(variants, {})
            .setdefault(pl_type, {})
            .setdefault(mod_test_name, {})
        )

        # Make sure we're never mixing data
        if "extra_options" in test_data:
            assert test_data["extra_options"] == set(list(extras))
        else:
            test_data["extra_options"] = set(list(extras))
        # if "tags" in test_data:
        #     print("awlkhwalkhd")
        #     print(test_data["tags"])
        #     print(tags)
        #     assert test_data["tags"] == set(list(tags))
        # else:
        #     test_data["tags"] = set(list(tags))

        test_data.setdefault("values", {}).setdefault(entry[time_ind], []).append(
            float(entry[val_ind])
        )

    if not org_data:
        possible_platforms = set([entry[platform_ind] for entry in data])
        raise Exception(
            "Could not find any requested platforms in the data. Possible choices are: "
            f"{possible_platforms}"
        )

    return org_data