def parse_and_export_stats()

in analytics/s3_test_stats_analyze.py [0:0]

• 16 lines of code
• 7 McCabe index (conditional complexity)

def parse_and_export_stats(folder: str, granularity: str, commit_sha_lists: Optional[List[str]] = None):
    dataframe = None
    for (dirpath, _, filenames) in os.walk(folder):
        for filename in tqdm(filenames):
            splits = dirpath.split("/")
            job_name = splits[-1]
            sha = splits[-2]
            if not commit_sha_lists or sha in commit_sha_lists:
                with bz2.open(os.path.join(dirpath, filename), 'r') as zf:
                    string = zf.read().decode("utf-8")
                    data = json.loads(string)
                    # create a deep json with sha and job info
                    data['sha'] = sha
                    data['job'] = job_name
                    df = _json_to_df(data, granularity)
                    dataframe = df if dataframe is None else dataframe.append(df)
    return dataframe