def remote_storage_pull()

in probe_scraper/remote_storage.py [0:0]

• 16 lines of code
• 6 McCabe index (conditional complexity)

def remote_storage_pull(src: str, dst: Path, decompress: bool = False):
    sync = _get_sync_function(src)
    if sync is _gcs_sync:
        # gsutil will decompress files
        decompress = False
        # prevent error from gsutil when dst and src do not exist
        dst.mkdir(parents=True, exist_ok=True)

    if decompress:
        with TemporaryDirectory() as tmp:
            tmp_path = Path(tmp)
            sync(src, tmp_path)
            for in_file in tmp_path.rglob("*"):
                if not in_file.is_dir():
                    out_file = dst / in_file.relative_to(tmp_path)
                    out_file.parent.mkdir(parents=True, exist_ok=True)
                    out_file.write_bytes(gzip.decompress(in_file.read_bytes()))
    else:
        sync(src, dst)