def download()

in neuralcompression/data/_clic_2020_video.py [0:0]

• 23 lines of code
• 3 McCabe index (conditional complexity)

    def download(self):
        self._root.mkdir(exist_ok=True, parents=True)

        with urllib.request.urlopen(f"{self._destination_root}/video_urls.txt") as file:
            endpoints = file.read().decode("utf-8").splitlines()

        def f(endpoint: str):
            time.sleep(0.001)

            path, _ = urllib.request.urlretrieve(endpoint)

            with ZipFile(path, "r") as archive:
                archive.extractall(self._root)

        with tqdm.tqdm(total=len(endpoints)) as progress:
            with ThreadPoolExecutor() as executor:
                futures = {
                    executor.submit(
                        f,
                        endpoint,
                    ): endpoint
                    for endpoint in endpoints
                }

                completed = {}

                for future in concurrent.futures.as_completed(futures):
                    endpoint = futures[future]

                    completed[endpoint] = future.result()

                    progress.update()