def _fetch_range_proxy()

in awswrangler/s3/_fs.py [0:0]

• 38 lines of code
• 4 McCabe index (conditional complexity)

    def _fetch_range_proxy(self, start: int, end: int) -> bytes:
        _logger.debug("Fetching: s3://%s/%s - Range: %s-%s", self._bucket, self._key, start, end)
        boto3_kwargs: dict[str, Any] = get_botocore_valid_kwargs(
            function_name="get_object", s3_additional_kwargs=self._s3_additional_kwargs
        )
        cpus: int = _utils.ensure_cpu_count(use_threads=self._use_threads)
        range_size: int = end - start
        if cpus < 2 or range_size < (2 * _MIN_PARALLEL_READ_BLOCK):
            return _fetch_range(
                range_values=(start, end),
                bucket=self._bucket,
                key=self._key,
                s3_client=self._client,
                boto3_kwargs=boto3_kwargs,
                version_id=self._version_id,
            )[1]
        sizes: tuple[int, ...] = _utils.get_even_chunks_sizes(
            total_size=range_size, chunk_size=_MIN_PARALLEL_READ_BLOCK, upper_bound=False
        )
        ranges: list[tuple[int, int]] = []
        chunk_start: int = start
        for size in sizes:
            ranges.append((chunk_start, chunk_start + size))
            chunk_start += size
        with concurrent.futures.ThreadPoolExecutor(max_workers=cpus) as executor:
            return self._merge_range(
                ranges=list(
                    executor.map(
                        _fetch_range,
                        ranges,
                        itertools.repeat(self._bucket),
                        itertools.repeat(self._key),
                        itertools.repeat(self._client),
                        itertools.repeat(boto3_kwargs),
                        itertools.repeat(self._version_id),
                    )
                ),
            )