in scripts/parser.py [0:0]
def fetch(self, start, end=None):
"""
Ensure buffer[start..end] is loaded (inclusive).
If end=None, do one fetch from `start` to EOF (no min-size).
Otherwise, fetch only the missing sub-ranges, each >= MIN_CHUNK_SIZE.
"""
if end is None:
# openāended final fetch
headers = {"Range": f"bytes={start}-"}
self.progress_bar.set_description(f"Fetching bytes {start}-EOF")
resp = self.session.get(self.url, headers=headers); resp.raise_for_status()
chunk = resp.content
self.total_downloaded += len(chunk)
self.progress_bar.update(len(chunk))
self._add_range(start, chunk)
return self.buffer[start:start+len(chunk)]
# find holes in [start..end]
to_fetch = []
cursor = start
for (a, b) in self.loaded_ranges:
if b < cursor: continue
if a > end: break
if a > cursor:
to_fetch.append((cursor, a-1))
cursor = max(cursor, b+1)
if cursor <= end:
to_fetch.append((cursor, end))
# fetch each hole (enforcing MIN_CHUNK_SIZE)
for (s, e) in to_fetch:
if (e - s + 1) < MIN_CHUNK_SIZE:
e = s + MIN_CHUNK_SIZE - 1
headers = {"Range": f"bytes={s}-{e}"}
self.progress_bar.set_description(f"Fetching bytes {s}-{e}")
resp = self.session.get(self.url, headers=headers); resp.raise_for_status()
chunk = resp.content
self.total_downloaded += len(chunk)
self.progress_bar.update(len(chunk))
self._add_range(s, chunk)
return self.buffer[start:end+1]