in misc/reference_datasets/multilingual/copy_raw_data.py [0:0]
def __init__(self,
data_folder,
dump_to_proc: str,
compression="infer",
limit: int = -1,
skip: int = 0,
file_progress: bool = False,
doc_progress: bool = False,
adapter = None,
text_key: str = "text",
id_key: str = "id",
default_metadata: dict = None,
recursive: bool = True,
glob_pattern: str | None = None,
shuffle_files: bool = False):
super().__init__(
data_folder,
None,
compression,
limit,
skip,
file_progress,
doc_progress,
adapter,
text_key,
id_key,
default_metadata,
recursive,
glob_pattern,
shuffle_files,
)
self.dump_to_proc = dump_to_proc