in src/datasets/utils/_dill.py [0:0]
def save_code(pickler, obj):
dill._dill.logger.trace(pickler, "Co: %s", obj)
############################################################################################################
# Modification here for huggingface/datasets
# The filename of a function is the .py file where it is defined.
# Filenames of functions created in notebooks or shells start with '<'
# ex: <ipython-input-13-9ed2afe61d25> for ipython, and <stdin> for shell
# Filenames of functions created in ipykernel the filename
# look like f"{tempdir}/ipykernel_{id1}/{id2}.py"
# Moreover lambda functions have a special name: '<lambda>'
# ex: (lambda x: x).__code__.co_name == "<lambda>" # True
#
# For the hashing mechanism we ignore where the function has been defined
# More specifically:
# - we ignore the filename of special functions (filename starts with '<')
# - we always ignore the line number
# - we only use the base name of the file instead of the whole path,
# to be robust in case a script is moved for example.
#
# Only those two lines are different from the original implementation:
co_filename = (
""
if obj.co_filename.startswith("<")
or (
len(obj.co_filename.split(os.path.sep)) > 1
and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_")
)
or obj.co_name == "<lambda>"
else os.path.basename(obj.co_filename)
)
co_firstlineno = 1
# The rest is the same as in the original dill implementation, except for the replacements:
# - obj.co_filename => co_filename
# - obj.co_firstlineno => co_firstlineno
# - obj.co_lnotab => obj.co_linetable for >= 3.10 since co_lnotab was deprecated
############################################################################################################
if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args)
args = (
obj.co_linetable, # Modification for huggingface/datasets ############################################
obj.co_argcount,
obj.co_posonlyargcount,
obj.co_kwonlyargcount,
obj.co_nlocals,
obj.co_stacksize,
obj.co_flags,
obj.co_code,
obj.co_consts,
obj.co_names,
obj.co_varnames,
co_filename, # Modification for huggingface/datasets ############################################
obj.co_name,
obj.co_qualname,
co_firstlineno, # Modification for huggingface/datasets #########################################
obj.co_linetable,
obj.co_endlinetable,
obj.co_columntable,
obj.co_exceptiontable,
obj.co_freevars,
obj.co_cellvars,
)
elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args)
args = (
obj.co_linetable, # Modification for huggingface/datasets #######################################
obj.co_argcount,
obj.co_posonlyargcount,
obj.co_kwonlyargcount,
obj.co_nlocals,
obj.co_stacksize,
obj.co_flags,
obj.co_code,
obj.co_consts,
obj.co_names,
obj.co_varnames,
co_filename, # Modification for huggingface/datasets ############################################
obj.co_name,
obj.co_qualname,
co_firstlineno, # Modification for huggingface/datasets #########################################
obj.co_linetable,
obj.co_exceptiontable,
obj.co_freevars,
obj.co_cellvars,
)
elif hasattr(obj, "co_linetable"): # python 3.10 (16 args)
args = (
obj.co_linetable, # Modification for huggingface/datasets #######################################
obj.co_argcount,
obj.co_posonlyargcount,
obj.co_kwonlyargcount,
obj.co_nlocals,
obj.co_stacksize,
obj.co_flags,
obj.co_code,
obj.co_consts,
obj.co_names,
obj.co_varnames,
co_filename, # Modification for huggingface/datasets ############################################
obj.co_name,
co_firstlineno, # Modification for huggingface/datasets #########################################
obj.co_linetable,
obj.co_freevars,
obj.co_cellvars,
)
elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args)
args = (
obj.co_argcount,
obj.co_posonlyargcount,
obj.co_kwonlyargcount,
obj.co_nlocals,
obj.co_stacksize,
obj.co_flags,
obj.co_code,
obj.co_consts,
obj.co_names,
obj.co_varnames,
co_filename, # Modification for huggingface/datasets ############################################
obj.co_name,
co_firstlineno, # Modification for huggingface/datasets #########################################
obj.co_lnotab,
obj.co_freevars,
obj.co_cellvars,
)
else: # python 3.7 (15 args)
args = (
obj.co_argcount,
obj.co_kwonlyargcount,
obj.co_nlocals,
obj.co_stacksize,
obj.co_flags,
obj.co_code,
obj.co_consts,
obj.co_names,
obj.co_varnames,
co_filename, # Modification for huggingface/datasets ############################################
obj.co_name,
co_firstlineno, # Modification for huggingface/datasets #########################################
obj.co_lnotab,
obj.co_freevars,
obj.co_cellvars,
)
pickler.save_reduce(dill._dill._create_code, args, obj=obj)
dill._dill.logger.trace(pickler, "# Co")
return