in odps/lib/importer.py [0:0]
def __init__(self, *compressed_files, **kwargs):
"""
Constructor.
Args:
compressed_files zipfile.ZipFile or tarfile.TarFile
"""
self._files = []
self._prefixes = defaultdict(lambda: set(['']))
self._extract_binary = kwargs.get('extract_binary', kwargs.get('extract', False))
self._extract_all = kwargs.get('extract_all', False)
self._supersede = kwargs.get('supersede', True)
for f in compressed_files:
if isinstance(f, zipfile.ZipFile):
bin_package = any(
n.endswith('.so') or n.endswith('.pxd') or n.endswith('.dylib')
for n in f.namelist()
)
need_extract = True
elif isinstance(f, tarfile.TarFile):
bin_package = any(
m.name.endswith('.so') or m.name.endswith('.pxd') or m.name.endswith('.dylib')
for m in f.getmembers()
)
need_extract = True
elif isinstance(f, dict):
bin_package = any(
name.endswith('.so') or name.endswith('.pxd') or name.endswith('.dylib')
for name in iterkeys(f)
)
need_extract = False
elif isinstance(f, list):
bin_package = any(
name.endswith('.so') or name.endswith('.pxd') or name.endswith('.dylib')
for name in f
)
need_extract = False
else:
raise TypeError('Compressed file can only be zipfile.ZipFile or tarfile.TarFile')
if bin_package:
# binary packages need to be extracted before use
if not ALLOW_BINARY:
raise SystemError(
'Cannot load binary package. It is quite possible that you are using an old '
'MaxCompute service which does not support binary packages. If this is '
'not true, please set `odps.isolation.session.enable` to True or ask your '
'project owner to change project-level configuration.'
)
if need_extract:
f = self._extract_archive(f)
elif need_extract and self._extract_all:
# when it is forced to extract even if it is a text package, also extract
f = self._extract_archive(f)
prefixes = set([''])
dir_prefixes = set() # only for lists or dicts
if isinstance(f, zipfile.ZipFile):
for name in f.namelist():
name = name if name.endswith('/') else (name.rsplit('/', 1)[0] + '/')
if name in prefixes:
continue
try:
f.getinfo(name + '__init__.py')
except KeyError:
prefixes.add(name)
elif isinstance(f, tarfile.TarFile):
for member in f.getmembers():
name = member.name if member.isdir() else member.name.rsplit('/', 1)[0]
if name in prefixes:
continue
try:
f.getmember(name + '/__init__.py')
except KeyError:
prefixes.add(name + '/')
elif isinstance(f, (list, dict)):
# Force ArchiveResource to run under binary mode to resolve manually
# opening __file__ paths in pure-python code.
if ALLOW_BINARY:
bin_package = True
rendered_names = set()
for name in f:
name = name.replace(os.sep, '/')
rendered_names.add(name)
for name in rendered_names:
name = name if name.endswith('/') else (name.rsplit('/', 1)[0] + '/')
if name in prefixes or '/tests/' in name or '/__pycache__/' in name:
continue
if name + '__init__.py' not in rendered_names:
prefixes.add(name)
dir_prefixes.add(name)
else:
if '/' in name.rstrip('/'):
ppath = name.rstrip('/').rsplit('/', 1)[0]
else:
ppath = ''
prefixes.add(ppath)
dir_prefixes.add(ppath)
# make sure only root packages are included, otherwise relative imports might be broken
# NOTE that it is needed to check sys.path duplication after all pruning done,
# otherwise path might be error once CompressImporter is called twice.
path_patch = []
for p in sorted(dir_prefixes):
parent_exist = False
for pp in path_patch:
if p[:len(pp)] == pp:
parent_exist = True
break
if parent_exist:
continue
path_patch.append(p)
if bin_package:
path_patch = [p for p in path_patch if p not in sys.path]
if self._supersede:
sys.path = path_patch + sys.path
else:
sys.path = sys.path + path_patch
try:
self._load_dynlibs(path_patch)
except:
warnings.warn(
"Failed to call _load_dynlibs. Dynamic libraries might not be loaded properly."
)
else:
self._files.append(f)
if path_patch:
path_patch = [p for p in path_patch if p not in sys.path]
self._prefixes[id(f)] = sorted([''] + path_patch)
elif prefixes:
self._prefixes[id(f)] = sorted(prefixes)