in src/huggingface_hub/file_download.py [0:0]
def _create_symlink(src: str, dst: str, new_blob: bool = False) -> None:
"""Create a symbolic link named dst pointing to src.
By default, it will try to create a symlink using a relative path. Relative paths have 2 advantages:
- If the cache_folder is moved (example: back-up on a shared drive), relative paths within the cache folder will
not break.
- Relative paths seems to be better handled on Windows. Issue was reported 3 times in less than a week when
changing from relative to absolute paths. See https://github.com/huggingface/huggingface_hub/issues/1398,
https://github.com/huggingface/diffusers/issues/2729 and https://github.com/huggingface/transformers/pull/22228.
NOTE: The issue with absolute paths doesn't happen on admin mode.
When creating a symlink from the cache to a local folder, it is possible that a relative path cannot be created.
This happens when paths are not on the same volume. In that case, we use absolute paths.
The result layout looks something like
└── [ 128] snapshots
├── [ 128] 2439f60ef33a0d46d85da5001d52aeda5b00ce9f
│ ├── [ 52] README.md -> ../../../blobs/d7edf6bd2a681fb0175f7735299831ee1b22b812
│ └── [ 76] pytorch_model.bin -> ../../../blobs/403450e234d65943a7dcf7e05a771ce3c92faa84dd07db4ac20f592037a1e4bd
If symlinks cannot be created on this platform (most likely to be Windows), the workaround is to avoid symlinks by
having the actual file in `dst`. If it is a new file (`new_blob=True`), we move it to `dst`. If it is not a new file
(`new_blob=False`), we don't know if the blob file is already referenced elsewhere. To avoid breaking existing
cache, the file is duplicated on the disk.
In case symlinks are not supported, a warning message is displayed to the user once when loading `huggingface_hub`.
The warning message can be disabled with the `DISABLE_SYMLINKS_WARNING` environment variable.
"""
try:
os.remove(dst)
except OSError:
pass
abs_src = os.path.abspath(os.path.expanduser(src))
abs_dst = os.path.abspath(os.path.expanduser(dst))
abs_dst_folder = os.path.dirname(abs_dst)
# Use relative_dst in priority
try:
relative_src = os.path.relpath(abs_src, abs_dst_folder)
except ValueError:
# Raised on Windows if src and dst are not on the same volume. This is the case when creating a symlink to a
# local_dir instead of within the cache directory.
# See https://docs.python.org/3/library/os.path.html#os.path.relpath
relative_src = None
try:
commonpath = os.path.commonpath([abs_src, abs_dst])
_support_symlinks = are_symlinks_supported(commonpath)
except ValueError:
# Raised if src and dst are not on the same volume. Symlinks will still work on Linux/Macos.
# See https://docs.python.org/3/library/os.path.html#os.path.commonpath
_support_symlinks = os.name != "nt"
except PermissionError:
# Permission error means src and dst are not in the same volume (e.g. destination path has been provided
# by the user via `local_dir`. Let's test symlink support there)
_support_symlinks = are_symlinks_supported(abs_dst_folder)
except OSError as e:
# OS error (errno=30) means that the commonpath is readonly on Linux/MacOS.
if e.errno == errno.EROFS:
_support_symlinks = are_symlinks_supported(abs_dst_folder)
else:
raise
# Symlinks are supported => let's create a symlink.
if _support_symlinks:
src_rel_or_abs = relative_src or abs_src
logger.debug(f"Creating pointer from {src_rel_or_abs} to {abs_dst}")
try:
os.symlink(src_rel_or_abs, abs_dst)
return
except FileExistsError:
if os.path.islink(abs_dst) and os.path.realpath(abs_dst) == os.path.realpath(abs_src):
# `abs_dst` already exists and is a symlink to the `abs_src` blob. It is most likely that the file has
# been cached twice concurrently (exactly between `os.remove` and `os.symlink`). Do nothing.
return
else:
# Very unlikely to happen. Means a file `dst` has been created exactly between `os.remove` and
# `os.symlink` and is not a symlink to the `abs_src` blob file. Raise exception.
raise
except PermissionError:
# Permission error means src and dst are not in the same volume (e.g. download to local dir) and symlink
# is supported on both volumes but not between them. Let's just make a hard copy in that case.
pass
# Symlinks are not supported => let's move or copy the file.
if new_blob:
logger.info(f"Symlink not supported. Moving file from {abs_src} to {abs_dst}")
shutil.move(abs_src, abs_dst, copy_function=_copy_no_matter_what)
else:
logger.info(f"Symlink not supported. Copying file from {abs_src} to {abs_dst}")
shutil.copyfile(abs_src, abs_dst)