in src/doc_builder/build_doc.py [0:0]
def build_mdx_files(package, doc_folder, output_dir, page_info, version_tag_suffix):
"""
Build the MDX files for a given package.
Args:
package (`types.ModuleType`): The package where to look for objects to document.
doc_folder (`str` or `os.PathLike`): The folder where the doc source files are.
output_dir (`str` or `os.PathLike`): The folder where to put the files built.
page_info (`Dict[str, str]`): Some information about the page.
version_tag_suffix (`str`, *optional*, defaults to `"src/"`):
Suffix to add after the version tag (e.g. 1.3.0 or main) in the documentation links.
For example, the default `"src/"` suffix will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/src/`.
For example, `version_tag_suffix=""` will result in a base link as `https://github.com/huggingface/{package_name}/blob/{version_tag}/`.
"""
doc_folder = Path(doc_folder)
output_dir = Path(output_dir)
os.makedirs(output_dir, exist_ok=True)
anchor_mapping = {}
source_files_mapping = {}
if "package_name" not in page_info:
page_info["package_name"] = package.__name__
all_files = list(doc_folder.glob("**/*"))
all_errors = []
for file in tqdm(all_files, desc="Building the MDX files"):
new_anchors = None
errors = None
page_info["path"] = file
try:
if file.suffix in [".md", ".mdx"]:
dest_file = output_dir / (file.with_suffix(".mdx").relative_to(doc_folder))
page_info["page"] = file.with_suffix(".html").relative_to(doc_folder).as_posix()
os.makedirs(dest_file.parent, exist_ok=True)
with open(file, "r", encoding="utf-8-sig") as reader:
content = reader.read()
content = convert_md_to_mdx(content, page_info)
content = resolve_open_in_colab(content, page_info)
content, new_anchors, source_files, errors = resolve_autodoc(
content, package, return_anchors=True, page_info=page_info, version_tag_suffix=version_tag_suffix
)
if source_files is not None:
source_files_mapping[source_files] = str(file)
with open(dest_file, "w", encoding="utf-8") as writer:
writer.write(content)
# Make sure we clean up for next page.
del page_info["page"]
elif file.suffix in [".rst"]:
dest_file = output_dir / (file.with_suffix(".mdx").relative_to(doc_folder))
page_info["page"] = file.with_suffix(".html").relative_to(doc_folder)
os.makedirs(dest_file.parent, exist_ok=True)
with open(file, "r", encoding="utf-8") as reader:
content = reader.read()
content = convert_rst_to_mdx(content, page_info)
content = resolve_open_in_colab(content, page_info)
content, new_anchors, source_files, errors = resolve_autodoc(
content, package, return_anchors=True, page_info=page_info, version_tag_suffix=version_tag_suffix
)
if source_files is not None:
source_files_mapping[source_files] = str(file)
with open(dest_file, "w", encoding="utf-8") as writer:
writer.write(content)
# Make sure we clean up for next page.
del page_info["page"]
elif file.is_file() and "__" not in str(file):
# __ is a reserved svelte file/folder prefix
dest_file = output_dir / (file.relative_to(doc_folder))
os.makedirs(dest_file.parent, exist_ok=True)
shutil.copy(file, dest_file)
except Exception as e:
raise type(e)(f"There was an error when converting {file} to the MDX format.\n" + e.args[0]) from e
if new_anchors is not None:
page_name = str(file.with_suffix("").relative_to(doc_folder))
for anchor in new_anchors:
if isinstance(anchor, tuple):
anchor_mapping.update(
{a: f"{page_name}#{anchor[0]}" for a in anchor[1:] if a not in anchor_mapping}
)
anchor = anchor[0]
anchor_mapping[anchor] = page_name
if errors is not None:
all_errors.extend(errors)
if len(all_errors) > 0:
raise ValueError(
"The deployment of the documentation will fail because of the following errors:\n" + "\n".join(all_errors)
)
return anchor_mapping, source_files_mapping