in src_python/habitat_sim/utils/datasets_download.py [0:0]
def initialize_test_data_sources(data_path):
global data_sources
global data_groups
r"""Initializes data_sources and data_groups dicts with a variable data_path path.
"""
# dict keyed by uids with download specs for various individual test assets and datasets
data_sources = {
# "example_uid": {
# "source": the URL download link
# "package_name": the filename of the downloaded compressed package
# "download_pre_args": (optional)(wget) commands preceding filename
# "download_post_args": (optional)(wget) commands follow filename
# "link": symlink to the data directory pointing to the active version directory
# "version": data version tag
# }
"habitat_test_scenes": {
"source": "http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes_v1.0.zip",
"package_name": "habitat-test-scenes_v1.0.zip",
"link": data_path + "scene_datasets/habitat-test-scenes",
"version": "1.0",
},
"habitat_test_pointnav_dataset": {
"source": "http://dl.fbaipublicfiles.com/habitat/habitat-test-pointnav-dataset_v1.0.zip",
"package_name": "habitat-test-pointnav-dataset_v1.0.zip",
"link": data_path + "datasets/pointnav/habitat-test-scenes",
"version": "1.0",
},
"habitat_example_objects": {
"source": "http://dl.fbaipublicfiles.com/habitat/objects_v0.2.zip",
"package_name": "objects_v0.2.zip",
"link": data_path + "objects/example_objects",
"version": "0.2",
},
"locobot_merged": {
"source": "http://dl.fbaipublicfiles.com/habitat/locobot_merged_v0.2.zip",
"package_name": "locobot_merged_v0.2.zip",
"link": data_path + "objects/locobot_merged",
"version": "0.2",
},
"mp3d_example_scene": {
"source": "http://dl.fbaipublicfiles.com/habitat/mp3d/mp3d_example_v1.1.zip",
"package_name": "mp3d_example_v1.1.zip",
"link": data_path + "scene_datasets/mp3d_example",
"version": "1.1",
},
"coda_scene": {
"source": "https://dl.fbaipublicfiles.com/habitat/coda_v1.0.zip",
"package_name": "coda_v1.0.zip",
"download_pre_args": "--no-check-certificate ",
"download_post_args": " -O " + data_path + "coda_v1.0.zip",
"link": data_path + "scene_datasets/coda",
"version": "1.0",
},
"webxr_hand_demo": {
"source": "https://dl.fbaipublicfiles.com/habitat/data/scene_datasets/webxr_hand_demo_data.zip",
"package_name": "webxr_hand_demo_data.zip",
"link": data_path + "webxr_hand_demo_dataset",
"version": "1.0",
},
"replica_cad_dataset": {
"source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_dataset_v1.2.zip",
"package_name": "ReplicaCAD_dataset_v1.2.zip",
"link": data_path + "replica_cad",
"version": "1.2",
},
"replica_cad_baked_lighting": {
"source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_baked_lighting_v1.1.zip",
"package_name": "ReplicaCAD_baked_lighting_v1.1.zip",
"link": data_path + "replica_cad_baked_lighting",
"version": "1.1",
},
"ycb": {
"source": "https://dl.fbaipublicfiles.com/habitat/ycb/hab_ycb_v1.1.zip",
"package_name": "hab_ycb_v1.1.zip",
"link": data_path + "objects/ycb",
"version": "1.1",
},
"hab_fetch": {
"source": "http://dl.fbaipublicfiles.com/habitat/hab_fetch_v1.0.zip",
"package_name": "hab_fetch_v1.0.zip",
"link": data_path + "robots/hab_fetch",
"version": "1.0",
},
"rearrange_pick_dataset_v0": {
"source": "https://dl.fbaipublicfiles.com/habitat/data/datasets/rearrange_pick/replica_cad/v0/rearrange_pick_replica_cad_v0.zip",
"package_name": "rearrange_pick_replica_cad_v0.zip",
"link": data_path + "datasets/rearrange_pick/replica_cad/v0",
"version": "1.0",
},
"hab2_bench_assets": {
"source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/hab2_bench_assets.zip",
"package_name": "hab2_bench_assets.zip",
"link": data_path + "hab2_bench_assets",
"version": "1.0",
},
}
data_sources.update(
{
f"hm3d_{split}_{data_format}": {
"source": "https://api.matterport.com/resources/habitat/hm3d-{split}-{data_format}.tar{ext}".format(
ext=".gz" if data_format == "obj+mtl" else "",
split=split,
data_format=data_format,
),
"download_pre_args": "--location",
"package_name": "hm3d-{split}-{data_format}.tar{ext}".format(
ext=".gz" if data_format == "obj+mtl" else "",
split=split,
data_format=data_format,
),
"link": data_path + "scene_datasets/hm3d",
"version": "1.0",
"version_dir": "hm3d-{version}/hm3d",
"extract_postfix": f"{split}",
"downloaded_file_list": f"hm3d-{{version}}/{split}-{data_format}-files.json.gz",
"requires_auth": True,
"use_curl": True,
"post_extract_fn": hm3d_train_configs_post
if split == "train" and data_format == "configs"
else None,
}
for split, data_format in itertools.product(
["minival", "train", "val"],
["glb", "obj+mtl", "habitat", "configs"],
)
}
)
data_sources.update(
{
f"hm3d_example_{data_format}": {
"source": "https://github.com/matterport/habitat-matterport-3dresearch/raw/main/example/hm3d-example-{data_format}.tar{ext}".format(
ext=".gz" if data_format == "obj+mtl" else "",
data_format=data_format,
),
"package_name": "hm3d-example-{data_format}.tar{ext}".format(
ext=".gz" if data_format == "obj+mtl" else "",
data_format=data_format,
),
"link": data_path + "scene_datasets/hm3d",
"version": "1.0",
"version_dir": "hm3d-{version}/hm3d",
"extract_postfix": "example",
"downloaded_file_list": f"hm3d-{{version}}/example-{data_format}-files.json.gz",
}
for data_format in ["glb", "obj+mtl", "habitat", "configs"]
}
)
# data sources can be grouped for batch commands with a new uid
data_groups = {
"ci_test_assets": [
"habitat_test_scenes",
"habitat_test_pointnav_dataset",
"habitat_example_objects",
"locobot_merged",
"mp3d_example_scene",
"coda_scene",
"replica_cad_dataset",
"hab_fetch",
],
"rearrange_task_assets": [
"replica_cad_dataset",
"hab_fetch",
"ycb",
"rearrange_pick_dataset_v0",
],
"hm3d_example": ["hm3d_example_habitat", "hm3d_example_configs"],
"hm3d_val": ["hm3d_val_habitat", "hm3d_val_configs"],
"hm3d_train": ["hm3d_train_habitat", "hm3d_train_configs"],
"hm3d_minival": ["hm3d_minival_habitat", "hm3d_minival_configs"],
"hm3d_full": list(filter(lambda k: k.startswith("hm3d_"), data_sources.keys())),
}
data_groups.update(
{
f"hm3d_{split}_full": list(
filter(lambda k: k.startswith(f"hm3d_{split}"), data_sources.keys())
)
for split in ["train", "val", "minival", "example"]
}
)
data_groups["hm3d"] = (
data_groups["hm3d_val"]
+ data_groups["hm3d_train"]
+ data_groups["hm3d_minival"]
)
data_groups["ci_test_assets"].extend(data_groups["hm3d_example"])