Skip to content
Python
4.1 KB
Raw Blame
#!/usr/bin/env python3
"""Download parquet annotation shards from facebook/sam-3d-body-dataset.

The gated HF repo needs an accepted licence and a token (`hf auth login`, or
HF_TOKEN in the environment). Downloads are resumable and skip shards already
on disk, so re-running after a killed job is free.

    # everything for one split
    python datasets_pipeline/fetch_annotations.py --splits coco_train --out $ANN_DIR

    # a slice, for a cheap first pass on a huge split
    python datasets_pipeline/fetch_annotations.py --splits sa1b_train --out $ANN_DIR \
        --max-shards 50

    # just tell me what is available and how big it is
    python datasets_pipeline/fetch_annotations.py --list
"""

from __future__ import annotations

import argparse
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

from huggingface_hub import HfApi, hf_hub_download
from tqdm import tqdm

sys.path.insert(0, str(Path(__file__).resolve().parent))
from registry import HF_ANNOTATION_REPO, SPLIT_TO_DATASET  # noqa: E402


def shard_index(api: HfApi) -> dict[str, list[str]]:
    """split -> sorted list of repo-relative parquet paths."""
    out: dict[str, list[str]] = {}
    for f in api.list_repo_files(HF_ANNOTATION_REPO, repo_type="dataset"):
        if not f.endswith(".parquet"):
            continue
        parts = f.split("/")
        if len(parts) < 3:
            continue
        out.setdefault(parts[1], []).append(f)
    return {k: sorted(v) for k, v in sorted(out.items())}


def main() -> None:
    p = argparse.ArgumentParser(description=__doc__,
                                formatter_class=argparse.RawDescriptionHelpFormatter)
    p.add_argument("--splits", nargs="+", help="e.g. coco_train harmony4d_train")
    p.add_argument("--out", type=Path, help="destination root; one folder per split")
    p.add_argument("--max-shards", type=int, default=None,
                   help="take only the first N shards of each split")
    p.add_argument("--workers", type=int, default=8)
    p.add_argument("--list", action="store_true", help="print available splits and exit")
    args = p.parse_args()

    api = HfApi()
    index = shard_index(api)

    if args.list:
        print(f"{'split':26s} {'shards':>7s}  ~persons")
        for split, files in index.items():
            # ~1,750 rows/shard, measured across coco/aic/sa1b/egoexo4d/egohumans.
            print(f"{split:26s} {len(files):7d}  {len(files) * 1750:>9,d}")
        return

    if not args.splits or not args.out:
        p.error("--splits and --out are required unless --list is given")

    unknown = [s for s in args.splits if s not in index]
    if unknown:
        p.error(f"unknown split(s): {unknown}. Run --list for the full set.")
    for s in args.splits:
        if s not in SPLIT_TO_DATASET:
            print(f"[warn] {s} has no entry in registry.py; build_split.py will "
                  f"not know where its images live.", file=sys.stderr)

    for split in args.splits:
        files = index[split]
        if args.max_shards:
            files = files[: args.max_shards]
        dest = args.out / split
        dest.mkdir(parents=True, exist_ok=True)
        todo = [f for f in files if not (dest / Path(f).name).exists()]
        print(f"{split}: {len(files)} shards, {len(todo)} to download -> {dest}")
        if not todo:
            continue

        def grab(repo_file: str) -> None:
            hf_hub_download(HF_ANNOTATION_REPO, repo_file, repo_type="dataset",
                            local_dir=str(dest.parent / ".hf_staging"))
            src = dest.parent / ".hf_staging" / repo_file
            src.replace(dest / src.name)

        with ThreadPoolExecutor(max_workers=args.workers) as ex:
            futs = {ex.submit(grab, f): f for f in todo}
            for fut in tqdm(as_completed(futs), total=len(futs), unit="shard"):
                try:
                    fut.result()
                except Exception as e:  # noqa: BLE001
                    tqdm.write(f"[fail] {futs[fut]}: {e}")

    staging = args.out / ".hf_staging"
    if staging.exists():
        import shutil
        shutil.rmtree(staging, ignore_errors=True)


if __name__ == "__main__":
    main()