The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
#!/usr/bin/env python3
"""Prepare $RCB_MANIGUARD_DATA, the host's BEHAVIOR-1K 3.7.2 data root that every ManiGuard task mounts read-only at /data
(tasks/maniguard/README.md). Host side, Python 3 standard library only; never run inside a task.
python3 images/maniguard/prep_data.py --root /path/maniguard_data --key /path/to/omnigibson.key # task subset
python3 images/maniguard/prep_data.py --root /path/maniguard_data --key /path/to/omnigibson.key --full # every member
python3 images/maniguard/prep_data.py --root /path/maniguard_data --check # verify only
What it writes (the layout OmniGibson 3.7.2 reads with OMNIGIBSON_DATA_PATH=/data):
behavior-1k-assets/ members of behavior-1k-assets-3.7.2rc1.zip (Hugging Face behavior-1k/zipped-datasets, pinned
revision; the file BEHAVIOR-1K v3.7.2's asset_utils downloads). Default: only the members the
selected ManiGuard tasks load (VERSION, metadata/, objects/<category>/<model>/ of every
DatasetObject in their scene snapshots, and every particle system: objects' abilities name
systems by name, e.g. a pepper shaker's black_pepper), read from the zip by HTTP range requests;
--full: every member, streamed (no zip is stored either way).
omnigibson-robot-assets/ every member of omnigibson-robot-assets.zip (the 2025-09-02 file BEHAVIOR-1K 3.7.2 downloads),
plus ManiGuard's long-finger Franka at models/franka/franka_panda_longfinger/ (Hugging Face
IDEAS-Lab-Northwestern/franka-panda-longfinger, pinned revision, Apache-2.0).
omnigibson.key copied from --key: the BEHAVIOR Data Bundle key of a licence the user already accepted. This
script never downloads a key and never accepts a licence on anyone's behalf.
MANIGUARD_DATA.json what is there: pins, member counts and digests, the tasks the subset covers.
Every member is checked against the zip's own CRC-32 and size; the long-finger files against Hugging Face's sha256 / git
blob ids. --check re-verifies an existing root against MANIGUARD_DATA.json (sizes and CRC-32 of every file)."""
from __future__ import annotations
import argparse, binascii, hashlib, json, os, pathlib, shutil, struct, sys, time, urllib.request, zlib
HERE = pathlib.Path(__file__).resolve().parent
ROOT = HERE.parents[1] # the repository (this file: images/maniguard/prep_data.py)
HF = "https://huggingface.co/datasets"
ASSETS = {"url": f"{HF}/behavior-1k/zipped-datasets/resolve/9922e2ffabbdc881e7ba1db72db91097c74cadf7/behavior-1k-assets-3.7.2rc1.zip",
"size": 31280662307, "sha256": "3bd7cdc1a8c5621768c2aaaf57c12a024560ab4f49d0162e540e769d8b165e8b", "dest": "behavior-1k-assets"}
ROBOT = {"url": f"{HF}/behavior-1k/zipped-datasets/resolve/015e37a26823a110fedb6a0c3d943f573de00ffe/omnigibson-robot-assets.zip",
"size": 501401227, "sha256": "5e9fe726172c8b4beb4ba4e12dbf5cf6c5791553a05c52be661eb2d6a3d241b8", "dest": "omnigibson-robot-assets"}
LONGFINGER_REV = "5f65ec0a715bc88ea865c2c69133b08b395c66bc"
LONGFINGER = { # path in the HF repo -> ("sha256", hex) for LFS files, ("git", blob sha1) for the others
"usd/franka_panda_longfinger.usda": ("sha256", "49424b434343b544266e49274f7a5eb0f4b3d16673d2e19100045a3988616b9b"),
"curobo/franka_panda_longfinger_description_curobo_default.yaml": ("git", "dba47c37c3cc572881aabe724421a723143ab822"),
"misc/metadata.json": ("git", "f747e3396445b9c959e59e24bc773089fae816d6"),
"LICENSE": ("git", "d8184a62e171197f62c5423712dbf37090e3476f"),
"LICENSE-OmniGibson.txt": ("git", "3883c7d16fc54952b28c4ad57e3c3f8e8e85a9e2"),
"NOTICE": ("git", "792ed208263d24ad34dde89a9299c9a3973223e7"),
"README.md": ("git", "d98e88464253b0d327d8ff5848616763c770edbb"),
}
LONGFINGER_DEST = "omnigibson-robot-assets/models/franka/franka_panda_longfinger"
def log(*a) -> None:
print("[prep]", *a, file=sys.stderr, flush=True)
def get(url: str, start: int | None = None, end: int | None = None, tries: int = 6) -> bytes:
for k in range(tries):
try:
req = urllib.request.Request(url, headers={"Range": f"bytes={start}-{end}"} if start is not None else {})
with urllib.request.urlopen(req, timeout=300) as r:
data = r.read()
if start is not None and len(data) != end - start + 1:
raise IOError(f"short range read {len(data)} != {end - start + 1}")
return data
except Exception as exc: # noqa: BLE001
if k == tries - 1:
raise
log(f"retry {k + 1} after {exc!r}"); time.sleep(5 * (k + 1))
raise RuntimeError("unreachable")
def central_directory(z: dict) -> list[dict]:
"""The zip's members (name, method, crc, compressed and file size, local header offset) from its central directory,
read by range requests (zip64 aware)."""
size = z["size"]; tail = get(z["url"], size - 65536, size - 1); base = size - len(tail)
i = tail.rfind(b"PK\x05\x06")
_, _, _, _, n, cd_size, cd_off, _ = struct.unpack("<IHHHHIIH", tail[i:i + 22])
if cd_off == 0xFFFFFFFF or n == 0xFFFF:
j = tail.rfind(b"PK\x06\x07"); _, _, z64, _ = struct.unpack("<IIQI", tail[j:j + 20])
f = struct.unpack("<IQHHIIQQQQ", tail[z64 - base:z64 - base + 56]); n, cd_size, cd_off = f[7], f[8], f[9]
cd = get(z["url"], cd_off, cd_off + cd_size - 1)
out, p = [], 0
while p < len(cd):
if cd[p:p + 4] != b"PK\x01\x02":
raise IOError(f"bad central directory entry at {p}")
(_, _, _, flag, meth, _, _, crc, cs, us, nl, xl, cl, _, _, _, off) = struct.unpack("<IHHHHHHIIIHHHHHII", cd[p:p + 46])
name = cd[p + 46:p + 46 + nl].decode("utf-8"); extra = cd[p + 46 + nl:p + 46 + nl + xl]
q = 0
while q + 4 <= len(extra):
hid, hl = struct.unpack("<HH", extra[q:q + 4]); body = extra[q + 4:q + 4 + hl]
if hid == 1:
r = 0
if us == 0xFFFFFFFF: us = struct.unpack("<Q", body[r:r + 8])[0]; r += 8
if cs == 0xFFFFFFFF: cs = struct.unpack("<Q", body[r:r + 8])[0]; r += 8
if off == 0xFFFFFFFF: off = struct.unpack("<Q", body[r:r + 8])[0]; r += 8
q += 4 + hl
out.append({"name": name, "method": meth, "crc": crc, "csize": cs, "usize": us, "offset": off, "flag": flag})
p += 46 + nl + xl + cl
if len(out) != n:
raise IOError(f"central directory has {len(out)} entries, the end record says {n}")
return out
def write_member(root: pathlib.Path, m: dict, raw: bytes) -> None:
"""raw = the member's local header + data; inflate, check CRC-32 and size, write (directories just made)."""
dest = root / m["name"]
if m["name"].endswith("/"):
dest.mkdir(parents=True, exist_ok=True); return
if raw[:4] != b"PK\x03\x04":
raise IOError(f"{m['name']}: no local header")
nl, xl = struct.unpack("<HH", raw[26:30]); data = raw[30 + nl + xl:30 + nl + xl + m["csize"]]
body = data if m["method"] == 0 else zlib.decompress(data, -15) if m["method"] == 8 else None
if body is None:
raise IOError(f"{m['name']}: compression method {m['method']}")
if len(body) != m["usize"] or (binascii.crc32(body) & 0xFFFFFFFF) != m["crc"]:
raise IOError(f"{m['name']}: size or CRC-32 differs from the zip's central directory")
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_name(dest.name + ".part"); tmp.write_bytes(body); os.replace(tmp, dest)
def have(root: pathlib.Path, m: dict) -> bool:
p = root / m["name"]
if m["name"].endswith("/"):
return p.is_dir()
return p.is_file() and p.stat().st_size == m["usize"] and (binascii.crc32(p.read_bytes()) & 0xFFFFFFFF) == m["crc"]
def extract_members(z: dict, members: list[dict], root: pathlib.Path, gap: int = 1 << 20) -> None:
"""Range requests over runs of nearby members (each read with its local header: 30 bytes + its name + up to 1 KiB of
extra fields); members already present and correct are skipped, so an interrupted run resumes."""
todo = sorted((m for m in members if not have(root, m)), key=lambda m: m["offset"])
log(f"{z['dest']}: {len(members)} members selected, {len(todo)} to fetch")
span = lambda m: m["offset"] + 30 + len(m["name"].encode()) + 1024 + m["csize"] # noqa: E731
i = 0
while i < len(todo):
j = i; start = todo[i]["offset"]; end = span(todo[i])
while j + 1 < len(todo) and todo[j + 1]["offset"] - end < gap and end - start < (64 << 20):
j += 1; end = max(end, span(todo[j]))
blob = get(z["url"], start, min(end, z["size"]) - 1)
for m in todo[i:j + 1]:
write_member(root, m, blob[m["offset"] - start:])
i = j + 1
if i % 200 == 0 or i == len(todo):
log(f"{z['dest']}: {i}/{len(todo)}")
def stream_all(z: dict, members: list[dict], root: pathlib.Path) -> None:
"""--full: one sequential GET of the whole zip, every member written as it passes, its sha256 checked at the end."""
order = sorted(members, key=lambda m: m["offset"]); h = hashlib.sha256(); pos = 0; buf = b""
with urllib.request.urlopen(z["url"], timeout=600) as r:
for k, m in enumerate(order):
need = (order[k + 1]["offset"] if k + 1 < len(order) else z["size"]) - m["offset"]
while len(buf) < (m["offset"] - pos) + need:
chunk = r.read(8 << 20)
if not chunk:
break
h.update(chunk); buf += chunk
raw = buf[m["offset"] - pos:m["offset"] - pos + need]
if not have(root, m):
write_member(root, m, raw)
buf = buf[m["offset"] - pos + need:]; pos = m["offset"] + need
if k % 5000 == 0:
log(f"{z['dest']}: {k}/{len(order)}")
while True:
chunk = r.read(8 << 20)
if not chunk:
break
h.update(chunk)
if h.hexdigest() != z["sha256"]:
raise IOError(f"{z['dest']}: the zip's sha256 {h.hexdigest()} != {z['sha256']}")
def instance_dir(task: str) -> pathlib.Path:
"""A task's instance files in the repository's tasks/maniguard/<task>/: the privileged task's environment/instance/,
the standard task's environment/sim/instance/."""
t = ROOT / "tasks" / "maniguard" / task
for d in (t / "environment/instance", t / "environment/sim/instance"):
if (d / "instance.json").is_file():
return d
raise RuntimeError(f"{task}: no instance files in tasks/maniguard/{task}/")
def all_tasks() -> list[str]:
return sorted(d.name for d in (ROOT / "tasks" / "maniguard").iterdir() if d.is_dir())
def subset_prefixes(tasks: list[str], extra_scenes: list[str] = ()) -> tuple[set, list]:
# every particle system: objects' abilities name systems their scene never holds (a pepper shaker's black_pepper), and
# OmniGibson checks a system name against the systems/ directories of the dataset (0.83 GB)
pre = {"VERSION", "metadata/", "systems/"}; used = []
for key in list(tasks) + list(extra_scenes):
f = pathlib.Path(key) if key in extra_scenes else instance_dir(key) / "scene_ep1.json"
scene = json.loads(f.read_text())
for name, o in scene["objects_info"]["init_info"].items():
if o.get("class_name") == "DatasetObject":
a = o["args"]; pre.add(f"objects/{a['category']}/{a['model']}/")
for sysname in (scene["state"]["registry"].get("system_registry") or {}):
pre.add(f"systems/{sysname}/")
used.append(key)
return pre, used
def blob_sha1(b: bytes) -> str:
return hashlib.sha1(b"blob %d\0" % len(b) + b).hexdigest()
def longfinger(root: pathlib.Path) -> dict:
out = {}
for rel, (kind, digest) in LONGFINGER.items():
dest = root / LONGFINGER_DEST / rel
ok = dest.is_file() and ((hashlib.sha256(dest.read_bytes()).hexdigest() if kind == "sha256" else blob_sha1(dest.read_bytes())) == digest)
if not ok:
b = get(f"{HF}/IDEAS-Lab-Northwestern/franka-panda-longfinger/resolve/{LONGFINGER_REV}/{rel}")
got = hashlib.sha256(b).hexdigest() if kind == "sha256" else blob_sha1(b)
if got != digest:
raise IOError(f"long-finger {rel}: {kind} {got} != {digest}")
dest.parent.mkdir(parents=True, exist_ok=True); dest.write_bytes(b)
out[rel] = {kind: digest}
return out
def manifest_files(root: pathlib.Path) -> dict:
out = {}
for p in sorted(root.rglob("*")):
if p.is_file() and p.name not in ("MANIGUARD_DATA.json", "omnigibson.key"):
out[p.relative_to(root).as_posix()] = [p.stat().st_size, binascii.crc32(p.read_bytes()) & 0xFFFFFFFF]
return out
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--root", required=True, help="the data root to create or complete ($RCB_MANIGUARD_DATA)")
ap.add_argument("--key", help="an existing omnigibson.key (the BEHAVIOR Data Bundle key of a licence already accepted)")
ap.add_argument("--full", action="store_true", help="every member of behavior-1k-assets-3.7.2rc1 (34.7 GB) instead of the task subset")
ap.add_argument("--tasks", nargs="*", default=None, help="ManiGuard task dirs (tasks/maniguard/<task>) for the subset (default: every one)")
ap.add_argument("--extra-scenes", nargs="*", default=[], help="more scene_ep1.json files whose objects the subset includes (e.g. a smoke-test task)")
ap.add_argument("--check", action="store_true", help="verify an existing root against its MANIGUARD_DATA.json; write nothing")
a = ap.parse_args()
root = pathlib.Path(a.root).resolve()
if a.check:
man = json.loads((root / "MANIGUARD_DATA.json").read_text())
files = manifest_files(root)
bad = [f for f, v in man["files"].items() if files.get(f) != v] + [f for f in files if f not in man["files"]]
ok = not bad and (root / "omnigibson.key").is_file()
print(json.dumps({"root": str(root), "ok": ok, "files": len(files), "bad": bad[:20], "key": (root / "omnigibson.key").is_file()}))
return 0 if ok else 1
if not a.key or not pathlib.Path(a.key).is_file():
raise SystemExit("--key: the path of an existing omnigibson.key is required (this script never downloads one or accepts a licence)")
root.mkdir(parents=True, exist_ok=True)
tasks = a.tasks if a.tasks is not None else all_tasks()
t0 = time.time()
cd = central_directory(ASSETS)
if a.full:
stream_all(ASSETS, cd, root / ASSETS["dest"]); covered = "all"
else:
pre, covered = subset_prefixes(tasks, a.extra_scenes)
sel = [m for m in cd if any(m["name"] == p or m["name"].startswith(p) for p in pre)]
missing = [p for p in pre if not any(m["name"] == p or m["name"].startswith(p) for m in cd)]
if missing:
raise SystemExit(f"the 3.7.2rc1 zip has no {missing}")
extract_members(ASSETS, sel, root / ASSETS["dest"])
rcd = central_directory(ROBOT)
extract_members(ROBOT, rcd, root / ROBOT["dest"])
lf = longfinger(root)
shutil.copyfile(a.key, root / "omnigibson.key"); os.chmod(root / "omnigibson.key", 0o644)
man = {"created": time.strftime("%Y-%m-%dT%H:%M:%S%z"), "behavior_1k_assets": {**{k: ASSETS[k] for k in ("url", "size", "sha256")}, "members": "all" if a.full else "task subset",
"tasks": covered}, "omnigibson_robot_assets": {k: ROBOT[k] for k in ("url", "size", "sha256")},
"longfinger": {"repo": "IDEAS-Lab-Northwestern/franka-panda-longfinger", "revision": LONGFINGER_REV, "files": lf},
"key": "omnigibson.key (copied from --key; not in this manifest)", "files": manifest_files(root)}
(root / "MANIGUARD_DATA.json").write_text(json.dumps(man, indent=1) + "\n")
du = sum(v[0] for v in man["files"].values())
print(json.dumps({"root": str(root), "files": len(man["files"]), "bytes": du, "wall_s": round(time.time() - t0, 1)}))
return 0
if __name__ == "__main__":
sys.exit(main())