The first exam for code agents in the physical world: frozen robot tasks from many benchmark suites, each answered with a program and verified in a separate sandbox. Runs on Harbor.
Python ai-agentsbenchmarkcoding-agentsembodied-aievaluationharborllm-agentsrobot-learningroboticssimulation
Merge pull request #49 from JamesKrW/dev/changhe-libero-max-label (+1 more)
Changhe Chenc0f85b0e96e7
diff --git a/images/mujoco/Dockerfile b/images/mujoco/Dockerfileindex da605c3383..1fa9c1dc63 100644--- a/images/mujoco/Dockerfile+++ b/images/mujoco/Dockerfile@@ -7,7 +7,8 @@ RUN pip install --no-cache-dir mujoco==3.3.7 "numpy<2" scipy imageio imageio-ffmCOPY rcb_mj /opt/rcb_mj# glvnd's vendor file for the NVIDIA EGL library the container toolkit mounts. Without it EGL silently resolves to Mesa's# llvmpipe (software) even with a GPU: 0.1.1 rendered a 4-camera frame in ~1.3 s instead of ~7 ms. Tagged by hand-# (rcb-mujoco only; images/VERSION is shared): 0.1.2 = 0.1.1 + this file.+# (rcb-mujoco only; images/VERSION is shared): 0.1.2 = 0.1.1 + this file. 0.1.3 = 0.1.2 + the quadruped assets+# (MuJoCo Playground's locomotion/{go1,spot} scenes, Menagerie's Go1, Spot and Barkour; prepare_assets.sh).RUN mkdir -p /usr/share/glvnd/egl_vendor.d && printf '{"file_format_version":"1.0.0","ICD":{"library_path":"libEGL_nvidia.so.0"}}\n' > /usr/share/glvnd/egl_vendor.d/10_nvidia.jsonENV MUJOCO_GL=egl PYOPENGL_PLATFORM=egl NVIDIA_DRIVER_CAPABILITIES=all RCB_MJ_ROOT=/opt/rcb_mjRUN python -c "import mujoco, numpy; print('mujoco', mujoco.__version__)"diff --git a/images/mujoco/VERSION b/images/mujoco/VERSIONindex d917d3e26a..b1e80bb248 100644--- a/images/mujoco/VERSION+++ b/images/mujoco/VERSION@@ -1 +1 @@-0.1.2+0.1.3diff --git a/images/mujoco/prepare_assets.sh b/images/mujoco/prepare_assets.shindex 98be8e1d49..c5823403fb 100755--- a/images/mujoco/prepare_assets.sh+++ b/images/mujoco/prepare_assets.sh@@ -1,16 +1,19 @@#!/bin/bash# Assemble the rcb-mujoco asset tree (build context images/mujoco/rcb_mj, not committed) from pinned upstream commits.-# MuJoCo Menagerie 1b86ece (the commit MuJoCo Playground pins): aloha, kuka_iiwa_14, franka_emika_panda, robotiq_2f85-# MuJoCo Playground 4057c14 (Apache-2.0): manipulation/{aloha,franka_emika_panda,franka_emika_panda_robotiq}/xmls+# MuJoCo Menagerie 1b86ece (the commit MuJoCo Playground pins): aloha, kuka_iiwa_14, franka_emika_panda, robotiq_2f85,+# unitree_go1, boston_dynamics_spot, google_barkour_vb+# MuJoCo Playground 4057c14 (Apache-2.0): manipulation/{aloha,franka_emika_panda,franka_emika_panda_robotiq}/xmls,+# locomotion/{go1,spot}/xmls (Go1's with its heightfield and texture)# SimToolReal 313d5ae (MIT): left Sharpa HA4 hand, DexToolBench objects + goal trajectories (no policy, no RL code)set -euo pipefail-HERE=$(cd "$(dirname "$0")" && pwd); OUT=$HERE/rcb_mj; TMP=$(mktemp -d)+HERE=$(cd "$(dirname "$0")" && pwd); OUT=${RCB_MJ_OUT:-$HERE/rcb_mj}; TMP=$(mktemp -d)git clone -q https://github.com/google-deepmind/mujoco_menagerie.git $TMP/menagerie && git -C $TMP/menagerie checkout -q 1b86ece576591213e2b666ebf59508454200ca97git clone -q https://github.com/google-deepmind/mujoco_playground.git $TMP/playground && git -C $TMP/playground checkout -q 4057c14git clone -q https://github.com/tylerlum/simtoolreal.git $TMP/simtoolreal && git -C $TMP/simtoolreal checkout -q 313d5ae-rm -rf "${OUT:?}"; mkdir -p $OUT/menagerie $OUT/playground/aloha $OUT/playground/franka_emika_panda $OUT/playground/franka_emika_panda_robotiq $OUT/simtoolreal/assets/urdf $OUT/simtoolreal/dextoolbench-cp -r $TMP/menagerie/aloha $TMP/menagerie/kuka_iiwa_14 $TMP/menagerie/franka_emika_panda $TMP/menagerie/robotiq_2f85 $OUT/menagerie/; cp $TMP/menagerie/LICENSE $OUT/menagerie/+rm -rf "${OUT:?}"; mkdir -p $OUT/menagerie $OUT/playground/aloha $OUT/playground/franka_emika_panda $OUT/playground/franka_emika_panda_robotiq $OUT/playground/locomotion/go1 $OUT/playground/locomotion/spot $OUT/simtoolreal/assets/urdf $OUT/simtoolreal/dextoolbench+cp -r $TMP/menagerie/aloha $TMP/menagerie/kuka_iiwa_14 $TMP/menagerie/franka_emika_panda $TMP/menagerie/robotiq_2f85 $TMP/menagerie/unitree_go1 $TMP/menagerie/boston_dynamics_spot $TMP/menagerie/google_barkour_vb $OUT/menagerie/; cp $TMP/menagerie/LICENSE $OUT/menagerie/for e in aloha franka_emika_panda franka_emika_panda_robotiq; do cp -r $TMP/playground/mujoco_playground/_src/manipulation/$e/xmls $OUT/playground/$e/; done; cp $TMP/playground/LICENSE $OUT/playground/+for e in go1 spot; do cp -r $TMP/playground/mujoco_playground/_src/locomotion/$e/xmls $OUT/playground/locomotion/$e/; donecp -r $TMP/simtoolreal/assets/urdf/left_sharpa_ha4 $TMP/simtoolreal/assets/urdf/dextoolbench $OUT/simtoolreal/assets/urdf/cp -r $TMP/simtoolreal/dextoolbench/trajectories $OUT/simtoolreal/dextoolbench/; cp $TMP/simtoolreal/LICENSE $OUT/simtoolreal/rm -rf "${TMP:?}"; du -sh $OUTdiff --git a/scripts/task_conventions.py b/scripts/task_conventions.pyindex 81fdc2c911..ee16b20d64 100644--- a/scripts/task_conventions.py+++ b/scripts/task_conventions.py@@ -51,6 +51,7 @@ BENCHMARKS = [ # (directory prefix, label)("vlabench-", "VLABench"),("robocasa-gr1-", "RoboCasa-GR1"), # before "robocasa-": the first matching prefix wins("robocasa-", "RoboCasa"), ("robocasa365-", "RoboCasa365"),+ ("libero-max-", "LIBERO-MAX"),]HEADER_KEYS = {"Benchmark", "Mode", "Twin", "Simulator / robot", "Family / instance", "Oracle", "Agent time budget","Base image"}