Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""The E2B coverage matrix: every public symbol of the Python and TypeScript
SDKs, and what the Cua gateway does with it.
The symbols come from the committed surface snapshots (surface.py for Python,
surface_ts.mjs for TypeScript, one per SDK version). `coverage.json` says, for
each symbol (by regular expression over a normalised `Class.member`), whether
it is `implemented` (with the cases that test it), `client` (the SDK does it
without calling a server), or `native-error` (the gateway answers with E2B's
own error and a documented reason). A symbol no rule covers fails the check:
that is how a method that ships in a new SDK release fails the weekly drift
job until someone decides what the gateway does with it.
python coverage.py --check # exit 1 on an unmapped symbol or a missing test
python coverage.py --markdown OUT.md # the matrix, for maps/e2b-coverage.md
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import defaultdict
from pathlib import Path
HERE = Path(__file__).resolve().parent
FIX = HERE / "fixtures"
def snake(s: str) -> str:
return re.sub(r"(?<=[a-z0-9])([A-Z])", r"_\1", s).lower()
def vkey(v: str):
return tuple(int(x) for x in re.findall(r"\d+", v))
def norm_class(name: str) -> str:
"""`e2b.AsyncSandbox` and `Sandbox` are one symbol."""
name = name.removeprefix("e2b.")
name = name.split(".")[-1] if name.count(".") > 0 else name
for pre in ("Async",):
if name.startswith(pre) and len(name) > len(pre) and name[len(pre)].isupper():
name = name[len(pre) :]
return name
KINDS: dict[str, str] = {}
def py_rows() -> dict[str, set[str]]:
rows: dict[str, set[str]] = defaultdict(set)
for f in sorted(FIX.glob("surface-[0-9]*.json"), key=lambda p: vkey(p.stem)):
ver = f.stem.removeprefix("surface-")
d = json.loads(f.read_text())
for name, e in d.get("exports", {}).items():
cls = norm_class(name)
if e["kind"] in ("module",):
continue
rows[cls].add(ver)
for m, sig in e.get("members", {}).items():
if m.startswith("__") and m != "__init__":
continue
rows[f"{cls}.{m}"].add(ver)
KINDS.setdefault(f"{cls}.{m}", "method" if str(sig).startswith("(") else "field")
for path, members in d.get("classes", {}).items():
cls = norm_class(path)
rows[cls].add(ver)
for m in members:
if not m.startswith("__"):
rows[f"{cls}.{m}"].add(ver)
for route in d.get("routes", []):
rows["route " + route].add(ver)
return rows
def ts_rows() -> dict[str, set[str]]:
rows: dict[str, set[str]] = defaultdict(set)
for f in sorted(FIX.glob("surface-ts-*.json"), key=lambda p: vkey(p.stem)):
ver = f.stem.removeprefix("surface-ts-")
d = json.loads(f.read_text())
for name, e in d["exports"].items():
if name in ("components", "paths", "default"):
if name == "paths":
for p in e.get("members", {}):
rows["route " + p].add(ver)
continue
rows[name].add(ver)
for part in ("members", "static"):
for m, sig in e.get(part, {}).items():
if m == "prototype":
continue
rows[f"{name}.{snake(m)}"].add(ver)
KINDS.setdefault(
f"{name}.{snake(m)}",
"method" if str(sig).lstrip().startswith(("(", "<")) else "field",
)
return rows
def load_rules():
d = json.loads((HERE / "coverage.json").read_text())
rules = []
for r in d["rules"]:
r["_re"] = [
re.compile(p) for p in (r["match"] if isinstance(r["match"], list) else [r["match"]])
]
rules.append(r)
return d, rules
def classify(sym: str, rules):
for r in rules:
if any(p.fullmatch(sym) for p in r["_re"]):
return r
return None
def case_names() -> set[str]:
names: set[str] = set()
for f in (HERE / "cases").glob("*.json"):
names |= {c["name"] for c in json.loads(f.read_text())["cases"]}
return names
def rust_tests() -> set[str]:
out: set[str] = set()
root = HERE.parents[2] / "libs/cua/crates/cua-compat"
for f in list((root / "src").rglob("*.rs")) + list((root / "tests").rglob("*.rs")):
out |= set(re.findall(r"fn\s+(\w+)\s*\(", f.read_text()))
return out
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--check", action="store_true")
ap.add_argument("--markdown")
a = ap.parse_args()
cfg, rules = load_rules()
py, ts = py_rows(), ts_rows()
symbols = sorted(set(py) | set(ts))
cases, rust = case_names(), rust_tests()
problems: list[str] = []
table = []
for sym in symbols:
r = classify(sym, rules)
if r is None:
problems.append(
f"unmapped: {sym} (py {sorted(py.get(sym, []), key=vkey)[-1:]} ts {sorted(ts.get(sym, []), key=vkey)[-1:]})"
)
continue
if r["status"] == "todo":
problems.append(f"todo: {sym}")
for t in r.get("tests", []):
if t.startswith("rust:"):
if t[5:] not in rust:
problems.append(f"{r['id']}: unknown Rust test {t}")
elif t.startswith("conformance:"):
pass
elif t not in cases:
problems.append(f"{r['id']}: unknown case {t}")
if r["status"] == "implemented" and not r.get("tests"):
problems.append(f"{r['id']}: implemented without a test")
if r["status"] == "native-error" and not r.get("reason"):
problems.append(f"{r['id']}: native-error without a reason")
table.append((sym, r))
if a.markdown:
write_markdown(Path(a.markdown), cfg, table, py, ts)
counts = defaultdict(int)
for _, r in table:
counts[r["status"]] += 1
print(f"{len(symbols)} symbols: " + ", ".join(f"{k} {v}" for k, v in sorted(counts.items())))
for p in problems:
print(" " + p)
return 1 if (a.check and problems) else 0
PRIMARY = {
"Sandbox",
"Commands",
"Filesystem",
"Pty",
"Git",
"CommandHandle",
"CommandInputHandle",
"WatchHandle",
"SandboxPaginator",
"SnapshotPaginator",
"SecretPaginator",
"Secret",
"Volume",
"Template",
"TemplateBase",
"TemplateBuilder",
"ConnectionConfig",
}
def first_seen(sym: str, rows: dict[str, set[str]]) -> str:
v = sorted(rows.get(sym, []), key=vkey)
return v[0] if v else "-"
def write_markdown(path: Path, cfg: dict, table, py, ts) -> None:
groups: dict[str, list] = defaultdict(list)
for sym, r in table:
groups[r["id"]].append(sym)
by_id = {r["id"]: r for _, r in table}
lines = [cfg["preamble"].rstrip(), ""]
pyv = sorted({v for s in py.values() for v in s}, key=vkey)
tsv = sorted({v for s in ts.values() for v in s}, key=vkey)
lines.append(
f"SDK versions covered: Python `e2b` {', '.join(pyv)}; npm `e2b` {', '.join(tsv)}. Sources: the installed wheels (`surface.py`) and the `.d.ts` of the npm packages (`surface_ts.mjs`), read as data."
)
lines.append("")
lines.append("## Methods of the main classes")
lines.append("")
lines.append(
"One row per method, with the first SDK version that has it (`py` Python sync and async, `ts` TypeScript; the Python `Async*` classes and the sync ones are one row). Names are Python's snake_case; the TypeScript method is the camelCase of the same name."
)
lines.append("")
lines.append("| Method | py since | ts since | Status | Cases | Notes |")
lines.append("|---|---|---|---|---|---|")
rowsets = []
for sym, r in table:
cls = sym.split(".")[0]
if (
"." in sym
and cls in PRIMARY
and KINDS.get(sym) == "method"
and not sym.endswith(".__init__")
):
rowsets.append((sym, r))
for sym, r in sorted(rowsets):
tests = ", ".join(f"`{t}`" for t in r.get("tests", [])[:4]) + (
" ..." if len(r.get("tests", [])) > 4 else ""
)
note = r.get("reason") or r.get("note", "")
note = (note[:160] + "...") if len(note) > 160 else note
lines.append(
f"| `{sym}` | {first_seen(sym, py)} | {first_seen(sym, ts)} | {r['status']} | {tests or 'n/a'} | {note} |"
)
lines.append("")
lines.append("## Everything else, by family")
lines.append("")
lines.append(
"Option objects, result types, enums, exceptions, routes and members not listed above, grouped by the rule that covers them (`tests/compat/e2b/coverage.json`)."
)
lines.append("")
sections: dict[str, list[str]] = defaultdict(list)
for rid in groups:
sections[by_id[rid].get("section", "Other")].append(rid)
for sec in cfg["sections"]:
ids = sections.get(sec, [])
if not ids:
continue
lines.append(f"### {sec}")
lines.append("")
lines.append("| Rule | Symbols | Status | Cases | Notes |")
lines.append("|---|---|---|---|---|")
for rid in ids:
r = by_id[rid]
syms = groups[rid]
tests = ", ".join(f"`{t}`" for t in r.get("tests", [])) or "n/a"
note = r.get("reason") or r.get("note", "")
lines.append(f"| `{rid}` | {len(syms)} | {r['status']} | {tests} | {note} |")
lines.append("")
path.write_text("\n".join(lines))
if __name__ == "__main__":
sys.exit(main())