Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
"""Generates `COVERAGE.md`: every public name of the e2b Python SDK (2.42.0,
2.56.0, 2.58.0), whether cua-e2b implements it, the test that proves it and,
for what it does not implement, the reason.
python -m conformance.matrix > ../COVERAGE.md
python -m conformance.matrix --check ../COVERAGE.md # drift guard
Inputs: the recorded surfaces (`golden/e2b-<v>-surface.json`), the live
package (`surface.dump()`), the waivers (`surface_waivers.py`) and the
evidence map (`coverage_evidence.py`). A name present in the real package is
`implemented` when the live surface has it with the real signature, and the
waiver says what it is otherwise. Every implemented class needs a test in
the evidence map (`test_coverage_matrix.py` enforces it).
"""
from __future__ import annotations
import json
import pathlib
import sys
from collections import Counter, OrderedDict
from fnmatch import fnmatchcase
from typing import Dict, List, Optional, Tuple
from conformance import surface
from conformance.coverage_evidence import EVIDENCE
from conformance.native_errors import LIMITS
from conformance.surface_waivers import waiver_for
GOLDEN = pathlib.Path(__file__).resolve().parent / "golden"
VERSIONS = ("2.42.0", "2.56.0", "2.58.0")
def load() -> Dict[str, dict]:
return {v: json.loads((GOLDEN / f"e2b-{v}-surface.json").read_text()) for v in VERSIONS}
def evidence_for(path: str) -> Optional[str]:
best = None
for pattern, tests in EVIDENCE:
if fnmatchcase(path, pattern) and (best is None or len(pattern) > len(best[0])):
best = (pattern, tests)
return best[1] if best else None
def rows() -> List[Tuple[str, str, str, str, str]]:
"""(path, kind, releases, status, detail) for every public name."""
goldens = load()
ours = surface.dump()
problems = {v: surface.compare(goldens[v], ours) for v in VERSIONS}
by_path: Dict[str, List[str]] = {}
for v in VERSIONS:
for p in problems[v]:
by_path.setdefault(p.split(": ", 1)[0].split("(", 1)[0], []).append(p)
seen: "OrderedDict[str, Tuple[str, set]]" = OrderedDict()
for v in VERSIONS:
for mod, items in goldens[v].items():
if "__import_error__" in items:
continue
for name, spec in items.items():
if name == "__all__":
continue
kind = spec["kind"]
base = f"{mod}.{name}"
seen.setdefault(base, (kind, set()))[1].add(v)
for m in spec.get("members", {}):
if m == "__init__":
continue
seen.setdefault(f"{base}.{m}", ("member", set()))[1].add(v)
# constructors count as a row of their own when they differ
if "__init__" in spec.get("members", {}):
seen.setdefault(f"{base}.__init__", ("constructor", set()))[1].add(v)
out = []
for path, (kind, vs) in seen.items():
rel = ", ".join(v for v in VERSIONS if v in vs)
issues = [
p
for key, ps in by_path.items()
if path == key or path.startswith(key + ".")
for p in ps
]
if not issues:
out.append((path, kind, rel, "implemented", ""))
continue
waivers = [waiver_for(p) for p in issues]
if all(waivers):
pick = next((w for w in waivers if w[1] == "native-error"), waivers[0])
out.append((path, kind, rel, pick[1], pick[2]))
else:
out.append((path, kind, rel, "OPEN", next(p for p, w in zip(issues, waivers) if not w)))
return out
def _owner(path: str, kind_of: Dict[str, str]) -> str:
"""The class (or module) a member row belongs to."""
head, _, tail = path.rpartition(".")
return head if head in kind_of else path
def render() -> str:
rs = rows()
counts = Counter(r[3] for r in rs)
kind_of = {r[0]: r[1] for r in rs}
lines = [
"# cua-e2b coverage of the e2b Python SDK",
"",
"Generated by `python -m conformance.matrix` from the recorded public surfaces of e2b "
f"{', '.join(VERSIONS)} (`tests/conformance/golden/e2b-<version>-surface.json`), the live "
"package, `tests/conformance/surface_waivers.py` and `tests/conformance/coverage_evidence.py`. "
"Do not edit by hand; `tests/test_coverage_matrix.py` fails when this file is stale.",
"",
"Status: **implemented** (same name, signature, fields and bases as the real release; the "
"test column names the proof), **native-error** (the call raises e2b's own exception class "
"and says why), **internal** (transport, generated or plumbing code that the SDK itself "
"calls; no Cua counterpart). Behaviour (not just shape) is proven by the shared "
"conformance suite against goldens recorded from the real service: see the cases listed "
'in `tests/conformance/suite.py` and `docs` page "E2B Python shim".',
"",
"## Summary (public names: modules' classes, functions, members, fields)",
"",
"| Status | Names |",
"|---|---|",
]
for status in ("implemented", "native-error", "internal", "OPEN"):
if counts.get(status):
lines.append(f"| {status} | {counts[status]} |")
lines.append(f"| total | {len(rs)} |")
lines.append("")
notable: "OrderedDict[Tuple[str, str], List[str]]" = OrderedDict()
for p, k, rel, st, d in rs:
if st != "implemented":
notable.setdefault((st, d), []).append(p)
lines += [
"## Not implemented one-to-one",
"",
"| Status | Names | Reason |",
"|---|---|---|",
]
for (st, d), names in notable.items():
shown = ", ".join(f"`{n}`" for n in names[:6]) + (
f" and {len(names) - 6} more" if len(names) > 6 else ""
)
lines.append(f"| {st} | {len(names)}: {shown} | {d.replace('|', '/')} |")
lines += [
"",
"## Calls that refuse or differ in some case",
"",
"These names exist and work; in the case below they raise e2b's own exception class "
"(with a message that names the way out) or differ in a documented way.",
"",
"| Call | Case | Outcome | Why | Test |",
"|---|---|---|---|---|",
]
for api, case, outcome, reason, test in LIMITS:
lines.append(f"| `{api}` | {case} | {outcome} | {reason.replace('|', '/')} | {test} |")
groups: "OrderedDict[str, List[Tuple[str, str]]]" = OrderedDict()
for p, k, rel, st, d in rs:
if st != "implemented":
continue
owner = _owner(p, kind_of)
groups.setdefault(owner, []).append((p, rel))
lines += [
"",
"## Implemented",
"",
"| Name | Kind | Releases | Members | Test |",
"|---|---|---|---|---|",
]
for owner, items in groups.items():
k = kind_of.get(owner, "")
rel = (
next(r for p, r in items if p == owner)
if any(p == owner for p, _ in items)
else items[0][1]
)
members = [p.rsplit(".", 1)[1] for p, _ in items if p != owner]
shown = ", ".join(sorted(members)) if members else ""
ev = evidence_for(owner) or "MISSING"
lines.append(f"| `{owner}` | {k} | {rel} | {shown} | {ev} |")
lines.append("")
return "\n".join(lines) + "\n"
def main(argv: List[str]) -> int:
text = render()
if "--check" in argv:
path = pathlib.Path(argv[argv.index("--check") + 1])
if path.read_text() != text:
print(f"{path} is stale: run `python -m conformance.matrix > {path}`", file=sys.stderr)
return 1
return 0
sys.stdout.write(text)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))