Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
#!/usr/bin/env python3
"""Extract Python API documentation with griffe (static: nothing is imported).
Reads a JSON request on stdin and prints JSON on stdout::
{
"search_paths": ["libs/python/cua-sandbox"],
"objects": ["cua_sandbox.image.Image", "cua_sandbox.options.http"],
"modules": ["cua"] # optional: module docstring + static tables
}
Every requested object comes back with its kind, a rendered signature, a
Markdown description (RST literal blocks and ``double backticks`` converted),
the parsed Google-style sections (parameters, returns, raises) and, for
classes, its public members in source order. Output is deterministic: no
paths, dates or host details. The TypeScript side (python-sdk.ts) groups the
objects into pages.
Run with the pinned griffe from requirements.txt::
uv run --no-project --python 3.12 --with-requirements \\
scripts/docs-generators/requirements.txt \\
python scripts/docs-generators/extract_python_docs.py < request.json
"""
from __future__ import annotations
import ast
import json
import re
import sys
from pathlib import Path
from typing import Any
import griffe
from griffe import Alias, Attribute, Class, DocstringSectionKind, Function, Module, ParameterKind
REPO = Path(__file__).resolve().parents[2]
# --------------------------------------------------------------------- text
def rst_to_markdown(text: str) -> str:
"""Converts the RST idioms our docstrings use into Markdown."""
text = re.sub(r":(?:py:)?(?:class|meth|func|attr|mod|exc|data|obj):`~?([^`]+)`", r"`\1`", text)
text = re.sub(r"``([^`]+)``", r"`\1`", text)
lines = text.splitlines()
out: list[str] = []
i = 0
while i < len(lines):
line = lines[i]
stripped = line.rstrip()
if stripped.endswith("::") and not stripped.lstrip().startswith(("-", "*", ">>>")):
lead = stripped[:-2].rstrip()
base = len(line) - len(line.lstrip())
j = i + 1
while j < len(lines) and not lines[j].strip():
j += 1
block: list[str] = []
while j < len(lines) and (
not lines[j].strip() or len(lines[j]) - len(lines[j].lstrip()) > base
):
block.append(lines[j])
j += 1
while block and not block[-1].strip():
block.pop()
if block:
indent = min(len(b) - len(b.lstrip()) for b in block if b.strip())
if lead:
out.append(lead + ":")
out.append("")
out.append("```python")
out.extend(b[indent:] if b.strip() else "" for b in block)
out.append("```")
out.append("")
i = j
continue
if stripped.lstrip().startswith(">>>"):
j = i
block = []
while j < len(lines) and lines[j].strip():
block.append(lines[j].strip())
j += 1
out.extend(["", "```python", *block, "```", ""])
i = j
continue
out.append(line)
i += 1
joined = "\n".join(out)
return re.sub(r"\n{3,}", "\n\n", joined).strip()
def expr(value: Any) -> str:
return "" if value is None else str(value)
# ---------------------------------------------------------------- docstrings
def param_default(obj: Any, name: str) -> str:
"""The Default cell for a documented parameter: ``required`` when the
signature gives it no default, the literal default otherwise, ``()`` /
``{}`` for ``*args`` / ``**kwargs``, '' when the signature does not have it.
A class's parameters are its ``__init__``'s (or, for a dataclass without
one, its fields)."""
name = name.lstrip("*")
fn = obj
if isinstance(obj, Class):
fn = resolve(obj.members.get("__init__")) if "__init__" in obj.members else None
if isinstance(fn, Function):
for p in fn.parameters:
if p.name != name:
continue
if p.kind == ParameterKind.var_positional:
return "()"
if p.kind == ParameterKind.var_keyword:
return "{}"
return "required" if p.default is None else expr(p.default)
if isinstance(obj, Class):
field = obj.members.get(name)
if isinstance(field, Attribute) and "instance-attribute" not in field.labels:
return "required" if field.value is None else expr(field.value)
return ""
def sections(obj: Any) -> dict[str, Any]:
doc: dict[str, Any] = {"description": "", "params": [], "returns": "", "raises": []}
if not obj.docstring:
return doc
text_parts: list[str] = []
for section in obj.docstring.parse("google"):
kind = section.kind
if kind == DocstringSectionKind.text:
text_parts.append(section.value)
elif kind == DocstringSectionKind.parameters or kind == DocstringSectionKind.other_parameters:
for p in section.value:
doc["params"].append(
{
"name": p.name,
"type": expr(p.annotation),
"default": param_default(obj, p.name),
"description": rst_to_markdown(p.description or ""),
}
)
elif kind == DocstringSectionKind.returns:
doc["returns"] = rst_to_markdown(
" ".join(r.description or "" for r in section.value).strip()
)
elif kind == DocstringSectionKind.raises:
for r in section.value:
doc["raises"].append(
{"type": expr(r.annotation), "description": rst_to_markdown(r.description or "")}
)
elif kind == DocstringSectionKind.examples:
for sub_kind, value in section.value:
if str(sub_kind).endswith("examples"):
text_parts.append("Example:\n\n```python\n" + value.strip() + "\n```")
else:
text_parts.append(value)
elif kind == DocstringSectionKind.admonition:
title = section.title or section.value.kind or ""
text_parts.append(f"{title.capitalize()}: {section.value.description}")
else:
# Unknown sections keep their text so nothing is silently lost.
value = section.value
if isinstance(value, str):
text_parts.append(value)
doc["description"] = rst_to_markdown("\n\n".join(t.strip() for t in text_parts if t.strip()))
return doc
# ---------------------------------------------------------------- signatures
def signature(fn: Function, name: str | None = None, *, drop_self: bool = False) -> str:
params: list[str] = []
saw_kw_only = saw_pos_only = False
parameters = list(fn.parameters)
if drop_self and parameters and parameters[0].name in ("self", "cls"):
parameters = parameters[1:]
# Private (underscore) parameters are implementation details.
parameters = [p for p in parameters if not (p.name.startswith("_") and p.default is not None)]
for p in parameters:
if p.kind == ParameterKind.positional_only:
saw_pos_only = True
elif saw_pos_only:
params.append("/")
saw_pos_only = False
if p.kind == ParameterKind.keyword_only and not saw_kw_only:
params.append("*")
saw_kw_only = True
text = p.name
if p.kind == ParameterKind.var_positional:
text = "*" + text
saw_kw_only = True
elif p.kind == ParameterKind.var_keyword:
text = "**" + text
if p.annotation is not None:
text += f": {p.annotation}"
if p.default is not None and p.kind not in (
ParameterKind.var_positional,
ParameterKind.var_keyword,
):
text += f" = {p.default}" if p.annotation is not None else f"={p.default}"
params.append(text)
if saw_pos_only:
params.append("/")
prefix = "async def " if "async" in fn.labels else "def "
sig = f"{prefix}{name or fn.name}({', '.join(params)})"
if fn.returns is not None:
sig += f" -> {fn.returns}"
if len(sig) > 88 and params:
inner = ",\n".join(" " + p for p in params)
sig = f"{prefix}{name or fn.name}(\n{inner},\n)" + (
f" -> {fn.returns}" if fn.returns is not None else ""
)
return sig
# -------------------------------------------------------------------- objects
def resolve(obj: Any) -> Any:
seen = 0
while isinstance(obj, Alias) and seen < 10:
try:
obj = obj.target
except Exception: # noqa: BLE001 - external or unresolvable target
return obj
seen += 1
return obj
def labels(obj: Any) -> list[str]:
return sorted(str(label) for label in getattr(obj, "labels", set()))
def function_doc(fn: Function, owner: str | None = None) -> dict[str, Any]:
is_method = owner is not None
shown = fn.name
if fn.name == "__init__" and owner:
shown = owner.rsplit(".", 1)[-1]
return {
"kind": "function",
"name": fn.name,
"labels": labels(fn),
"signature": signature(fn, shown, drop_self=is_method),
"lineno": fn.lineno or 0,
**sections(fn),
}
def attribute_doc(attr: Attribute) -> dict[str, Any]:
return {
"kind": "attribute",
"name": attr.name,
"labels": labels(attr),
"type": expr(attr.annotation),
"value": expr(attr.value) if {"class-attribute", "module-attribute"} & set(attr.labels) else "",
"lineno": attr.lineno or 0,
**sections(attr),
}
_LITERAL_TYPES = {"int": "int", "float": "float", "str": "str", "bool": "bool", "NoneType": ""}
def inferred_type(attr: Attribute, cls: Class) -> str:
"""The type of an unannotated instance attribute, read from what
``__init__`` assigns it: a constructor parameter (its annotation), a class
instantiation (``self.screen = Screen(t)``) or a literal. '' when unknown."""
value = attr.value
if value is None:
return ""
text = expr(value).split(" or ", 1)[0].strip()
init = cls.members.get("__init__")
if isinstance(init, Function):
for param in init.parameters:
if param.name == text and param.annotation is not None:
return expr(param.annotation)
head = text.split("(", 1)[0]
if "(" in text and (head.split(".")[-1][:1].isupper() or head in ("list", "dict", "set", "tuple")):
return head.split(".")[-1]
try:
import ast
literal = ast.literal_eval(text)
except (ValueError, SyntaxError):
return ""
return _LITERAL_TYPES.get(type(literal).__name__, type(literal).__name__)
PUBLIC_DUNDERS = {"__init__", "__call__", "__aenter__", "__aexit__", "__enter__", "__exit__"}
def class_doc(cls: Class) -> dict[str, Any]:
members: list[dict[str, Any]] = []
for name, member in cls.members.items():
member = resolve(member)
if isinstance(member, Alias):
continue
if name.startswith("_") and name not in ("__init__", "__call__"):
continue
if isinstance(member, Function):
if name == "__init__" and not member.parameters:
continue
if "property" in member.labels:
members.append(
{
"kind": "attribute",
"name": name,
"labels": labels(member),
"type": expr(member.returns),
"value": "",
"lineno": member.lineno or 0,
**sections(member),
}
)
else:
members.append(function_doc(member, owner=cls.path))
elif isinstance(member, Attribute):
doc = attribute_doc(member)
if not doc["type"]:
doc["type"] = inferred_type(member, cls)
members.append(doc)
elif isinstance(member, Class):
continue
members.sort(key=lambda m: (m["name"] != "__init__", m["lineno"], m["name"]))
for m in members:
m.pop("lineno", None)
return {
"kind": "class",
"name": cls.name,
"labels": labels(cls),
"bases": [expr(b) for b in cls.bases],
"members": members,
**sections(cls),
}
def object_doc(root: Module, path: str) -> dict[str, Any]:
rel = path.split(".", 1)[1] if "." in path else ""
obj = root[rel] if rel else root
obj = resolve(obj)
if isinstance(obj, Attribute) and obj.parent is not None and isinstance(obj.value, str) is False:
# `ImageFileReference = Source`: document the class under the public name.
target = obj.parent.members.get(expr(obj.value))
if isinstance(target, Class):
doc = class_doc(target)
doc["name"] = obj.name
doc["path"] = path
doc["defined_in"] = target.path
return doc
if isinstance(obj, Alias):
return {"path": path, "kind": "external", "name": path.rsplit(".", 1)[-1], "target": obj.target_path}
if isinstance(obj, Class):
doc = class_doc(obj)
elif isinstance(obj, Function):
doc = function_doc(obj)
elif isinstance(obj, Attribute):
doc = attribute_doc(obj)
elif isinstance(obj, Module):
doc = {"kind": "module", "name": obj.name, **sections(obj)}
else: # pragma: no cover - griffe has no other object kinds
raise TypeError(f"unsupported object {path}")
doc.pop("lineno", None)
doc["path"] = path
doc["defined_in"] = obj.path
return doc
# ------------------------------------------------------ module-level tables
def static_tables(module: Module) -> dict[str, Any]:
"""`__all__`, the lazily resolved names (`_LAZY`) and plain rebinding
aliases (``OldName = _NewName``) of a module, read from its AST."""
source = Path(module.filepath).read_text()
tree = ast.parse(source)
wanted = {"_SANDBOX_NAMES", "_LAZY", "__all__"}
namespace: dict[str, Any] = {}
aliases: list[dict[str, str]] = []
imported_as: dict[str, str] = {}
imports: dict[str, str] = {}
for node in tree.body:
if isinstance(node, ast.ImportFrom):
for a in node.names:
if a.asname:
imported_as[a.asname] = a.name
if node.level == 0 and node.module:
imports[a.asname or a.name] = node.module
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
if isinstance(target, ast.Name) and target.id in wanted:
code = compile(ast.Module(body=[node], type_ignores=[]), "<static>", "exec")
exec(code, {"__builtins__": {}}, namespace) # literals and comprehensions only
elif (
isinstance(target, ast.Name)
and not target.id.startswith("_")
and isinstance(node.value, ast.Name)
and node.value.id in imported_as
):
aliases.append({"name": target.id, "target": imported_as[node.value.id]})
if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
if node.target.id in wanted and node.value is not None:
assign = ast.Assign(targets=[node.target], value=node.value)
ast.copy_location(assign, node)
code = compile(
ast.fix_missing_locations(ast.Module(body=[assign], type_ignores=[])),
"<static>",
"exec",
)
exec(code, {"__builtins__": {}}, namespace)
lazy = namespace.get("_LAZY", {})
return {
"all": list(namespace.get("__all__", [])),
"lazy": [
{"name": name, "module": mod, "attribute": attr, "extra": extra}
for name, (mod, attr, extra) in sorted(lazy.items())
],
"aliases": aliases,
"imports": dict(sorted(imports.items())),
}
def module_doc(root: Module, path: str) -> dict[str, Any]:
rel = path.split(".", 1)[1] if "." in path else ""
module = root[rel] if rel else root
doc = {
"path": path,
"kind": "module",
"name": module.name,
**sections(module),
**static_tables(module),
}
return doc
def main() -> int:
request = json.load(sys.stdin)
search = [str((REPO / p).resolve()) for p in request["search_paths"]]
packages: dict[str, Module] = {}
def package(path: str) -> Module:
top = path.split(".", 1)[0]
if top not in packages:
packages[top] = griffe.load(
top,
search_paths=search,
docstring_parser="google",
allow_inspection=False,
resolve_aliases=False,
)
return packages[top]
objects = [object_doc(package(p), p) for p in request.get("objects", [])]
modules = [module_doc(package(p), p) for p in request.get("modules", [])]
json.dump(
{"griffe": griffe.__name__, "objects": objects, "modules": modules},
sys.stdout,
indent=1,
sort_keys=True,
)
sys.stdout.write("\n")
return 0
if __name__ == "__main__":
sys.exit(main())