Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

7.7 KB
Raw Blame
"""Static audit of an ONNX decoder graph before shipping it on WebGPU.

Catches the problems that made Ternary Bonsai 2 prefill take minutes in the
browser (see docs/webgpu-models.md):

* control-flow ops that ONNX Runtime Web's WebGPU execution provider does not
  implement (`Scan`, `Loop`), including inside `If` branches and model-local
  functions. ONNX Runtime places them on the CPU and their bodies then run
  once per iteration with a GPU round trip, so a per-token recurrence becomes
  a per-token CPU loop;
* a `logits` output that keeps the sequence dimension, so prefill computes and
  downloads `[sequence_length, vocab]` scores (0.5 MB per prompt token for a
  248k vocabulary in FP16) and can exceed a 2 GB WebGPU buffer limit.

A `num_logits_to_keep` input does not lift the logits blocker by default:
Transformers.js passes 1 only from text `generate()`; its multimodal path and
direct `forward()` calls pass 0, which keeps every position. Pass
`--text-generate-only` for a graph that is used only through text
`generate()`, and the finding becomes a warning.

Only the graph structure is read (external weights are not loaded). Exit code
1 means at least one blocker was found. This is a static check: also confirm
node placement and prefill scaling on hardware WebGPU as the doc describes.

  python scripts/models/audit_webgpu_onnx.py path/to/decoder_model_merged_q4f16.onnx [--json] [--text-generate-only]
"""

import argparse
import collections
import json
import sys
from pathlib import Path

import onnx

# Verified with onnxruntime-web 1.27: the WebGPU EP logs "webgpu kernel not found
# in registries for Op type: Scan" (and the same for Loop) and places them on
# the CPU. `If` has a WebGPU kernel and is fine.
CPU_ONLY_CONTROL_FLOW = {"Scan", "Loop"}
LOGITS_NAMES = {"logits"}


def walk(graph, location=""):
    for node in graph.node:
        here = f"{location}/{node.name or node.op_type}"
        yield node, here
        for attribute in node.attribute:
            if attribute.type == onnx.AttributeProto.GRAPH:
                yield from walk(attribute.g, f"{here}:{attribute.name}")
            elif attribute.type == onnx.AttributeProto.GRAPHS:
                for index, subgraph in enumerate(attribute.graphs):
                    yield from walk(subgraph, f"{here}:{attribute.name}[{index}]")


def dims(value_info):
    return [d.dim_param or d.dim_value for d in value_info.type.tensor_type.shape.dim]


def audit(path, text_generate_only=False):
    model = onnx.load(path, load_external_data=False)
    graph = model.graph
    blockers, warnings, info = [], [], {}

    control_flow = collections.defaultdict(list)
    op_counts = collections.Counter()
    # Model-local functions are inlined by ONNX Runtime, so audit their bodies too.
    bodies = [(graph, "")] + [(function, f"function:{function.domain}::{function.name}")
                              for function in model.functions]
    nodes = [entry for body, location in bodies for entry in walk(body, location)]
    for node, location in nodes:
        domain = node.domain or "ai.onnx"
        op_counts[f"{domain}::{node.op_type}"] += 1
        if node.op_type in CPU_ONLY_CONTROL_FLOW and domain == "ai.onnx":
            control_flow[node.op_type].append(location)
    for op_type, locations in control_flow.items():
        blockers.append({
            "check": "cpu-only-control-flow",
            "op": op_type,
            "count": len(locations),
            "examples": locations[:3],
            "why": f"No WebGPU kernel for {op_type}: it runs on the CPU and its body is dispatched per iteration "
                   "with a GPU round trip (Bonsai 2: ~6.5 ms per prompt token per layer on an RTX 5090).",
        })

    inputs = {i.name for i in graph.input}
    for output in graph.output:
        if output.name not in LOGITS_NAMES:
            continue
        shape = dims(output)
        info["logits_shape"] = shape
        if len(shape) == 3 and shape[1] != 1:
            vocab = shape[2] if isinstance(shape[2], int) and shape[2] else None
            entry = {
                "check": "full-sequence-logits",
                "shape": shape,
                "why": "Prefill computes and downloads logits for every prompt token"
                       + (f" ({vocab * 2 / 1e6:.2f} MB per token in FP16)" if vocab else "")
                       + "; generation reads only the last position.",
            }
            if "num_logits_to_keep" in inputs and text_generate_only:
                entry["why"] += (" The graph accepts num_logits_to_keep, which Transformers.js sets to 1 in text "
                                 "generate(); --text-generate-only says no other path runs this graph.")
                warnings.append(entry)
            else:
                if "num_logits_to_keep" in inputs:
                    entry["why"] += (" The graph accepts num_logits_to_keep, but Transformers.js passes 0 (keep all) "
                                     "on its multimodal path and in direct forward() calls, so slice to the last "
                                     "position in the graph, or pass --text-generate-only if text generate() is "
                                     "the only caller.")
                blockers.append(entry)

    local_functions = {f"{function.domain}::{function.name}" for function in model.functions}
    contrib = sorted(k for k in op_counts if not k.startswith("ai.onnx::") and k not in local_functions)
    info.update({
        "opsets": {o.domain or "ai.onnx": o.version for o in model.opset_import},
        "nodes": sum(op_counts.values()),
        "if_nodes": op_counts.get("ai.onnx::If", 0),
        "local_functions": sorted(local_functions),
        "contrib_ops": {k: op_counts[k] for k in contrib},
        "external_data_files": sorted({
            entry.value for tensor in graph.initializer if tensor.data_location == onnx.TensorProto.EXTERNAL
            for entry in tensor.external_data if entry.key == "location"
        }),
        "inputs": [i.name for i in graph.input if not i.name.startswith(("past", "present"))],
    })
    return {"graph": str(path), "blockers": blockers, "warnings": warnings, "info": info}


def main():
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("graphs", type=Path, nargs="+")
    parser.add_argument("--json", action="store_true",
                        help='print one JSON document: {"blocked": bool, "graphs": [report, ...]}')
    parser.add_argument("--text-generate-only", action="store_true",
                        help="the graph runs only through Transformers.js text generate(), which passes "
                             "num_logits_to_keep=1; downgrade full-sequence logits with that input to a warning")
    args = parser.parse_args()
    failed = False
    reports = []
    for path in args.graphs:
        report = audit(path, text_generate_only=args.text_generate_only)
        failed = failed or bool(report["blockers"])
        reports.append(report)
        if args.json:
            continue
        status = "BLOCKED" if report["blockers"] else "ok"
        print(f"{status}: {path}")
        for item in report["blockers"]:
            print(f"  blocker  {item['check']}: {item.get('op') or item.get('shape')} "
                  f"(x{item.get('count', 1)}) - {item['why']}")
        for item in report["warnings"]:
            print(f"  warning  {item['check']}: {item.get('shape')} - {item['why']}")
        info = report["info"]
        print(f"  logits {info.get('logits_shape')}, {info['nodes']} nodes, {info['if_nodes']} If, "
              f"contrib {', '.join(info['contrib_ops']) or 'none'}")
    if args.json:
        print(json.dumps({"blocked": failed, "graphs": reports}, indent=2))
    sys.exit(1 if failed else 0)


if __name__ == "__main__":
    main()