Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

9.4 KB
Raw Blame
"""Offline AOTInductor packaging/parity check and CUDA forward benchmark.

Run with --model pointing at a local Laya checkpoint. Without it, use a tiny
random ModernBERT (no downloads). Outputs and compiler caches belong outside git.
"""
import argparse
import copy
import json
import os
from pathlib import Path
import shutil
import statistics
import subprocess
import sys
import time

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("--model", help="Local checkpoint directory; never downloads weights")
    ap.add_argument("--output-dir", required=True)
    ap.add_argument("--expect-export-failure", action="store_true", help="Measure the unpatched baseline")
    ap.add_argument("--compare-eager", help="Baseline eager.pt to check bit identity")
    args = ap.parse_args()
    try:
        import torch
        from torch.export import Dim
        from torch._inductor import aoti_compile_and_package, aoti_load_package
    except ImportError:
        print("SKIP: torch.export / AOTInductor unavailable")
        return
    if not torch.cuda.is_available() or not shutil.which("g++"):
        print("SKIP: CUDA and a C++ compiler are required")
        return
    from laya.common import DecisionModel, QTYPES, build_sequence

    outdir = Path(args.output_dir)
    outdir.mkdir(parents=True, exist_ok=True)
    torch.manual_seed(0)
    torch.set_num_threads(4)
    report = {"torch": torch.__version__, "model": args.model or "tiny random ModernBERT",
              "gpu": torch.cuda.get_device_name(), "cache": os.environ.get("TORCHINDUCTOR_CACHE_DIR"),
              "gpu_state_before": subprocess.check_output(["nvidia-smi"], text=True)}
    if args.model:
        if not Path(args.model).is_dir():
            ap.error("--model must be an existing local checkpoint directory")
        from laya.agent import Agent
        agent = Agent(args.model, device="cuda", compile=False)
        model = agent.model.eval()
    else:
        from transformers import ModernBertConfig, ModernBertModel
        cfg = ModernBertConfig(vocab_size=128, hidden_size=64, intermediate_size=128,
                               num_hidden_layers=1, num_attention_heads=4, max_position_embeddings=512,
                               pad_token_id=0, bos_token_id=1, eos_token_id=2, cls_token_id=1, sep_token_id=2,
                               reference_compile=False, attn_implementation="sdpa")
        model = DecisionModel(ModernBertModel(cfg), head_layers=1, dropout=0).cuda().eval()

    cases = []
    for rows, tokens, markers in ((2, 128, 3), (1, 256, 3), (4, 160, 5)):
        ids = torch.zeros((rows, tokens), dtype=torch.long)
        att = torch.zeros_like(ids)
        pos = torch.zeros((rows, markers), dtype=torch.long)
        mask = torch.zeros_like(pos, dtype=torch.bool)
        qt = torch.arange(rows) % 3
        for row in range(rows):
            count = markers if row == 0 else max(1, markers - row)
            if args.model:
                kind = ("choice", "score", "noul")[row % 3]
                question = {"type": kind, "instructions": "Which team should handle this request?"}
                if kind == "choice":
                    question["criteria"] = dict(list({"billing": "refunds", "tech": "bugs", "sales": "pricing",
                                                      "account": "login", "support": "other"}.items())[:count])
                elif kind == "score":
                    question["criteria"] = ["low", "medium", "high"][:count]
                else:
                    question["instructions"] = "Is the customer asking for a refund?"
                qq = agent._to_internal(question)
                seq, mp = build_sequence(agent.tok, {"body": "We were billed twice. Please refund the duplicate charge."},
                                         qq, tokens, min(tokens, agent.cfg["head_max_len"]))
                qt[row] = QTYPES[qq["t"]]
                ids[row].fill_(agent.tok.pad_token_id or 0)
            else:
                seq = torch.randint(1, 128, (tokens - 16 * row,)).tolist()
                mp = list(range(1, count + 1))
            assert len(seq) <= tokens and len(mp) <= markers
            ids[row, :len(seq)] = torch.tensor(seq)
            att[row, :len(seq)] = 1
            pos[row, :len(mp)] = torch.tensor(mp)
            mask[row, :len(mp)] = True
        cases.append(tuple(t.cuda() for t in (ids, att, pos, mask, qt)))
    torch.save([[t.cpu() for t in case] for case in cases], outdir / "inputs.pt")

    def latency(fn, inputs):
        for _ in range(10):
            fn(*inputs)
        torch.cuda.synchronize()
        samples = []
        for _ in range(5):
            start = time.perf_counter()
            for _ in range(30):
                fn(*inputs)
            torch.cuda.synchronize()
            samples.append((time.perf_counter() - start) * 1000 / 30)
        return round(statistics.median(samples), 4)

    def delta(ref, got):
        result = {}
        for name, a, b in zip(("decision", "action"), ref, got):
            assert torch.isfinite(b).all()
            result[name + "_logits"] = (a.float() - b.float()).abs().max().item()
            result[name + "_probabilities"] = (a.float().softmax(-1) - b.float().softmax(-1)).abs().max().item()
            result[name + "_answers_equal"] = bool(torch.equal(a.argmax(-1), b.argmax(-1)))
        return result

    with torch.no_grad():
        # Pin the existing fp32-parameter path, both without AMP and with fp16/bf16 AMP.
        eager = {}
        for dtype in (torch.float32, torch.float16, torch.bfloat16):
            with torch.autocast("cuda", dtype=dtype, enabled=dtype != torch.float32):
                eager[str(dtype)] = [[x.cpu() for x in model(*case)] for case in cases]
        torch.save(eager, outdir / "eager.pt")
        if args.compare_eager:
            baseline = torch.load(args.compare_eager, weights_only=True)
            for key in eager:
                for old, new in zip(baseline[key], eager[key]):
                    for a, b in zip(old, new):
                        torch.testing.assert_close(a, b, rtol=0, atol=0)
            report["existing_eager_max_delta"] = 0.0

        with torch.autocast("cuda", dtype=torch.bfloat16):
            report["amp_eager_ms"] = [latency(model, case) for case in cases]
            compiled = torch.compile(model, dynamic=True)
            report["amp_compiled_ms"] = [latency(compiled, case) for case in cases]
        print(json.dumps(report), flush=True)
        del compiled
        candidate = copy.deepcopy(model).to(torch.bfloat16)
        rows, markers = Dim("rows", min=1, max=8), Dim("markers", min=2, max=8)
        tokens = 16 * Dim("tokens16", min=2, max=32)
        dynamic = ({0: rows, 1: tokens}, {0: rows, 1: tokens}, {0: rows, 1: markers},
                   {0: rows, 1: markers}, {0: rows})
        start = time.perf_counter()
        stage = "export"
        try:
            exported = torch.export.export(candidate, cases[0], dynamic_shapes=dynamic, strict=False)
            report["export_s"] = time.perf_counter() - start
            print("Export complete; packaging", flush=True)
            stage = "package"
            start = time.perf_counter()
            path = aoti_compile_and_package(exported, package_path=str(outdir / "decision.pt2"))
            report["package_s"] = time.perf_counter() - start
            report["package_bytes"] = Path(path).stat().st_size
        except Exception as exc:
            if not args.expect_export_failure or not any(s in str(exc) for s in ("same dtype", "dtype mismatch")):
                raise
            report.update(failure_stage=stage, failure_s=time.perf_counter() - start, error=str(exc),
                          package_s=None, package_bytes=None, aoti_ms=None)
        else:
            assert not args.expect_export_failure, "Expected the baseline dtype failure"
            start = time.perf_counter()
            runner = aoti_load_package(path)
            report["load_s"] = time.perf_counter() - start
            report["cases"] = []
            for i, case in enumerate(cases):
                reference = candidate(*case)
                actual = runner(*case)
                diff = delta(reference, actual)
                amp_diff = delta([x.cuda() for x in eager["torch.bfloat16"][i]], actual)
                for comparison in (diff, amp_diff):
                    for name in ("decision", "action"):
                        assert comparison[name + "_probabilities"] < 0.02, comparison
                        assert comparison[name + "_answers_equal"], comparison
                item = {"shape": [*case[0].shape, case[2].shape[1]], "aoti_vs_bf16_eager": diff,
                        "aoti_vs_amp_eager": amp_diff,
                        "bf16_eager_ms": latency(candidate, case), "aoti_ms": latency(runner, case)}
                report["cases"].append(item)
            compiled = torch.compile(candidate, dynamic=True)
            for item, case in zip(report["cases"], cases):
                item["bf16_compiled_ms"] = latency(compiled, case)
                item["compiled_vs_bf16_eager"] = delta(candidate(*case), compiled(*case))
    report["gpu_state_after"] = subprocess.check_output(["nvidia-smi"], text=True)
    (outdir / "results.json").write_text(json.dumps(report, indent=2) + "\n")
    print(json.dumps(report, indent=2), flush=True)
    print("PASS: expected baseline failure" if args.expect_export_failure else "PASS: AOTI package, parity and latency")


if __name__ == "__main__":
    main()