Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
ci(dotnet): regenerate goldens from Python and run parity tests on every push
kevin-gatimu8ccaf969f2cf
diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.ymlnew file mode 100644index 00000000..0312bd55--- /dev/null+++ b/.github/workflows/dotnet.yml@@ -0,0 +1,201 @@+name: .NET++# Parity lane for laya-dotnet. The golden fixtures under laya-dotnet/tests/Laya.Tests/golden are+# recorded from the Python `laya` package, so this runs on EVERY push and pull request (no path+# filter): a change to laya/ that moves what the goldens capture must show up here as a red+# check, not as silent drift. Each run re-records the goldens from the Python code at that commit+# (tools/regen_golden.py) and runs the C# tests against the fresh copies.++on:+ pull_request:+ push:+ workflow_dispatch:++permissions:+ contents: read++concurrency:+ group: dotnet-${{ github.event_name == 'pull_request' && github.ref || github.sha }}+ # Cancel outdated pull request runs. Pushes get a per-commit group so each one finishes.+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}++env:+ # transformers probes for TensorFlow at import; when TF is present its abseil runtime can+ # deadlock model construction. Laya is torch-only.+ USE_TF: "0"+ USE_TORCH: "1"+ TOKENIZERS_PARALLELISM: "false"+ DOTNET_NOLOGO: "true"+ DOTNET_CLI_TELEMETRY_OPTOUT: "true"+ GOLDEN: laya-dotnet/tests/Laya.Tests/golden++jobs:+ build:+ name: build + model-free tests+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0+ with:+ python-version: "3.12"+ cache: pip+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt+ - name: Install+ run: |+ python -m pip install --upgrade pip+ # CPU-only torch; the routing goldens only import laya, they never run a model+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu+ pip install -r laya-dotnet/tools/requirements-regen.txt+ - name: Test regeneration cache and skip guard+ run: python laya-dotnet/tools/test_ci_tools.py+ - name: Regenerate the routing goldens from Python+ run: python laya-dotnet/tools/regen_golden.py --checkpoint routing+ - name: Golden drift vs. committed (informational)+ run: |+ git --no-pager diff --stat -- "$GOLDEN"+ git status --short -- "$GOLDEN"+ - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0+ with:+ global-json-file: laya-dotnet/global.json+ - name: Build (library, samples and tests, warnings are errors)+ working-directory: laya-dotnet+ run: dotnet build Laya.slnx -c Release+ - name: Tests that need no model+ working-directory: laya-dotnet+ run: >+ dotnet test --solution Laya.slnx -c Release --no-build+ --results-directory "$RUNNER_TEMP/trx" --report-xunit-trx --report-xunit-trx-filename build.trx+ --filter-not-class Laya.Tests.PredictParityTests+ --filter-not-class Laya.Tests.ShapeSweepTests+ --filter-not-class Laya.Tests.TokenizerParityTests+ --filter-not-class Laya.Tests.RouterEndToEndTests+ --filter-not-class Laya.Tests.ShortlistEndToEndTests+ # With the model-backed classes filtered out nothing may be skipped, so every checkpoint+ # is listed as expected to run.+ - name: No test was skipped+ if: ${{ !cancelled() }}+ run: >+ python laya-dotnet/tools/check_test_skips.py "$RUNNER_TEMP/trx/build.trx"+ --have english multilingual typed-decisions --min-passed 600+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ if: ${{ failure() }}+ with:+ name: dotnet-build-diagnostics+ path: |+ ${{ runner.temp }}/trx+ ${{ env.GOLDEN }}/routing++ parity:+ name: parity (${{ matrix.name }})+ runs-on: ubuntu-latest+ timeout-minutes: 60+ strategy:+ fail-fast: false+ matrix:+ include:+ - name: english+ checkpoints: english+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests+ min-passed: 50+ - name: multilingual+ checkpoints: multilingual+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests+ min-passed: 50+ - name: typed-decisions+ checkpoints: typed-decisions+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests+ min-passed: 50+ # The router end-to-end tests load english and multilingual side by side.+ - name: router+ checkpoints: english multilingual+ filter: --filter-class Laya.Tests.RouterEndToEndTests+ min-passed: 1+ steps:+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0+ with:+ python-version: "3.12"+ cache: pip+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt+ - name: Install+ run: |+ python -m pip install --upgrade pip+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu+ pip install -r laya-dotnet/tools/requirements-regen.txt++ # The exports are the slow part (download, trace, verify). They depend only on the pinned+ # Hugging Face revision (HF_REVISION in regen_golden.py), the two exporters, the pinned+ # toolchain and the model definition in laya/common.py, so that is the cache key; the+ # script re-checks the same inputs itself before it reuses a directory. The goldens are+ # never cached: they are always re-recorded.+ - name: Cache pinned Hugging Face downloads+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: ~/.cache/huggingface/hub+ key: laya-hf-${{ runner.os }}-${{ matrix.name }}-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py') }}+ - name: Cache exported english artifacts+ if: ${{ contains(matrix.checkpoints, 'english') }}+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: |+ onnx/english+ onnx-split/english+ key: laya-onnx-${{ runner.os }}-english-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}+ - name: Cache exported multilingual artifacts+ if: ${{ contains(matrix.checkpoints, 'multilingual') }}+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: |+ onnx/multilingual+ onnx-split/multilingual+ key: laya-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}+ - name: Cache exported typed-decisions artifacts+ if: ${{ contains(matrix.checkpoints, 'typed-decisions') }}+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0+ with:+ path: |+ onnx/typed-decisions+ onnx-split/typed-decisions+ key: laya-onnx-${{ runner.os }}-typed-decisions-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}++ - name: Export the checkpoints and regenerate their goldens from Python+ run: |+ args=()+ for c in ${{ matrix.checkpoints }}; do args+=(--checkpoint "$c"); done+ python laya-dotnet/tools/regen_golden.py "${args[@]}"+ # Informational only: float noise across platforms is expected, the C# tests and their+ # tolerances are the gate.+ - name: Golden drift vs. committed (informational)+ run: |+ git --no-pager diff --stat -- "$GOLDEN"+ git status --short -- "$GOLDEN"++ - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0+ with:+ global-json-file: laya-dotnet/global.json+ # The fused layout is found through LAYA_ONNX_ROOT; the split layout through the test+ # project's walk-up to <repo>/onnx-split, which is where the script wrote it.+ - name: Run the tests against the fresh goldens+ working-directory: laya-dotnet+ env:+ LAYA_ONNX_ROOT: ${{ github.workspace }}/onnx+ LAYA_TEST_CHECKPOINTS: ${{ matrix.checkpoints }}+ run: >+ dotnet test --solution Laya.slnx -c Release+ --results-directory "$RUNNER_TEMP/trx" --report-xunit-trx --report-xunit-trx-filename parity.trx+ ${{ matrix.filter }}+ # Missing artifacts make the model-backed tests skip, and a skip is not a failure to+ # `dotnet test`. Fail the run unless every skip is for a checkpoint this job does not own.+ - name: No model-backed test was skipped+ if: ${{ !cancelled() }}+ run: >+ python laya-dotnet/tools/check_test_skips.py "$RUNNER_TEMP/trx/parity.trx"+ --have ${{ matrix.checkpoints }} --min-passed ${{ matrix.min-passed }}+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2+ if: ${{ failure() }}+ with:+ name: dotnet-parity-diagnostics-${{ matrix.name }}+ path: |+ ${{ runner.temp }}/trx+ ${{ env.GOLDEN }}diff --git a/laya-dotnet/README.md b/laya-dotnet/README.mdindex e3c4585a..9ff56336 100644--- a/laya-dotnet/README.md+++ b/laya-dotnet/README.md@@ -797,6 +797,52 @@ same ONNX graph. The vectors come from the shipping Python code, not from a secoThe same golden answers back both the fused and the split parity theories: the split graph is thefused graph cut into two ONNX files, not a different model, so one recording serves both.+### Regenerating golden fixtures / CI++The goldens are recorded from the Python package, so they have to be re-recorded whenever a change+to `laya/` moves something they capture (sequence building, option rendering, calibration,+language detection, ...). One command does the whole chain: it downloads each checkpoint at the+Hugging Face revision pinned in `tools/regen_golden.py` (`HF_REVISION`), exports the fused layout+(`tools/export_onnx.py`) and the split layout (`laya-ts/scripts/export_onnx.py`), then runs+`tools/dump_golden.py` and `tools/dump_routing_golden.py` into `tests/Laya.Tests/golden/`.++```bash+# once: CPU-only torch plus the toolchain the committed goldens were recorded with+pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu+pip install -r laya-dotnet/tools/requirements-regen.txt++# from the repository root; all three checkpoints and the routing goldens+python laya-dotnet/tools/regen_golden.py++# or one piece, with the exported artifacts somewhere that has room (1-3 GB per checkpoint)+python laya-dotnet/tools/regen_golden.py --checkpoint english --artifacts-root /path/to/artifacts+python laya-dotnet/tools/regen_golden.py --checkpoint routing # no model needed++cd laya-dotnet # global.json selects the SDK and Microsoft.Testing.Platform runner+LAYA_ONNX_ROOT=/path/to/artifacts/onnx dotnet test --solution Laya.slnx -c Release+```++Exports are reused when they are already present and were made from the same inputs, so only the+first run is slow. The tests find the split layout only at `<some ancestor of the test binary>/onnx-split`,+so with a custom `--artifacts-root` use a directory junction/symlink from the repository's+`onnx-split` to the exported `onnx-split` directory, or export at the default repository root.+A diff in `git diff -- laya-dotnet/tests/Laya.Tests/golden` after+regenerating is expected whenever the Python behavior changed on purpose; review it, and commit it+together with the C# change that follows it.++`.github/workflows/dotnet.yml` runs the same chain on every push and pull request (there is no path+filter, so a change under `laya/` triggers it): the `build` job re-records the routing goldens,+builds the whole solution with warnings as errors and runs the tests that need no model; one+`parity` job per checkpoint (and one for the router end-to-end tests, which need english and+multilingual together) exports that checkpoint, re-records its goldens from the Python code at+that commit and runs its model-backed classes against them. `LAYA_TEST_CHECKPOINTS` selects the+checkpoint theory rows (ordinary local runs still cover all checkpoints). Every job then runs+`tools/check_test_skips.py`, because `dotnet test` reports skipped tests as success: it fails the+run when any test that should have run was skipped, and tolerates only the skips for checkpoints+the job does not export. The job logs print `git diff --stat` of the regenerated goldens against+the committed ones; that is informational (the C# tests and their tolerances are the gate).+Exports are cached on the pinned revision, the exporters, `requirements-regen.txt` and+`laya/common.py`; pinned Hugging Face downloads are cached separately.The test project is an xUnit **v3** application running on Microsoft.Testing.Platform. The .NET 10SDK no longer runs these projects through VSTest. That's why `laya-dotnet` has a`global.json` that selects the `Microsoft.Testing.Platform` runner, and why the solution is passeddiff --git a/laya-dotnet/global.json b/laya-dotnet/global.jsonindex 3140116d..1d364c6a 100644--- a/laya-dotnet/global.json+++ b/laya-dotnet/global.json@@ -1,4 +1,8 @@{+ "sdk": {+ "version": "10.0.100",+ "rollForward": "latestFeature"+ },"test": {"runner": "Microsoft.Testing.Platform"}diff --git a/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs b/laya-dotnet/tests/Laya.Tests/PredictParityTests.csindex 6f2a8dd9..a68cbe48 100644--- a/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs+++ b/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs@@ -27,7 +27,7 @@ public sealed class PredictParityTests(AllEnginesFixture fixture)/// with its checkpoint. Cases for checkpoints without golden data are omitted./// </summary>public static IEnumerable<object[]> AllSuccessCases() =>- from c in Enum.GetValues<LayaCheckpoint>()+ from c in TestArtifacts.ParityCheckpointswhere GoldenData.For(c).Available// Kind is null for a standard parity case; a "shortlist" case's `result` comes from// predict_shortlist over a reduced criteria set, so it is not comparable to a plain@@ -38,7 +38,7 @@ public sealed class PredictParityTests(AllEnginesFixture fixture)/// <summary>All error cases across every checkpoint whose golden data is available.</summary>public static IEnumerable<object[]> AllErrorCases() =>- from c in Enum.GetValues<LayaCheckpoint>()+ from c in TestArtifacts.ParityCheckpointswhere GoldenData.For(c).Availablefrom info in GoldenData.For(c).Index.Where(i => i.Kind is null && i.ExpectsError)select new object[] { c, info };diff --git a/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs b/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.csindex d3ff15be..73dc6dbc 100644--- a/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs+++ b/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs@@ -42,7 +42,7 @@ public sealed class ShapeSweepTests(AllEnginesFixture fixture)/// </summary>public static IEnumerable<object[]> CheckpointShapes(){- foreach (var checkpoint in Enum.GetValues<LayaCheckpoint>())+ foreach (var checkpoint in TestArtifacts.ParityCheckpoints){// Derive max_len from the artifact config without loading the ONNX graph.// Default to 1024 (the largest value across all checkpoints) when the artifact@@ -175,7 +175,7 @@ public sealed class ShapeSweepTests(AllEnginesFixture fixture)/// </summary>public static IEnumerable<object[]> CheckpointShapesSplit(){- foreach (var checkpoint in Enum.GetValues<LayaCheckpoint>())+ foreach (var checkpoint in TestArtifacts.ParityCheckpoints){var maxLen = 1024;var dir = TestArtifacts.For(checkpoint).SplitDirectoryOrNull;diff --git a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.csindex 34fb694b..b1356ba0 100644--- a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs+++ b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs@@ -23,7 +23,7 @@ public sealed class ShortlistEndToEndTests/// <summary>Tolerance for the model's own 4-dp probabilities/confidence, as in <see cref="PredictParityTests"/>.</summary>private const double ProbTolerance = 2e-4;- public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());+ public static TheoryData<LayaCheckpoint> Checkpoints => new(TestArtifacts.ParityCheckpoints);[Theory][MemberData(nameof(Checkpoints))]diff --git a/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs b/laya-dotnet/tests/Laya.Tests/TestArtifacts.csindex a909b8e9..54b63ba1 100644--- a/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs+++ b/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs@@ -153,6 +153,22 @@ public sealed class CheckpointTestArtifacts/// </remarks>public static class TestArtifacts{+ /// <summary>Checkpoints selected for a parity job; ordinary local runs still cover all three.</summary>+ public static IEnumerable<LayaCheckpoint> ParityCheckpoints+ {+ get+ {+ var selected = Environment.GetEnvironmentVariable("LAYA_TEST_CHECKPOINTS");+ var values = Enum.GetValues<LayaCheckpoint>();+ if (string.IsNullOrWhiteSpace(selected))+ return values;+ var names = selected.Split(' ', StringSplitOptions.RemoveEmptyEntries);+ if (names.Any(name => !values.Any(c => LayaOptions.CheckpointSubfolder(c) == name)))+ throw new ArgumentException($"unknown LAYA_TEST_CHECKPOINTS: {selected}");+ return values.Where(c => names.Contains(LayaOptions.CheckpointSubfolder(c)));+ }+ }+// One CheckpointTestArtifacts per enum value; indexed by the enum's integer value.private static readonly CheckpointTestArtifacts[] ByCheckpoint =Enum.GetValues<LayaCheckpoint>().Select(c => new CheckpointTestArtifacts(c)).ToArray();diff --git a/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs b/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.csindex 545f842a..b766fd0f 100644--- a/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs+++ b/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs@@ -16,14 +16,14 @@ namespace Laya.Tests;public sealed class TokenizerParityTests{/// <summary>All checkpoints to test against, one theory case per enum value.</summary>- public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());+ public static TheoryData<LayaCheckpoint> Checkpoints => new(TestArtifacts.ParityCheckpoints);/// <summary>/// Cross-product of every checkpoint with every success case in its golden data. Cases for a/// checkpoint are omitted entirely when its golden data has not been generated yet./// </summary>public static IEnumerable<object[]> CheckpointCases() =>- from c in Enum.GetValues<LayaCheckpoint>()+ from c in TestArtifacts.ParityCheckpointswhere GoldenData.For(c).Available// Kind is null for a standard parity case; see the comment in PredictParityTests.from info in GoldenData.For(c).Index.Where(i => i.Kind is null && !i.ExpectsError)diff --git a/laya-dotnet/tools/check_test_skips.py b/laya-dotnet/tools/check_test_skips.pynew file mode 100644index 00000000..1c025498--- /dev/null+++ b/laya-dotnet/tools/check_test_skips.py@@ -0,0 +1,116 @@+"""Fail a .NET test run that only looks green because model-backed tests were skipped.++The test project skips (rather than fails) every test whose ONNX artifacts are missing, so a+run with a broken or empty artifacts directory still ends with "Passed!". In CI that would turn+the whole parity lane into a rubber stamp. This reads the TRX file `dotnet test` wrote and exits+non-zero when++ * any test failed, errored, timed out or was aborted,+ * a test was skipped for a reason that is not an expected one (see below), or+ * fewer than `--min-passed` tests passed (a wrong filter or a missing test project).++A skip is expected only when a missing-artifact reason names a checkpoint that this job deliberately has no+artifacts for (`--have` lists the ones it does have), or it is the router end-to-end skip and the+job lacks english or multilingual. Everything else, including every skip that names a checkpoint+the job *should* have, is reported and fails the run. The reason strings come from the skip+messages in tests/Laya.Tests; if one of them is reworded this fails closed, not open.++Usage:+ python laya-dotnet/tools/check_test_skips.py TestResults/run.trx --have english --min-passed 50++ # The model-free subset (model-backed classes filtered out): nothing at all may be skipped+ python laya-dotnet/tools/check_test_skips.py TestResults/run.trx --have english multilingual typed-decisions --min-passed 600+"""++import argparse+import collections+import re+import sys+import xml.etree.ElementTree as ET++NS = "{http://microsoft.com/schemas/VisualStudio/TeamTest/2010}"+FAILING = {"Failed", "Error", "Timeout", "Aborted", "Disconnected", "NotRunnable"}+CHECKPOINT_NAMES = {"english": "english", "multilingual": "multilingual", "typeddecisions": "typed-decisions"}+QUOTED = re.compile(r"'(English|Multilingual|TypedDecisions)'")+ROUTER = "router end-to-end tests need both english and multilingual"+++def reason_of(result) -> str:+ out = result.find(NS + "Output")+ return " ".join("".join(out.itertext()).split()) if out is not None else ""+++def expected_skip(reason: str, have: set) -> bool:+ named = {CHECKPOINT_NAMES[m.lower()] for m in QUOTED.findall(reason)}+ missing_artifact = any(message in reason for message in (+ "no tokenizer.json for ", "no model.onnx for ", "no ONNX artifacts for ",+ "no split ONNX artifact for ",+ ))+ if named and missing_artifact:+ return not (named & have)+ if ROUTER in reason:+ return not {"english", "multilingual"} <= have+ return False+++def main() -> int:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("trx", nargs="+", help="TRX file(s) written by `dotnet test --report-xunit-trx`.")+ ap.add_argument(+ "--have",+ nargs="*",+ default=[],+ choices=sorted(set(CHECKPOINT_NAMES.values())),+ help="Checkpoints whose artifacts this job provides; their tests must run, not skip.",+ )+ ap.add_argument("--min-passed", type=int, default=1, help="Fail below this many passed tests (default: 1).")+ args = ap.parse_args()+ have = set(args.have)++ passed = failed = 0+ allowed = collections.Counter()+ bad_skips = collections.Counter()+ failures = []+ for path in args.trx:+ for r in ET.parse(path).getroot().iter(NS + "UnitTestResult"):+ outcome = r.get("outcome")+ if outcome == "Passed":+ passed += 1+ elif outcome in FAILING:+ failed += 1+ failures.append(r.get("testName", "<unnamed>"))+ elif outcome == "NotExecuted":+ reason = reason_of(r)+ if expected_skip(reason, have):+ allowed[reason] += 1+ else:+ bad_skips[(r.get("testName", "<unnamed>").split("(")[0], reason)] += 1+ else:+ bad_skips[(r.get("testName", "<unnamed>").split("(")[0], "outcome " + str(outcome))] += 1++ n_bad = sum(bad_skips.values())+ print("passed: %d failed: %d skipped (expected): %d skipped (UNEXPECTED): %d"+ % (passed, failed, sum(allowed.values()), n_bad))+ print("checkpoints expected to run: %s" % (", ".join(sorted(have)) or "none"))+ for reason, n in allowed.most_common():+ print(" expected skip x%d: %s" % (n, reason[:140]))++ ok = True+ if failed:+ ok = False+ print("\nFAIL: %d test(s) failed, e.g. %s" % (failed, ", ".join(failures[:5])))+ if n_bad:+ ok = False+ print("\nFAIL: %d test(s) were skipped that should have run (missing or unusable artifacts?):" % n_bad)+ for (name, reason), n in bad_skips.most_common(10):+ print(" x%d %s: %s" % (n, name, reason[:200]))+ if passed < args.min_passed:+ ok = False+ print("\nFAIL: only %d test(s) passed, expected at least %d" % (passed, args.min_passed))+ if ok:+ print("\nOK: nothing that should have run was skipped")+ return 0 if ok else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/laya-dotnet/tools/export_onnx.py b/laya-dotnet/tools/export_onnx.pyindex 0ea2d2d1..0487ce93 100644--- a/laya-dotnet/tools/export_onnx.py+++ b/laya-dotnet/tools/export_onnx.py@@ -44,7 +44,6 @@ import jsonimport osimport shutilimport sys-import tempfileimport timeTOOLS = os.path.dirname(os.path.abspath(__file__))@@ -58,7 +57,7 @@ from laya.common import ( # noqa: E402build_sequence,collate_items,)-from laya.router import BUNDLE_REPO, DEFAULT_MODELS, STANDALONE_MODELS # noqa: E402+from laya.router import DEFAULT_MODELS # noqa: E402# ─── model catalogue ────────────────────────────────────────────────────────@@ -67,7 +66,7 @@ from laya.router import BUNDLE_REPO, DEFAULT_MODELS, STANDALONE_MODELS # noqa:_NAMES = ["english", "multilingual", "typed-decisions"]-def _resolve_spec(name_or_path: str, token: str):+def _resolve_spec(name_or_path: str, token: str, revision: str = None):"""Return (model_dir, subfolder, label) for a name or a local path."""if os.path.exists(name_or_path):# Local directory: treat as a self-contained checkpoint.@@ -85,6 +84,7 @@ def _resolve_spec(name_or_path: str, token: str):prefix = (subfolder + "/") if subfolder else ""kw = {"token": token or os.environ.get("HF_TOKEN"),+ "revision": revision,"allow_patterns": [prefix + n for n in ("rl_agent_config.json",@@ -416,8 +416,6 @@ def verify(out_dir: str, cfg: dict, seq_lens=None, batch_sizes=None, marker_counimport onnxruntime as ortimport torch- from laya.common import build_model as _build_model-if seq_lens is None:seq_lens = [16, 32, 64, 128, 256, cfg.get("max_len", 512)]if batch_sizes is None:@@ -540,7 +538,6 @@ def _make_synthetic_feeds(B, seq, M):def _load_pt_model_if_cached(cfg, out_dir):"""Return a loaded PyTorch model if weights are available locally, else None."""from safetensors.torch import load_file- import torch# The export was done from a snapshot_download() cache; try to locate the# safetensors file there. If it is gone (evicted or never downloaded because@@ -659,7 +656,7 @@ def check_graph_signature(out_dir: str, reference_onnx: str):# ─── main per-model export ───────────────────────────────────────────────────def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_verify: bool,- force: bool = False):+ force: bool = False, revision: str = None):"""Download (if needed), export, and verify one checkpoint.Raises RuntimeError if <out_root>/<name>/model.onnx already exists and@@ -687,7 +684,7 @@ def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_veri)# 2. Resolve model directory (may trigger a HuggingFace download).- model_dir, subfolder, label = _resolve_spec(name_or_path, token)+ model_dir, subfolder, label = _resolve_spec(name_or_path, token, revision)if subfolder:model_dir = os.path.join(model_dir, subfolder)@@ -760,8 +757,6 @@ def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_veridef _check_against_reference(out_dir: str, label: str):"""If a pre-built reference exists in onnx/, compare signatures."""- import onnx-ref_candidates = {"english": os.path.join(REPO, "onnx", "english", "model.onnx"),"multilingual": os.path.join(REPO, "onnx", "multilingual", "model.onnx"),@@ -816,6 +811,13 @@ def main():help="After export, reload in ONNX Runtime and compare to PyTorch ""across a range of tensor shapes.",)+ ap.add_argument(+ "--revision",+ default=None,+ metavar="SHA",+ help="Hugging Face commit (or branch/tag) to download the checkpoint from "+ "(default: the repo's main branch).",+ )ap.add_argument("--force",action="store_true",@@ -850,6 +852,7 @@ def main():opset=args.opset,do_verify=args.verify,force=args.force,+ revision=args.revision,)out_dirs.append((target, od))# Signature check against the pre-built reference (if any).diff --git a/laya-dotnet/tools/regen_golden.py b/laya-dotnet/tools/regen_golden.pynew file mode 100644index 00000000..fa84f7e7--- /dev/null+++ b/laya-dotnet/tools/regen_golden.py@@ -0,0 +1,263 @@+"""Regenerate the .NET golden fixtures from the shipping Python code, end to end.++This is the single entry point the `.NET` CI lane runs (`.github/workflows/dotnet.yml`) and the+one command a contributor needs after changing anything in `laya/` that the goldens capture+(sequence building, option rendering, calibration, language detection, ...). It only chains the+existing tools; none of their logic is repeated here:++ 1. downloads the checkpoint at a *pinned* Hugging Face revision (`HF_REVISION` below),+ 2. exports the fused layout -> <artifacts-root>/onnx/<checkpoint>/ (tools/export_onnx.py)+ 3. exports the split layout -> <artifacts-root>/onnx-split/<checkpoint>/ (laya-ts/scripts/export_onnx.py)+ 4. records tests/Laya.Tests/golden/<checkpoint>/ from the real `laya.Agent` (tools/dump_golden.py)+ 5. records tests/Laya.Tests/golden/routing/ (tools/dump_routing_golden.py)++Exports are the slow part, so an artifact directory that is already present, complete and was+produced from the same inputs (revision + exporter scripts + `laya/common.py`, which holds the+model definition) is reused. That is what lets CI cache `<artifacts-root>/onnx*` between runs.+The goldens themselves are always rewritten.++The default artifacts root is the repository root, which is where the test project looks for+`onnx/<checkpoint>` and `onnx-split/<checkpoint>` on its own (it walks up from the test binary).+Point `--artifacts-root` elsewhere and set `LAYA_ONNX_ROOT=<root>/onnx` for the fused layout; the+split layout has no environment variable, so it is only found at `<some ancestor>/onnx-split`.++Usage:+ # Everything: three checkpoints, split layouts and the routing goldens+ python laya-dotnet/tools/regen_golden.py++ # One checkpoint, artifacts somewhere with room (each checkpoint is ~1-3 GB, fused + split)+ python laya-dotnet/tools/regen_golden.py --checkpoint english --artifacts-root D:/laya-artifacts++ # Only the model-free routing goldens+ python laya-dotnet/tools/regen_golden.py --checkpoint routing++ # Re-export even when a valid artifact directory exists+ python laya-dotnet/tools/regen_golden.py --checkpoint multilingual --force-export++Needs the packages in `tools/requirements-regen.txt` plus a CPU build of torch. Exits non-zero+as soon as any step fails.+"""++import argparse+import hashlib+import json+import os+import shutil+import subprocess+import sys+import time++from google.protobuf.message import DecodeError++TOOLS = os.path.dirname(os.path.abspath(__file__))+REPO = os.path.dirname(os.path.dirname(TOOLS))+sys.path.insert(0, REPO)++# The Hugging Face commit every checkpoint is exported from. The weights, encoder configs and+# tokenizers are byte-identical from 458d7563 (the commit that bundled the typed-decisions+# checkpoint) through this one; later commits only touch the model card. Bump it deliberately,+# then verify the regenerated goldens against the SDK.+HF_REPO = "convaiinnovations/laya"+HF_REVISION = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"++CHECKPOINTS = ["english", "multilingual", "typed-decisions"]++GOLDEN_ROOT = os.path.join(REPO, "laya-dotnet", "tests", "Laya.Tests", "golden")+EXPORT_FUSED = os.path.join(TOOLS, "export_onnx.py")+EXPORT_SPLIT = os.path.join(REPO, "laya-ts", "scripts", "export_onnx.py")+DUMP_GOLDEN = os.path.join(TOOLS, "dump_golden.py")+DUMP_ROUTING = os.path.join(TOOLS, "dump_routing_golden.py")++# Everything that decides what an export looks like. Its hash is stored beside each exported+# directory; a mismatch (or a missing stamp) means the export is stale and is redone.+EXPORT_INPUTS = [+ EXPORT_FUSED,+ EXPORT_SPLIT,+ os.path.join(TOOLS, "requirements-regen.txt"),+ os.path.join(REPO, "laya", "common.py"),+]++STAMP = ".regen-stamp.json"+FUSED_FILES = [+ "model.onnx",+ "model.onnx.data",+ "rl_agent_config.json",+ os.path.join("tokenizer", "tokenizer.json"),+ "fixtures.npz",+]+SPLIT_FILES = ["encoder.onnx", "head.onnx", "rl_agent_config.json", "tokenizer.json"]+++def inputs_hash(revision: str) -> str:+ h = hashlib.sha256(revision.encode("utf-8"))+ for path in EXPORT_INPUTS:+ with open(path, "rb") as f:+ # Normalise line endings so a CRLF checkout hashes like the LF one.+ h.update(f.read().replace(b"\r\n", b"\n"))+ return h.hexdigest()+++def is_valid(path: str, files: list, stamp: str) -> bool:+ """The directory holds a complete export made from exactly these inputs."""+ try:+ if not all(os.path.getsize(os.path.join(path, f)) > 0 for f in files):+ return False+ with open(os.path.join(path, STAMP), encoding="utf-8") as f:+ if json.load(f).get("inputs_sha256") != stamp:+ return False+ import onnx++ for file in files:+ if not file.endswith(".onnx"):+ continue+ model = onnx.load(os.path.join(path, file), load_external_data=False)+ for tensor in model.graph.initializer:+ if tensor.data_location != onnx.TensorProto.EXTERNAL:+ continue+ data = {entry.key: entry.value for entry in tensor.external_data}+ sidecar = os.path.join(path, data["location"])+ size = os.path.getsize(sidecar)+ if size <= 0 or size < int(data.get("offset", 0)) + int(data.get("length", 0)):+ return False+ return True+ except (OSError, ValueError, KeyError, DecodeError):+ return False+++def run(cmd: list, env: dict = None):+ print("\n$ " + " ".join(cmd), flush=True)+ t0 = time.perf_counter()+ subprocess.run(cmd, check=True, cwd=REPO, env=env)+ print(" (%.1f s)" % (time.perf_counter() - t0), flush=True)+++def snapshot(name: str, revision: str) -> str:+ """The pinned checkpoint directory in the Hugging Face cache (downloaded on first use)."""+ from huggingface_hub import snapshot_download++ from laya.router import DEFAULT_MODELS++ repo, subfolder = DEFAULT_MODELS[name]+ if repo != HF_REPO:+ raise SystemExit(+ "%s now lives in %r, but the pinned revision belongs to %r; update HF_REPO/HF_REVISION"+ % (name, repo, HF_REPO)+ )+ prefix = (subfolder + "/") if subfolder else ""+ # A superset of what the two exporters ask for, so each of them hits the cache.+ allow = [prefix + n for n in (+ "rl_agent_config.json", "model.safetensors", "config.json",+ "tokenizer.json", "tokenizer/*", "encoder/*",+ )]+ root = snapshot_download(repo, revision=revision, allow_patterns=allow)+ return os.path.join(root, subfolder) if subfolder else root+++def publish(partial: str, final: str, stamp: str):+ """Stamp a finished export and move it into place, so an interrupted run leaves nothing half-valid."""+ with open(os.path.join(partial, STAMP), "w", encoding="utf-8") as f:+ json.dump({"inputs_sha256": stamp}, f)+ if os.path.exists(final):+ shutil.rmtree(final)+ os.makedirs(os.path.dirname(final), exist_ok=True)+ os.replace(partial, final)+++def export_fused(name: str, root: str, revision: str, stamp: str, env: dict):+ final = os.path.join(root, "onnx", name)+ staging = os.path.join(root, ".partial", "onnx")+ shutil.rmtree(staging, ignore_errors=True)+ run([sys.executable, EXPORT_FUSED, "--model", name, "--out", staging, "--revision", revision], env=env)+ publish(os.path.join(staging, name), final, stamp)+++def export_split(name: str, root: str, model_dir: str, stamp: str, env: dict):+ final = os.path.join(root, "onnx-split", name)+ staging = os.path.join(root, ".partial", "onnx-split", name)+ shutil.rmtree(staging, ignore_errors=True)+ run([sys.executable, EXPORT_SPLIT, "--model-dir", model_dir, "--out-dir", staging], env=env)+ publish(staging, final, stamp)+++def regen_checkpoint(name: str, root: str, revision: str, force: bool):+ print("\n=== %s ===" % name)+ stamp = inputs_hash(revision)+ fused = os.path.join(root, "onnx", name)+ split = os.path.join(root, "onnx-split", name)+ need_fused = force or not is_valid(fused, FUSED_FILES, stamp)+ need_split = force or not is_valid(split, SPLIT_FILES, stamp)++ if need_fused or need_split:+ model_dir = snapshot(name, revision)+ # Everything the exporters need is in the cache now; going back to the network would+ # only be a way to pick up something that is not the pinned revision.+ env = dict(os.environ, HF_HUB_OFFLINE="1", PYTHONPATH=REPO + os.pathsep + os.environ.get("PYTHONPATH", ""))+ if need_fused:+ export_fused(name, root, revision, stamp, env)+ if need_split:+ export_split(name, root, model_dir, stamp, env)+ else:+ print("fused and split exports are up to date; reusing %s and %s" % (fused, split))++ run([sys.executable, DUMP_GOLDEN, "--checkpoint", name, "--onnx", fused])+++def main() -> int:+ ap = argparse.ArgumentParser(+ description=__doc__,+ formatter_class=argparse.RawDescriptionHelpFormatter,+ )+ ap.add_argument(+ "--checkpoint",+ action="append",+ choices=CHECKPOINTS + ["routing", "all"],+ metavar="{%s,routing,all}" % ",".join(CHECKPOINTS),+ help="What to regenerate; repeat to combine. 'routing' is the model-free routing/language/"+ "email/shortlist goldens, 'all' is every checkpoint plus routing (default: all).",+ )+ ap.add_argument(+ "--artifacts-root",+ default=REPO,+ help="Where onnx/<checkpoint> and onnx-split/<checkpoint> are written and reused "+ "(default: the repository root, which the tests find on their own).",+ )+ ap.add_argument(+ "--revision",+ default=HF_REVISION,+ metavar="SHA",+ help="Hugging Face commit to export from (default: %(default)s).",+ )+ ap.add_argument(+ "--force-export",+ action="store_true",+ help="Re-export even when a valid artifact directory already exists.",+ )+ args = ap.parse_args()++ # The exporters print check marks; a Windows console code page cannot encode them.+ os.environ.setdefault("PYTHONUTF8", "1")+ selected = args.checkpoint or ["all"]+ if "all" in selected:+ selected = CHECKPOINTS + ["routing"]+ checkpoints = [c for c in CHECKPOINTS if c in selected]+ root = os.path.abspath(args.artifacts_root)++ t0 = time.perf_counter()+ try:+ for name in checkpoints:+ regen_checkpoint(name, root, args.revision, args.force_export)+ if "routing" in selected:+ print("\n=== routing ===")+ run([sys.executable, DUMP_ROUTING, "--force"])+ except subprocess.CalledProcessError as e:+ print("\nFAILED (exit %d): %s" % (e.returncode, " ".join(e.cmd)), file=sys.stderr)+ return e.returncode or 1+ finally:+ shutil.rmtree(os.path.join(root, ".partial"), ignore_errors=True)++ print("\nregenerated %s in %.0f s" % (", ".join(checkpoints + (["routing"] if "routing" in selected else [])), time.perf_counter() - t0))+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/laya-dotnet/tools/requirements-regen.txt b/laya-dotnet/tools/requirements-regen.txtnew file mode 100644index 00000000..28e3a815--- /dev/null+++ b/laya-dotnet/tools/requirements-regen.txt@@ -0,0 +1,15 @@+# Python toolchain for laya-dotnet/tools/regen_golden.py (and the .NET parity CI lane).+#+# Pinned to the versions the committed goldens were recorded with (see the+# "*_version" fields in tests/Laya.Tests/golden/<checkpoint>/meta.json) so a toolchain+# release cannot move the goldens. Bump deliberately, together with a regeneration.+#+# torch is not listed: it needs the CPU-only index, so install it first with+# pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu+transformers==5.17.0+safetensors==0.8.0+huggingface_hub==1.33.0+numpy==2.5.3+onnx==1.23.0+onnxruntime==1.30.0+onnxscript==0.7.2diff --git a/laya-dotnet/tools/test_ci_tools.py b/laya-dotnet/tools/test_ci_tools.pynew file mode 100644index 00000000..548b9506--- /dev/null+++ b/laya-dotnet/tools/test_ci_tools.py@@ -0,0 +1,101 @@+"""Model-free regressions for the parity lane: python laya-dotnet/tools/test_ci_tools.py."""++import contextlib+import io+import types+import unittest+import xml.etree.ElementTree as ET+from unittest.mock import mock_open, patch++from google.protobuf.message import DecodeError++import check_test_skips as guard+import regen_golden as regen+++class SkipGuardTests(unittest.TestCase):+ def run_guard(self, outcomes, have=("english",), minimum=1):+ root = ET.Element(guard.NS + "TestRun")+ for outcome, reason in outcomes:+ result = ET.SubElement(root, guard.NS + "UnitTestResult", outcome=outcome, testName="parity")+ ET.SubElement(ET.SubElement(result, guard.NS + "Output"), guard.NS + "StdOut").text = reason+ argv = ["check_test_skips.py", "run.trx", "--min-passed", str(minimum), "--have", *have]+ with patch("sys.argv", argv), patch.object(guard.ET, "parse", return_value=ET.ElementTree(root)):+ with contextlib.redirect_stdout(io.StringIO()):+ return guard.main()++ def test_passed_run(self):+ self.assertEqual(0, self.run_guard([("Passed", "")]))++ def test_absent_checkpoint_is_allowed(self):+ self.assertEqual(0, self.run_guard([+ ("Passed", ""), ("NotExecuted", "no ONNX artifacts for 'Multilingual'; set LAYA_ONNX_ROOT"),+ ]))++ def test_owned_checkpoint_is_not_allowed(self):+ self.assertEqual(1, self.run_guard([+ ("Passed", ""), ("NotExecuted", "no split ONNX artifact for 'English'; export it"),+ ]))++ def test_missing_goldens_are_not_allowed(self):+ self.assertEqual(1, self.run_guard([+ ("Passed", ""), ("NotExecuted", "no golden data for 'Multilingual'; regenerate"),+ ]))++ def test_unknown_skip_fails_closed(self):+ self.assertEqual(1, self.run_guard([("Passed", ""), ("NotExecuted", "unexpected reason")]))++ def test_failure_and_unknown_outcome_fail(self):+ for outcome in [*guard.FAILING, "Unknown"]:+ with self.subTest(outcome=outcome):+ self.assertEqual(1, self.run_guard([("Passed", ""), (outcome, "")]))++ def test_empty_or_wrong_filter_fails(self):+ self.assertEqual(1, self.run_guard([]))+ self.assertEqual(1, self.run_guard([("Passed", "")], minimum=2))++ def test_router_requires_both_checkpoints(self):+ self.assertTrue(guard.expected_skip(guard.ROUTER, {"english"}))+ self.assertFalse(guard.expected_skip(guard.ROUTER, {"english", "multilingual"}))+++class ExportCacheTests(unittest.TestCase):+ def validate(self, stamp="expected", sidecar_size=100, length=100, broken_graph=False):+ import onnx++ tensor = onnx.TensorProto(data_location=onnx.TensorProto.EXTERNAL)+ for key, value in [("location", "encoder.onnx.data"), ("length", str(length))]:+ entry = tensor.external_data.add()+ entry.key, entry.value = key, value+ model = types.SimpleNamespace(graph=types.SimpleNamespace(initializer=[tensor]))++ def size(path):+ if path.endswith(".data"):+ if sidecar_size is None:+ raise FileNotFoundError(path)+ return sidecar_size+ return 100++ with patch("builtins.open", mock_open(read_data='{"inputs_sha256": "%s"}' % stamp)):+ with patch.object(regen.os.path, "getsize", side_effect=size):+ with patch.object(onnx, "load", return_value=model,+ side_effect=DecodeError("broken") if broken_graph else None):+ return regen.is_valid("artifacts", ["encoder.onnx"], "expected")++ def test_complete_cache_is_reused(self):+ self.assertTrue(self.validate())++ def test_stale_stamp_is_rejected(self):+ self.assertFalse(self.validate(stamp="old"))++ def test_missing_empty_or_truncated_sidecar_is_rejected(self):+ for size in [None, 0, 99]:+ with self.subTest(size=size):+ self.assertFalse(self.validate(sidecar_size=size))++ def test_malformed_graph_is_rejected(self):+ self.assertFalse(self.validate(broken_graph=True))+++if __name__ == "__main__":+ unittest.main()