Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

ci(dotnet): regenerate goldens from Python and run parity tests on every push

kevin-gatimu8ccaf969f2cf
diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml
new file mode 100644
index 00000000..0312bd55
--- /dev/null
+++ b/.github/workflows/dotnet.yml
@@ -0,0 +1,201 @@
+name: .NET
+
+# Parity lane for laya-dotnet. The golden fixtures under laya-dotnet/tests/Laya.Tests/golden are
+# recorded from the Python `laya` package, so this runs on EVERY push and pull request (no path
+# filter): a change to laya/ that moves what the goldens capture must show up here as a red
+# check, not as silent drift. Each run re-records the goldens from the Python code at that commit
+# (tools/regen_golden.py) and runs the C# tests against the fresh copies.
+
+on:
+ pull_request:
+ push:
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+concurrency:
+ group: dotnet-${{ github.event_name == 'pull_request' && github.ref || github.sha }}
+ # Cancel outdated pull request runs. Pushes get a per-commit group so each one finishes.
+ cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+ # transformers probes for TensorFlow at import; when TF is present its abseil runtime can
+ # deadlock model construction. Laya is torch-only.
+ USE_TF: "0"
+ USE_TORCH: "1"
+ TOKENIZERS_PARALLELISM: "false"
+ DOTNET_NOLOGO: "true"
+ DOTNET_CLI_TELEMETRY_OPTOUT: "true"
+ GOLDEN: laya-dotnet/tests/Laya.Tests/golden
+
+jobs:
+ build:
+ name: build + model-free tests
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt
+ - name: Install
+ run: |
+ python -m pip install --upgrade pip
+ # CPU-only torch; the routing goldens only import laya, they never run a model
+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+ pip install -r laya-dotnet/tools/requirements-regen.txt
+ - name: Test regeneration cache and skip guard
+ run: python laya-dotnet/tools/test_ci_tools.py
+ - name: Regenerate the routing goldens from Python
+ run: python laya-dotnet/tools/regen_golden.py --checkpoint routing
+ - name: Golden drift vs. committed (informational)
+ run: |
+ git --no-pager diff --stat -- "$GOLDEN"
+ git status --short -- "$GOLDEN"
+ - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+ with:
+ global-json-file: laya-dotnet/global.json
+ - name: Build (library, samples and tests, warnings are errors)
+ working-directory: laya-dotnet
+ run: dotnet build Laya.slnx -c Release
+ - name: Tests that need no model
+ working-directory: laya-dotnet
+ run: >
+ dotnet test --solution Laya.slnx -c Release --no-build
+ --results-directory "$RUNNER_TEMP/trx" --report-xunit-trx --report-xunit-trx-filename build.trx
+ --filter-not-class Laya.Tests.PredictParityTests
+ --filter-not-class Laya.Tests.ShapeSweepTests
+ --filter-not-class Laya.Tests.TokenizerParityTests
+ --filter-not-class Laya.Tests.RouterEndToEndTests
+ --filter-not-class Laya.Tests.ShortlistEndToEndTests
+ # With the model-backed classes filtered out nothing may be skipped, so every checkpoint
+ # is listed as expected to run.
+ - name: No test was skipped
+ if: ${{ !cancelled() }}
+ run: >
+ python laya-dotnet/tools/check_test_skips.py "$RUNNER_TEMP/trx/build.trx"
+ --have english multilingual typed-decisions --min-passed 600
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ if: ${{ failure() }}
+ with:
+ name: dotnet-build-diagnostics
+ path: |
+ ${{ runner.temp }}/trx
+ ${{ env.GOLDEN }}/routing
+
+ parity:
+ name: parity (${{ matrix.name }})
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - name: english
+ checkpoints: english
+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests
+ min-passed: 50
+ - name: multilingual
+ checkpoints: multilingual
+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests
+ min-passed: 50
+ - name: typed-decisions
+ checkpoints: typed-decisions
+ filter: --filter-class Laya.Tests.PredictParityTests Laya.Tests.TokenizerParityTests Laya.Tests.ShapeSweepTests Laya.Tests.ShortlistEndToEndTests
+ min-passed: 50
+ # The router end-to-end tests load english and multilingual side by side.
+ - name: router
+ checkpoints: english multilingual
+ filter: --filter-class Laya.Tests.RouterEndToEndTests
+ min-passed: 1
+ steps:
+ - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: laya-dotnet/tools/requirements-regen.txt
+ - name: Install
+ run: |
+ python -m pip install --upgrade pip
+ pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+ pip install -r laya-dotnet/tools/requirements-regen.txt
+
+ # The exports are the slow part (download, trace, verify). They depend only on the pinned
+ # Hugging Face revision (HF_REVISION in regen_golden.py), the two exporters, the pinned
+ # toolchain and the model definition in laya/common.py, so that is the cache key; the
+ # script re-checks the same inputs itself before it reuses a directory. The goldens are
+ # never cached: they are always re-recorded.
+ - name: Cache pinned Hugging Face downloads
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: ~/.cache/huggingface/hub
+ key: laya-hf-${{ runner.os }}-${{ matrix.name }}-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py') }}
+ - name: Cache exported english artifacts
+ if: ${{ contains(matrix.checkpoints, 'english') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/english
+ onnx-split/english
+ key: laya-onnx-${{ runner.os }}-english-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}
+ - name: Cache exported multilingual artifacts
+ if: ${{ contains(matrix.checkpoints, 'multilingual') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/multilingual
+ onnx-split/multilingual
+ key: laya-onnx-${{ runner.os }}-multilingual-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}
+ - name: Cache exported typed-decisions artifacts
+ if: ${{ contains(matrix.checkpoints, 'typed-decisions') }}
+ uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+ with:
+ path: |
+ onnx/typed-decisions
+ onnx-split/typed-decisions
+ key: laya-onnx-${{ runner.os }}-typed-decisions-${{ hashFiles('laya-dotnet/tools/regen_golden.py', 'laya-dotnet/tools/export_onnx.py', 'laya-ts/scripts/export_onnx.py', 'laya-dotnet/tools/requirements-regen.txt', 'laya/common.py') }}
+
+ - name: Export the checkpoints and regenerate their goldens from Python
+ run: |
+ args=()
+ for c in ${{ matrix.checkpoints }}; do args+=(--checkpoint "$c"); done
+ python laya-dotnet/tools/regen_golden.py "${args[@]}"
+ # Informational only: float noise across platforms is expected, the C# tests and their
+ # tolerances are the gate.
+ - name: Golden drift vs. committed (informational)
+ run: |
+ git --no-pager diff --stat -- "$GOLDEN"
+ git status --short -- "$GOLDEN"
+
+ - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0
+ with:
+ global-json-file: laya-dotnet/global.json
+ # The fused layout is found through LAYA_ONNX_ROOT; the split layout through the test
+ # project's walk-up to <repo>/onnx-split, which is where the script wrote it.
+ - name: Run the tests against the fresh goldens
+ working-directory: laya-dotnet
+ env:
+ LAYA_ONNX_ROOT: ${{ github.workspace }}/onnx
+ LAYA_TEST_CHECKPOINTS: ${{ matrix.checkpoints }}
+ run: >
+ dotnet test --solution Laya.slnx -c Release
+ --results-directory "$RUNNER_TEMP/trx" --report-xunit-trx --report-xunit-trx-filename parity.trx
+ ${{ matrix.filter }}
+ # Missing artifacts make the model-backed tests skip, and a skip is not a failure to
+ # `dotnet test`. Fail the run unless every skip is for a checkpoint this job does not own.
+ - name: No model-backed test was skipped
+ if: ${{ !cancelled() }}
+ run: >
+ python laya-dotnet/tools/check_test_skips.py "$RUNNER_TEMP/trx/parity.trx"
+ --have ${{ matrix.checkpoints }} --min-passed ${{ matrix.min-passed }}
+ - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+ if: ${{ failure() }}
+ with:
+ name: dotnet-parity-diagnostics-${{ matrix.name }}
+ path: |
+ ${{ runner.temp }}/trx
+ ${{ env.GOLDEN }}
diff --git a/laya-dotnet/README.md b/laya-dotnet/README.md
index e3c4585a..9ff56336 100644
--- a/laya-dotnet/README.md
+++ b/laya-dotnet/README.md
@@ -797,6 +797,52 @@ same ONNX graph. The vectors come from the shipping Python code, not from a seco
The same golden answers back both the fused and the split parity theories: the split graph is the
fused graph cut into two ONNX files, not a different model, so one recording serves both.
+### Regenerating golden fixtures / CI
+
+The goldens are recorded from the Python package, so they have to be re-recorded whenever a change
+to `laya/` moves something they capture (sequence building, option rendering, calibration,
+language detection, ...). One command does the whole chain: it downloads each checkpoint at the
+Hugging Face revision pinned in `tools/regen_golden.py` (`HF_REVISION`), exports the fused layout
+(`tools/export_onnx.py`) and the split layout (`laya-ts/scripts/export_onnx.py`), then runs
+`tools/dump_golden.py` and `tools/dump_routing_golden.py` into `tests/Laya.Tests/golden/`.
+
+```bash
+# once: CPU-only torch plus the toolchain the committed goldens were recorded with
+pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+pip install -r laya-dotnet/tools/requirements-regen.txt
+
+# from the repository root; all three checkpoints and the routing goldens
+python laya-dotnet/tools/regen_golden.py
+
+# or one piece, with the exported artifacts somewhere that has room (1-3 GB per checkpoint)
+python laya-dotnet/tools/regen_golden.py --checkpoint english --artifacts-root /path/to/artifacts
+python laya-dotnet/tools/regen_golden.py --checkpoint routing # no model needed
+
+cd laya-dotnet # global.json selects the SDK and Microsoft.Testing.Platform runner
+LAYA_ONNX_ROOT=/path/to/artifacts/onnx dotnet test --solution Laya.slnx -c Release
+```
+
+Exports are reused when they are already present and were made from the same inputs, so only the
+first run is slow. The tests find the split layout only at `<some ancestor of the test binary>/onnx-split`,
+so with a custom `--artifacts-root` use a directory junction/symlink from the repository's
+`onnx-split` to the exported `onnx-split` directory, or export at the default repository root.
+A diff in `git diff -- laya-dotnet/tests/Laya.Tests/golden` after
+regenerating is expected whenever the Python behavior changed on purpose; review it, and commit it
+together with the C# change that follows it.
+
+`.github/workflows/dotnet.yml` runs the same chain on every push and pull request (there is no path
+filter, so a change under `laya/` triggers it): the `build` job re-records the routing goldens,
+builds the whole solution with warnings as errors and runs the tests that need no model; one
+`parity` job per checkpoint (and one for the router end-to-end tests, which need english and
+multilingual together) exports that checkpoint, re-records its goldens from the Python code at
+that commit and runs its model-backed classes against them. `LAYA_TEST_CHECKPOINTS` selects the
+checkpoint theory rows (ordinary local runs still cover all checkpoints). Every job then runs
+`tools/check_test_skips.py`, because `dotnet test` reports skipped tests as success: it fails the
+run when any test that should have run was skipped, and tolerates only the skips for checkpoints
+the job does not export. The job logs print `git diff --stat` of the regenerated goldens against
+the committed ones; that is informational (the C# tests and their tolerances are the gate).
+Exports are cached on the pinned revision, the exporters, `requirements-regen.txt` and
+`laya/common.py`; pinned Hugging Face downloads are cached separately.
The test project is an xUnit **v3** application running on Microsoft.Testing.Platform. The .NET 10
SDK no longer runs these projects through VSTest. That's why `laya-dotnet` has a
`global.json` that selects the `Microsoft.Testing.Platform` runner, and why the solution is passed
diff --git a/laya-dotnet/global.json b/laya-dotnet/global.json
index 3140116d..1d364c6a 100644
--- a/laya-dotnet/global.json
+++ b/laya-dotnet/global.json
@@ -1,4 +1,8 @@
{
+ "sdk": {
+ "version": "10.0.100",
+ "rollForward": "latestFeature"
+ },
"test": {
"runner": "Microsoft.Testing.Platform"
}
diff --git a/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs b/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs
index 6f2a8dd9..a68cbe48 100644
--- a/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/PredictParityTests.cs
@@ -27,7 +27,7 @@ public sealed class PredictParityTests(AllEnginesFixture fixture)
/// with its checkpoint. Cases for checkpoints without golden data are omitted.
/// </summary>
public static IEnumerable<object[]> AllSuccessCases() =>
- from c in Enum.GetValues<LayaCheckpoint>()
+ from c in TestArtifacts.ParityCheckpoints
where GoldenData.For(c).Available
// Kind is null for a standard parity case; a "shortlist" case's `result` comes from
// predict_shortlist over a reduced criteria set, so it is not comparable to a plain
@@ -38,7 +38,7 @@ public sealed class PredictParityTests(AllEnginesFixture fixture)
/// <summary>All error cases across every checkpoint whose golden data is available.</summary>
public static IEnumerable<object[]> AllErrorCases() =>
- from c in Enum.GetValues<LayaCheckpoint>()
+ from c in TestArtifacts.ParityCheckpoints
where GoldenData.For(c).Available
from info in GoldenData.For(c).Index.Where(i => i.Kind is null && i.ExpectsError)
select new object[] { c, info };
diff --git a/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs b/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs
index d3ff15be..73dc6dbc 100644
--- a/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/ShapeSweepTests.cs
@@ -42,7 +42,7 @@ public sealed class ShapeSweepTests(AllEnginesFixture fixture)
/// </summary>
public static IEnumerable<object[]> CheckpointShapes()
{
- foreach (var checkpoint in Enum.GetValues<LayaCheckpoint>())
+ foreach (var checkpoint in TestArtifacts.ParityCheckpoints)
{
// Derive max_len from the artifact config without loading the ONNX graph.
// Default to 1024 (the largest value across all checkpoints) when the artifact
@@ -175,7 +175,7 @@ public sealed class ShapeSweepTests(AllEnginesFixture fixture)
/// </summary>
public static IEnumerable<object[]> CheckpointShapesSplit()
{
- foreach (var checkpoint in Enum.GetValues<LayaCheckpoint>())
+ foreach (var checkpoint in TestArtifacts.ParityCheckpoints)
{
var maxLen = 1024;
var dir = TestArtifacts.For(checkpoint).SplitDirectoryOrNull;
diff --git a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs
index 34fb694b..b1356ba0 100644
--- a/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/ShortlistEndToEndTests.cs
@@ -23,7 +23,7 @@ public sealed class ShortlistEndToEndTests
/// <summary>Tolerance for the model's own 4-dp probabilities/confidence, as in <see cref="PredictParityTests"/>.</summary>
private const double ProbTolerance = 2e-4;
- public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());
+ public static TheoryData<LayaCheckpoint> Checkpoints => new(TestArtifacts.ParityCheckpoints);
[Theory]
[MemberData(nameof(Checkpoints))]
diff --git a/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs b/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs
index a909b8e9..54b63ba1 100644
--- a/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs
+++ b/laya-dotnet/tests/Laya.Tests/TestArtifacts.cs
@@ -153,6 +153,22 @@ public sealed class CheckpointTestArtifacts
/// </remarks>
public static class TestArtifacts
{
+ /// <summary>Checkpoints selected for a parity job; ordinary local runs still cover all three.</summary>
+ public static IEnumerable<LayaCheckpoint> ParityCheckpoints
+ {
+ get
+ {
+ var selected = Environment.GetEnvironmentVariable("LAYA_TEST_CHECKPOINTS");
+ var values = Enum.GetValues<LayaCheckpoint>();
+ if (string.IsNullOrWhiteSpace(selected))
+ return values;
+ var names = selected.Split(' ', StringSplitOptions.RemoveEmptyEntries);
+ if (names.Any(name => !values.Any(c => LayaOptions.CheckpointSubfolder(c) == name)))
+ throw new ArgumentException($"unknown LAYA_TEST_CHECKPOINTS: {selected}");
+ return values.Where(c => names.Contains(LayaOptions.CheckpointSubfolder(c)));
+ }
+ }
+
// One CheckpointTestArtifacts per enum value; indexed by the enum's integer value.
private static readonly CheckpointTestArtifacts[] ByCheckpoint =
Enum.GetValues<LayaCheckpoint>().Select(c => new CheckpointTestArtifacts(c)).ToArray();
diff --git a/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs b/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs
index 545f842a..b766fd0f 100644
--- a/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs
+++ b/laya-dotnet/tests/Laya.Tests/TokenizerParityTests.cs
@@ -16,14 +16,14 @@ namespace Laya.Tests;
public sealed class TokenizerParityTests
{
/// <summary>All checkpoints to test against, one theory case per enum value.</summary>
- public static TheoryData<LayaCheckpoint> Checkpoints => new(Enum.GetValues<LayaCheckpoint>());
+ public static TheoryData<LayaCheckpoint> Checkpoints => new(TestArtifacts.ParityCheckpoints);
/// <summary>
/// Cross-product of every checkpoint with every success case in its golden data. Cases for a
/// checkpoint are omitted entirely when its golden data has not been generated yet.
/// </summary>
public static IEnumerable<object[]> CheckpointCases() =>
- from c in Enum.GetValues<LayaCheckpoint>()
+ from c in TestArtifacts.ParityCheckpoints
where GoldenData.For(c).Available
// Kind is null for a standard parity case; see the comment in PredictParityTests.
from info in GoldenData.For(c).Index.Where(i => i.Kind is null && !i.ExpectsError)
diff --git a/laya-dotnet/tools/check_test_skips.py b/laya-dotnet/tools/check_test_skips.py
new file mode 100644
index 00000000..1c025498
--- /dev/null
+++ b/laya-dotnet/tools/check_test_skips.py
@@ -0,0 +1,116 @@
+"""Fail a .NET test run that only looks green because model-backed tests were skipped.
+
+The test project skips (rather than fails) every test whose ONNX artifacts are missing, so a
+run with a broken or empty artifacts directory still ends with "Passed!". In CI that would turn
+the whole parity lane into a rubber stamp. This reads the TRX file `dotnet test` wrote and exits
+non-zero when
+
+ * any test failed, errored, timed out or was aborted,
+ * a test was skipped for a reason that is not an expected one (see below), or
+ * fewer than `--min-passed` tests passed (a wrong filter or a missing test project).
+
+A skip is expected only when a missing-artifact reason names a checkpoint that this job deliberately has no
+artifacts for (`--have` lists the ones it does have), or it is the router end-to-end skip and the
+job lacks english or multilingual. Everything else, including every skip that names a checkpoint
+the job *should* have, is reported and fails the run. The reason strings come from the skip
+messages in tests/Laya.Tests; if one of them is reworded this fails closed, not open.
+
+Usage:
+ python laya-dotnet/tools/check_test_skips.py TestResults/run.trx --have english --min-passed 50
+
+ # The model-free subset (model-backed classes filtered out): nothing at all may be skipped
+ python laya-dotnet/tools/check_test_skips.py TestResults/run.trx --have english multilingual typed-decisions --min-passed 600
+"""
+
+import argparse
+import collections
+import re
+import sys
+import xml.etree.ElementTree as ET
+
+NS = "{http://microsoft.com/schemas/VisualStudio/TeamTest/2010}"
+FAILING = {"Failed", "Error", "Timeout", "Aborted", "Disconnected", "NotRunnable"}
+CHECKPOINT_NAMES = {"english": "english", "multilingual": "multilingual", "typeddecisions": "typed-decisions"}
+QUOTED = re.compile(r"'(English|Multilingual|TypedDecisions)'")
+ROUTER = "router end-to-end tests need both english and multilingual"
+
+
+def reason_of(result) -> str:
+ out = result.find(NS + "Output")
+ return " ".join("".join(out.itertext()).split()) if out is not None else ""
+
+
+def expected_skip(reason: str, have: set) -> bool:
+ named = {CHECKPOINT_NAMES[m.lower()] for m in QUOTED.findall(reason)}
+ missing_artifact = any(message in reason for message in (
+ "no tokenizer.json for ", "no model.onnx for ", "no ONNX artifacts for ",
+ "no split ONNX artifact for ",
+ ))
+ if named and missing_artifact:
+ return not (named & have)
+ if ROUTER in reason:
+ return not {"english", "multilingual"} <= have
+ return False
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("trx", nargs="+", help="TRX file(s) written by `dotnet test --report-xunit-trx`.")
+ ap.add_argument(
+ "--have",
+ nargs="*",
+ default=[],
+ choices=sorted(set(CHECKPOINT_NAMES.values())),
+ help="Checkpoints whose artifacts this job provides; their tests must run, not skip.",
+ )
+ ap.add_argument("--min-passed", type=int, default=1, help="Fail below this many passed tests (default: 1).")
+ args = ap.parse_args()
+ have = set(args.have)
+
+ passed = failed = 0
+ allowed = collections.Counter()
+ bad_skips = collections.Counter()
+ failures = []
+ for path in args.trx:
+ for r in ET.parse(path).getroot().iter(NS + "UnitTestResult"):
+ outcome = r.get("outcome")
+ if outcome == "Passed":
+ passed += 1
+ elif outcome in FAILING:
+ failed += 1
+ failures.append(r.get("testName", "<unnamed>"))
+ elif outcome == "NotExecuted":
+ reason = reason_of(r)
+ if expected_skip(reason, have):
+ allowed[reason] += 1
+ else:
+ bad_skips[(r.get("testName", "<unnamed>").split("(")[0], reason)] += 1
+ else:
+ bad_skips[(r.get("testName", "<unnamed>").split("(")[0], "outcome " + str(outcome))] += 1
+
+ n_bad = sum(bad_skips.values())
+ print("passed: %d failed: %d skipped (expected): %d skipped (UNEXPECTED): %d"
+ % (passed, failed, sum(allowed.values()), n_bad))
+ print("checkpoints expected to run: %s" % (", ".join(sorted(have)) or "none"))
+ for reason, n in allowed.most_common():
+ print(" expected skip x%d: %s" % (n, reason[:140]))
+
+ ok = True
+ if failed:
+ ok = False
+ print("\nFAIL: %d test(s) failed, e.g. %s" % (failed, ", ".join(failures[:5])))
+ if n_bad:
+ ok = False
+ print("\nFAIL: %d test(s) were skipped that should have run (missing or unusable artifacts?):" % n_bad)
+ for (name, reason), n in bad_skips.most_common(10):
+ print(" x%d %s: %s" % (n, name, reason[:200]))
+ if passed < args.min_passed:
+ ok = False
+ print("\nFAIL: only %d test(s) passed, expected at least %d" % (passed, args.min_passed))
+ if ok:
+ print("\nOK: nothing that should have run was skipped")
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/laya-dotnet/tools/export_onnx.py b/laya-dotnet/tools/export_onnx.py
index 0ea2d2d1..0487ce93 100644
--- a/laya-dotnet/tools/export_onnx.py
+++ b/laya-dotnet/tools/export_onnx.py
@@ -44,7 +44,6 @@ import json
import os
import shutil
import sys
-import tempfile
import time
TOOLS = os.path.dirname(os.path.abspath(__file__))
@@ -58,7 +57,7 @@ from laya.common import ( # noqa: E402
build_sequence,
collate_items,
)
-from laya.router import BUNDLE_REPO, DEFAULT_MODELS, STANDALONE_MODELS # noqa: E402
+from laya.router import DEFAULT_MODELS # noqa: E402
# ─── model catalogue ────────────────────────────────────────────────────────
@@ -67,7 +66,7 @@ from laya.router import BUNDLE_REPO, DEFAULT_MODELS, STANDALONE_MODELS # noqa:
_NAMES = ["english", "multilingual", "typed-decisions"]
-def _resolve_spec(name_or_path: str, token: str):
+def _resolve_spec(name_or_path: str, token: str, revision: str = None):
"""Return (model_dir, subfolder, label) for a name or a local path."""
if os.path.exists(name_or_path):
# Local directory: treat as a self-contained checkpoint.
@@ -85,6 +84,7 @@ def _resolve_spec(name_or_path: str, token: str):
prefix = (subfolder + "/") if subfolder else ""
kw = {
"token": token or os.environ.get("HF_TOKEN"),
+ "revision": revision,
"allow_patterns": [
prefix + n for n in (
"rl_agent_config.json",
@@ -416,8 +416,6 @@ def verify(out_dir: str, cfg: dict, seq_lens=None, batch_sizes=None, marker_coun
import onnxruntime as ort
import torch
- from laya.common import build_model as _build_model
-
if seq_lens is None:
seq_lens = [16, 32, 64, 128, 256, cfg.get("max_len", 512)]
if batch_sizes is None:
@@ -540,7 +538,6 @@ def _make_synthetic_feeds(B, seq, M):
def _load_pt_model_if_cached(cfg, out_dir):
"""Return a loaded PyTorch model if weights are available locally, else None."""
from safetensors.torch import load_file
- import torch
# The export was done from a snapshot_download() cache; try to locate the
# safetensors file there. If it is gone (evicted or never downloaded because
@@ -659,7 +656,7 @@ def check_graph_signature(out_dir: str, reference_onnx: str):
# ─── main per-model export ───────────────────────────────────────────────────
def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_verify: bool,
- force: bool = False):
+ force: bool = False, revision: str = None):
"""Download (if needed), export, and verify one checkpoint.
Raises RuntimeError if <out_root>/<name>/model.onnx already exists and
@@ -687,7 +684,7 @@ def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_veri
)
# 2. Resolve model directory (may trigger a HuggingFace download).
- model_dir, subfolder, label = _resolve_spec(name_or_path, token)
+ model_dir, subfolder, label = _resolve_spec(name_or_path, token, revision)
if subfolder:
model_dir = os.path.join(model_dir, subfolder)
@@ -760,8 +757,6 @@ def export_one(name_or_path: str, out_root: str, token: str, opset: int, do_veri
def _check_against_reference(out_dir: str, label: str):
"""If a pre-built reference exists in onnx/, compare signatures."""
- import onnx
-
ref_candidates = {
"english": os.path.join(REPO, "onnx", "english", "model.onnx"),
"multilingual": os.path.join(REPO, "onnx", "multilingual", "model.onnx"),
@@ -816,6 +811,13 @@ def main():
help="After export, reload in ONNX Runtime and compare to PyTorch "
"across a range of tensor shapes.",
)
+ ap.add_argument(
+ "--revision",
+ default=None,
+ metavar="SHA",
+ help="Hugging Face commit (or branch/tag) to download the checkpoint from "
+ "(default: the repo's main branch).",
+ )
ap.add_argument(
"--force",
action="store_true",
@@ -850,6 +852,7 @@ def main():
opset=args.opset,
do_verify=args.verify,
force=args.force,
+ revision=args.revision,
)
out_dirs.append((target, od))
# Signature check against the pre-built reference (if any).
diff --git a/laya-dotnet/tools/regen_golden.py b/laya-dotnet/tools/regen_golden.py
new file mode 100644
index 00000000..fa84f7e7
--- /dev/null
+++ b/laya-dotnet/tools/regen_golden.py
@@ -0,0 +1,263 @@
+"""Regenerate the .NET golden fixtures from the shipping Python code, end to end.
+
+This is the single entry point the `.NET` CI lane runs (`.github/workflows/dotnet.yml`) and the
+one command a contributor needs after changing anything in `laya/` that the goldens capture
+(sequence building, option rendering, calibration, language detection, ...). It only chains the
+existing tools; none of their logic is repeated here:
+
+ 1. downloads the checkpoint at a *pinned* Hugging Face revision (`HF_REVISION` below),
+ 2. exports the fused layout -> <artifacts-root>/onnx/<checkpoint>/ (tools/export_onnx.py)
+ 3. exports the split layout -> <artifacts-root>/onnx-split/<checkpoint>/ (laya-ts/scripts/export_onnx.py)
+ 4. records tests/Laya.Tests/golden/<checkpoint>/ from the real `laya.Agent` (tools/dump_golden.py)
+ 5. records tests/Laya.Tests/golden/routing/ (tools/dump_routing_golden.py)
+
+Exports are the slow part, so an artifact directory that is already present, complete and was
+produced from the same inputs (revision + exporter scripts + `laya/common.py`, which holds the
+model definition) is reused. That is what lets CI cache `<artifacts-root>/onnx*` between runs.
+The goldens themselves are always rewritten.
+
+The default artifacts root is the repository root, which is where the test project looks for
+`onnx/<checkpoint>` and `onnx-split/<checkpoint>` on its own (it walks up from the test binary).
+Point `--artifacts-root` elsewhere and set `LAYA_ONNX_ROOT=<root>/onnx` for the fused layout; the
+split layout has no environment variable, so it is only found at `<some ancestor>/onnx-split`.
+
+Usage:
+ # Everything: three checkpoints, split layouts and the routing goldens
+ python laya-dotnet/tools/regen_golden.py
+
+ # One checkpoint, artifacts somewhere with room (each checkpoint is ~1-3 GB, fused + split)
+ python laya-dotnet/tools/regen_golden.py --checkpoint english --artifacts-root D:/laya-artifacts
+
+ # Only the model-free routing goldens
+ python laya-dotnet/tools/regen_golden.py --checkpoint routing
+
+ # Re-export even when a valid artifact directory exists
+ python laya-dotnet/tools/regen_golden.py --checkpoint multilingual --force-export
+
+Needs the packages in `tools/requirements-regen.txt` plus a CPU build of torch. Exits non-zero
+as soon as any step fails.
+"""
+
+import argparse
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+
+from google.protobuf.message import DecodeError
+
+TOOLS = os.path.dirname(os.path.abspath(__file__))
+REPO = os.path.dirname(os.path.dirname(TOOLS))
+sys.path.insert(0, REPO)
+
+# The Hugging Face commit every checkpoint is exported from. The weights, encoder configs and
+# tokenizers are byte-identical from 458d7563 (the commit that bundled the typed-decisions
+# checkpoint) through this one; later commits only touch the model card. Bump it deliberately,
+# then verify the regenerated goldens against the SDK.
+HF_REPO = "convaiinnovations/laya"
+HF_REVISION = "55cf4c4ebb4ebe31b2550e8bdf3bd21b99753851"
+
+CHECKPOINTS = ["english", "multilingual", "typed-decisions"]
+
+GOLDEN_ROOT = os.path.join(REPO, "laya-dotnet", "tests", "Laya.Tests", "golden")
+EXPORT_FUSED = os.path.join(TOOLS, "export_onnx.py")
+EXPORT_SPLIT = os.path.join(REPO, "laya-ts", "scripts", "export_onnx.py")
+DUMP_GOLDEN = os.path.join(TOOLS, "dump_golden.py")
+DUMP_ROUTING = os.path.join(TOOLS, "dump_routing_golden.py")
+
+# Everything that decides what an export looks like. Its hash is stored beside each exported
+# directory; a mismatch (or a missing stamp) means the export is stale and is redone.
+EXPORT_INPUTS = [
+ EXPORT_FUSED,
+ EXPORT_SPLIT,
+ os.path.join(TOOLS, "requirements-regen.txt"),
+ os.path.join(REPO, "laya", "common.py"),
+]
+
+STAMP = ".regen-stamp.json"
+FUSED_FILES = [
+ "model.onnx",
+ "model.onnx.data",
+ "rl_agent_config.json",
+ os.path.join("tokenizer", "tokenizer.json"),
+ "fixtures.npz",
+]
+SPLIT_FILES = ["encoder.onnx", "head.onnx", "rl_agent_config.json", "tokenizer.json"]
+
+
+def inputs_hash(revision: str) -> str:
+ h = hashlib.sha256(revision.encode("utf-8"))
+ for path in EXPORT_INPUTS:
+ with open(path, "rb") as f:
+ # Normalise line endings so a CRLF checkout hashes like the LF one.
+ h.update(f.read().replace(b"\r\n", b"\n"))
+ return h.hexdigest()
+
+
+def is_valid(path: str, files: list, stamp: str) -> bool:
+ """The directory holds a complete export made from exactly these inputs."""
+ try:
+ if not all(os.path.getsize(os.path.join(path, f)) > 0 for f in files):
+ return False
+ with open(os.path.join(path, STAMP), encoding="utf-8") as f:
+ if json.load(f).get("inputs_sha256") != stamp:
+ return False
+ import onnx
+
+ for file in files:
+ if not file.endswith(".onnx"):
+ continue
+ model = onnx.load(os.path.join(path, file), load_external_data=False)
+ for tensor in model.graph.initializer:
+ if tensor.data_location != onnx.TensorProto.EXTERNAL:
+ continue
+ data = {entry.key: entry.value for entry in tensor.external_data}
+ sidecar = os.path.join(path, data["location"])
+ size = os.path.getsize(sidecar)
+ if size <= 0 or size < int(data.get("offset", 0)) + int(data.get("length", 0)):
+ return False
+ return True
+ except (OSError, ValueError, KeyError, DecodeError):
+ return False
+
+
+def run(cmd: list, env: dict = None):
+ print("\n$ " + " ".join(cmd), flush=True)
+ t0 = time.perf_counter()
+ subprocess.run(cmd, check=True, cwd=REPO, env=env)
+ print(" (%.1f s)" % (time.perf_counter() - t0), flush=True)
+
+
+def snapshot(name: str, revision: str) -> str:
+ """The pinned checkpoint directory in the Hugging Face cache (downloaded on first use)."""
+ from huggingface_hub import snapshot_download
+
+ from laya.router import DEFAULT_MODELS
+
+ repo, subfolder = DEFAULT_MODELS[name]
+ if repo != HF_REPO:
+ raise SystemExit(
+ "%s now lives in %r, but the pinned revision belongs to %r; update HF_REPO/HF_REVISION"
+ % (name, repo, HF_REPO)
+ )
+ prefix = (subfolder + "/") if subfolder else ""
+ # A superset of what the two exporters ask for, so each of them hits the cache.
+ allow = [prefix + n for n in (
+ "rl_agent_config.json", "model.safetensors", "config.json",
+ "tokenizer.json", "tokenizer/*", "encoder/*",
+ )]
+ root = snapshot_download(repo, revision=revision, allow_patterns=allow)
+ return os.path.join(root, subfolder) if subfolder else root
+
+
+def publish(partial: str, final: str, stamp: str):
+ """Stamp a finished export and move it into place, so an interrupted run leaves nothing half-valid."""
+ with open(os.path.join(partial, STAMP), "w", encoding="utf-8") as f:
+ json.dump({"inputs_sha256": stamp}, f)
+ if os.path.exists(final):
+ shutil.rmtree(final)
+ os.makedirs(os.path.dirname(final), exist_ok=True)
+ os.replace(partial, final)
+
+
+def export_fused(name: str, root: str, revision: str, stamp: str, env: dict):
+ final = os.path.join(root, "onnx", name)
+ staging = os.path.join(root, ".partial", "onnx")
+ shutil.rmtree(staging, ignore_errors=True)
+ run([sys.executable, EXPORT_FUSED, "--model", name, "--out", staging, "--revision", revision], env=env)
+ publish(os.path.join(staging, name), final, stamp)
+
+
+def export_split(name: str, root: str, model_dir: str, stamp: str, env: dict):
+ final = os.path.join(root, "onnx-split", name)
+ staging = os.path.join(root, ".partial", "onnx-split", name)
+ shutil.rmtree(staging, ignore_errors=True)
+ run([sys.executable, EXPORT_SPLIT, "--model-dir", model_dir, "--out-dir", staging], env=env)
+ publish(staging, final, stamp)
+
+
+def regen_checkpoint(name: str, root: str, revision: str, force: bool):
+ print("\n=== %s ===" % name)
+ stamp = inputs_hash(revision)
+ fused = os.path.join(root, "onnx", name)
+ split = os.path.join(root, "onnx-split", name)
+ need_fused = force or not is_valid(fused, FUSED_FILES, stamp)
+ need_split = force or not is_valid(split, SPLIT_FILES, stamp)
+
+ if need_fused or need_split:
+ model_dir = snapshot(name, revision)
+ # Everything the exporters need is in the cache now; going back to the network would
+ # only be a way to pick up something that is not the pinned revision.
+ env = dict(os.environ, HF_HUB_OFFLINE="1", PYTHONPATH=REPO + os.pathsep + os.environ.get("PYTHONPATH", ""))
+ if need_fused:
+ export_fused(name, root, revision, stamp, env)
+ if need_split:
+ export_split(name, root, model_dir, stamp, env)
+ else:
+ print("fused and split exports are up to date; reusing %s and %s" % (fused, split))
+
+ run([sys.executable, DUMP_GOLDEN, "--checkpoint", name, "--onnx", fused])
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(
+ description=__doc__,
+ formatter_class=argparse.RawDescriptionHelpFormatter,
+ )
+ ap.add_argument(
+ "--checkpoint",
+ action="append",
+ choices=CHECKPOINTS + ["routing", "all"],
+ metavar="{%s,routing,all}" % ",".join(CHECKPOINTS),
+ help="What to regenerate; repeat to combine. 'routing' is the model-free routing/language/"
+ "email/shortlist goldens, 'all' is every checkpoint plus routing (default: all).",
+ )
+ ap.add_argument(
+ "--artifacts-root",
+ default=REPO,
+ help="Where onnx/<checkpoint> and onnx-split/<checkpoint> are written and reused "
+ "(default: the repository root, which the tests find on their own).",
+ )
+ ap.add_argument(
+ "--revision",
+ default=HF_REVISION,
+ metavar="SHA",
+ help="Hugging Face commit to export from (default: %(default)s).",
+ )
+ ap.add_argument(
+ "--force-export",
+ action="store_true",
+ help="Re-export even when a valid artifact directory already exists.",
+ )
+ args = ap.parse_args()
+
+ # The exporters print check marks; a Windows console code page cannot encode them.
+ os.environ.setdefault("PYTHONUTF8", "1")
+ selected = args.checkpoint or ["all"]
+ if "all" in selected:
+ selected = CHECKPOINTS + ["routing"]
+ checkpoints = [c for c in CHECKPOINTS if c in selected]
+ root = os.path.abspath(args.artifacts_root)
+
+ t0 = time.perf_counter()
+ try:
+ for name in checkpoints:
+ regen_checkpoint(name, root, args.revision, args.force_export)
+ if "routing" in selected:
+ print("\n=== routing ===")
+ run([sys.executable, DUMP_ROUTING, "--force"])
+ except subprocess.CalledProcessError as e:
+ print("\nFAILED (exit %d): %s" % (e.returncode, " ".join(e.cmd)), file=sys.stderr)
+ return e.returncode or 1
+ finally:
+ shutil.rmtree(os.path.join(root, ".partial"), ignore_errors=True)
+
+ print("\nregenerated %s in %.0f s" % (", ".join(checkpoints + (["routing"] if "routing" in selected else [])), time.perf_counter() - t0))
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/laya-dotnet/tools/requirements-regen.txt b/laya-dotnet/tools/requirements-regen.txt
new file mode 100644
index 00000000..28e3a815
--- /dev/null
+++ b/laya-dotnet/tools/requirements-regen.txt
@@ -0,0 +1,15 @@
+# Python toolchain for laya-dotnet/tools/regen_golden.py (and the .NET parity CI lane).
+#
+# Pinned to the versions the committed goldens were recorded with (see the
+# "*_version" fields in tests/Laya.Tests/golden/<checkpoint>/meta.json) so a toolchain
+# release cannot move the goldens. Bump deliberately, together with a regeneration.
+#
+# torch is not listed: it needs the CPU-only index, so install it first with
+# pip install "torch==2.14.0" --index-url https://download.pytorch.org/whl/cpu
+transformers==5.17.0
+safetensors==0.8.0
+huggingface_hub==1.33.0
+numpy==2.5.3
+onnx==1.23.0
+onnxruntime==1.30.0
+onnxscript==0.7.2
diff --git a/laya-dotnet/tools/test_ci_tools.py b/laya-dotnet/tools/test_ci_tools.py
new file mode 100644
index 00000000..548b9506
--- /dev/null
+++ b/laya-dotnet/tools/test_ci_tools.py
@@ -0,0 +1,101 @@
+"""Model-free regressions for the parity lane: python laya-dotnet/tools/test_ci_tools.py."""
+
+import contextlib
+import io
+import types
+import unittest
+import xml.etree.ElementTree as ET
+from unittest.mock import mock_open, patch
+
+from google.protobuf.message import DecodeError
+
+import check_test_skips as guard
+import regen_golden as regen
+
+
+class SkipGuardTests(unittest.TestCase):
+ def run_guard(self, outcomes, have=("english",), minimum=1):
+ root = ET.Element(guard.NS + "TestRun")
+ for outcome, reason in outcomes:
+ result = ET.SubElement(root, guard.NS + "UnitTestResult", outcome=outcome, testName="parity")
+ ET.SubElement(ET.SubElement(result, guard.NS + "Output"), guard.NS + "StdOut").text = reason
+ argv = ["check_test_skips.py", "run.trx", "--min-passed", str(minimum), "--have", *have]
+ with patch("sys.argv", argv), patch.object(guard.ET, "parse", return_value=ET.ElementTree(root)):
+ with contextlib.redirect_stdout(io.StringIO()):
+ return guard.main()
+
+ def test_passed_run(self):
+ self.assertEqual(0, self.run_guard([("Passed", "")]))
+
+ def test_absent_checkpoint_is_allowed(self):
+ self.assertEqual(0, self.run_guard([
+ ("Passed", ""), ("NotExecuted", "no ONNX artifacts for 'Multilingual'; set LAYA_ONNX_ROOT"),
+ ]))
+
+ def test_owned_checkpoint_is_not_allowed(self):
+ self.assertEqual(1, self.run_guard([
+ ("Passed", ""), ("NotExecuted", "no split ONNX artifact for 'English'; export it"),
+ ]))
+
+ def test_missing_goldens_are_not_allowed(self):
+ self.assertEqual(1, self.run_guard([
+ ("Passed", ""), ("NotExecuted", "no golden data for 'Multilingual'; regenerate"),
+ ]))
+
+ def test_unknown_skip_fails_closed(self):
+ self.assertEqual(1, self.run_guard([("Passed", ""), ("NotExecuted", "unexpected reason")]))
+
+ def test_failure_and_unknown_outcome_fail(self):
+ for outcome in [*guard.FAILING, "Unknown"]:
+ with self.subTest(outcome=outcome):
+ self.assertEqual(1, self.run_guard([("Passed", ""), (outcome, "")]))
+
+ def test_empty_or_wrong_filter_fails(self):
+ self.assertEqual(1, self.run_guard([]))
+ self.assertEqual(1, self.run_guard([("Passed", "")], minimum=2))
+
+ def test_router_requires_both_checkpoints(self):
+ self.assertTrue(guard.expected_skip(guard.ROUTER, {"english"}))
+ self.assertFalse(guard.expected_skip(guard.ROUTER, {"english", "multilingual"}))
+
+
+class ExportCacheTests(unittest.TestCase):
+ def validate(self, stamp="expected", sidecar_size=100, length=100, broken_graph=False):
+ import onnx
+
+ tensor = onnx.TensorProto(data_location=onnx.TensorProto.EXTERNAL)
+ for key, value in [("location", "encoder.onnx.data"), ("length", str(length))]:
+ entry = tensor.external_data.add()
+ entry.key, entry.value = key, value
+ model = types.SimpleNamespace(graph=types.SimpleNamespace(initializer=[tensor]))
+
+ def size(path):
+ if path.endswith(".data"):
+ if sidecar_size is None:
+ raise FileNotFoundError(path)
+ return sidecar_size
+ return 100
+
+ with patch("builtins.open", mock_open(read_data='{"inputs_sha256": "%s"}' % stamp)):
+ with patch.object(regen.os.path, "getsize", side_effect=size):
+ with patch.object(onnx, "load", return_value=model,
+ side_effect=DecodeError("broken") if broken_graph else None):
+ return regen.is_valid("artifacts", ["encoder.onnx"], "expected")
+
+ def test_complete_cache_is_reused(self):
+ self.assertTrue(self.validate())
+
+ def test_stale_stamp_is_rejected(self):
+ self.assertFalse(self.validate(stamp="old"))
+
+ def test_missing_empty_or_truncated_sidecar_is_rejected(self):
+ for size in [None, 0, 99]:
+ with self.subTest(size=size):
+ self.assertFalse(self.validate(sidecar_size=size))
+
+ def test_malformed_graph_is_rejected(self):
+ self.assertFalse(self.validate(broken_graph=True))
+
+
+if __name__ == "__main__":
+ unittest.main()