Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
ci: run one shared test list in CI and before a release
thread3deb7415496b5f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c97f733d..ecdc989a 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -98,116 +98,26 @@ jobs:- name: Import checkrun: python -c "import laya; print(laya.__version__); print(sorted(laya.__all__))"- - name: Routing and language-detection tests- run: |- python tests/test_router.py- python tests/test_criteria.py- python tests/test_batch.py- python tests/test_predict_long.py- python tests/test_attention_dynamic_shapes.py- python tests/test_option_order.py- python tests/test_state_budget.py- python tests/test_hooks.py- python tests/test_hooks_api.py- python tests/test_structured.py- python tests/test_structured_api.py- python tests/test_structured_docs.py- python tests/test_onnx_lang_parity.py- python tests/test_onnx_truncation_parity.py- python tests/test_onnx_batch.py- python tests/test_onnx_long.py- python tests/test_onnx_sort.py- python tests/test_download.py- python tests/test_verify_checkpoints.py- python tests/test_shortlist.py- python tests/test_decision_model.py- python tests/test_head_checkpointing.py- python tests/test_truncation.py- python tests/test_packaging.py- python tests/test_compose_env.py- python tests/test_audit_scope.py- python tests/test_doc_tables.py- python tests/test_env_docs.py- python tests/test_tokenizer_cache.py- python tests/test_tokenizer_concurrency.py- python tests/test_question_token_reuse.py- python tests/test_lazy_import.py- python tests/test_runtime_fixes.py- python tests/test_agent_gate.py- python tests/test_option_collapse.py- python tests/test_lang_guess.py- python tests/test_identifier_complexity.py- python tests/test_lang_stats.py- python tests/test_calibration_persistence.py- python tests/test_calibrate.py- python tests/test_context_manager.py- python tests/test_criteria_normalization.py- python tests/test_docker_entrypoint.py- python tests/test_modelscope_prefetch.py- python tests/test_empty_questions.py- python tests/test_router_memory.py- python tests/test_shortlist_cosine.py- python tests/test_temperature_loading.py- python tests/test_confidence.py- python tests/test_cli.py- python tests/test_cli_lang_guess.py- python tests/test_mcp.py- python tests/test_mcp_device.py- python tests/test_langchain.py- python tests/test_portability.py- python tests/test_training.py- python tests/test_example_server_limits.py- python tests/test_blank_lang_routing.py- python tests/test_export_onnx_safety.py- python tests/test_load_errors.py- python tests/test_predict_batch.py- python tests/test_revision_pinning.py- python tests/test_example_server_errors.py- python tests/test_example_server_per_call_controls.py- python tests/test_crewai.py- python tests/test_llamaindex.py-+ # The install above is what makes tests/test_langchain.py run on real Runnables, and+ # nothing proves it took: with langchain-core missing, the checks inside that file's+ # `if _RUNNABLE_AVAILABLE:` block vanish instead of failing and it still prints FAIL: 0.+ # Assert the install took before the shared list runs test_langchain_real_mode.py.- name: LangChain integration runs on the real base classrun: |- # The install above is what makes tests/test_langchain.py run on real Runnables, and- # nothing proves it took: with langchain-core missing, the eight checks inside that- # file's `if _RUNNABLE_AVAILABLE:` block vanish instead of failing, and it still- # prints FAIL: 0 -- measured here: PASS: 87 with the extra, PASS: 80 without, exit 0- # both ways. This step asserts the install took, then checks what only a real install- # can check.python -c "from laya.integrations.langchain import RunnableSerializable; \assert RunnableSerializable is not object, \'langchain-core is missing: laya.integrations.langchain fell back to a plain object, ' \'so the LangChain suites in this job test a shape no install produces'"- python tests/test_langchain_real_mode.py- - name: Email cleaning tests- run: python tests/test_email.py- # Full dependency tree here, so the stub pipeline and the metric pins that need numpy- # run instead of skipping. The audit itself runs in `feishu-benchmark`, where nothing- # is installed.+ # The canonical suite list lives in `scripts/test_suites.py`, which the release gate invokes+ # too. Naming the suites here and again in `release.yml` is how the publish gate came to run+ # 33 fewer than CI, so this step names one file and the script names them all (#374).+ - name: Test suites+ run: python scripts/test_suites.py+- name: Chinese short-command benchmark testsrun: python -m unittest discover -s research/benchmarks/zh_short_commands/tests -v- # These are pytest suites, and `python tests/<name>.py` does not run them: it defines- # the test functions and exits 0, having asserted nothing. They were counted by the suite- # list above without ever executing, which is how the HTTP surface came to have no coverage- # in CI at all (#374).- #- # test_onnx.py and test_fast.py are deliberately not here: both are skip-guarded on an extra- # this job does not install, so they would pass trivially, and their real execution belongs- # in a lane that installs `onnx` or has a GPU.- - name: Pytest suites (serve, batch, system-one, audit regressions, truncation, compile)- run: |- python -m pytest tests/test_serve.py \- tests/test_router_batch.py \- tests/test_predict_batch.py \- tests/test_system_one_lang.py \- tests/test_audit_regressions.py \- tests/test_truncation_direction.py \- tests/test_compile.py \- -q-# The `test` job above deliberately stays free of `onnx`: this is the lane that# installs it, so the exporter's quantization suite runs for real instead of# skip-guarding into a trivial pass. Weight-free -- it builds tiny syntheticdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 562aec32..4e00e94a 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -35,57 +35,15 @@ jobs:run: |python -m pip install --upgrade pip build twinepip install torch --index-url https://download.pytorch.org/whl/cpu- pip install -e . pytest+ # The same extras the CI `test` job installs, so the shared suite list below runs its+ # mcp/serve/langchain suites for real instead of letting them skip.+ pip install -e ".[mcp,serve,langchain]" httpx pytest+ # The canonical list CI runs (`scripts/test_suites.py`). This gate used to keep a hand-synced+ # copy that had drifted to 33 fewer suites than CI, so a regression in an omitted suite could+ # reach PyPI without any test failing.- name: Tests must pass before anything is published- run: |- python tests/test_router.py- python tests/test_criteria.py- python tests/test_batch.py- python tests/test_predict_long.py- python tests/test_state_budget.py- python tests/test_hooks.py- python tests/test_hooks_api.py- python tests/test_onnx_lang_parity.py- python tests/test_onnx_truncation_parity.py- python tests/test_onnx_batch.py- python tests/test_onnx_long.py- python tests/test_onnx_sort.py- python tests/test_email.py- python tests/test_download.py- python tests/test_shortlist.py- python tests/test_decision_model.py- python tests/test_head_checkpointing.py- python tests/test_truncation.py- python -m pytest tests/test_truncation_direction.py -q- python tests/test_packaging.py- python tests/test_compose_env.py- python tests/test_doc_tables.py- python tests/test_env_docs.py- python tests/test_tokenizer_cache.py- python tests/test_tokenizer_concurrency.py- python tests/test_lazy_import.py- python tests/test_runtime_fixes.py- python tests/test_agent_gate.py- python tests/test_lang_guess.py- python tests/test_option_collapse.py- python tests/test_identifier_complexity.py- python tests/test_lang_stats.py- python tests/test_calibration_persistence.py- python tests/test_context_manager.py- python tests/test_criteria_normalization.py- python tests/test_docker_entrypoint.py- python tests/test_empty_questions.py- python tests/test_router_memory.py- python tests/test_shortlist_cosine.py- python tests/test_temperature_loading.py- python tests/test_confidence.py- python tests/test_cli.py- python tests/test_training.py- python tests/test_portability.py- python tests/test_blank_lang_routing.py- python tests/test_load_errors.py- python tests/test_revision_pinning.py+ run: python scripts/test_suites.py- name: Buildrun: python -m builddiff --git a/scripts/test_suites.py b/scripts/test_suites.pynew file mode 100644index 00000000..8601e66d--- /dev/null+++ b/scripts/test_suites.py@@ -0,0 +1,152 @@+#!/usr/bin/env python3+"""The one list of test suites CI and the release gate run.++`ci.yml` used to name the script-style suites inline and `release.yml` kept its own copy of the+list; they drifted, and the release gate ("tests must pass before anything is published") named 33+fewer suites than CI, so a regression in an omitted one could ship to PyPI. Both workflows invoke+this module now, so there is a single list to keep and `tests/test_packaging.py` fails if either+workflow stops using it.++Two kinds of suite, because `python tests/<name>.py` does not execute a pytest suite -- it defines+the test functions and exits 0, having asserted nothing (#374):++* ``SCRIPT_SUITES`` print their own PASS/FAIL summary and exit non-zero on failure.+* ``PYTEST_SUITES`` are collected and run by pytest in a single invocation.++``tests/test_langchain_real_mode.py`` is in ``SCRIPT_SUITES`` and fails (it does not skip) when+``langchain-core`` is missing, so a lane that runs this list needs the ``langchain`` extra -- which+is why the release build installs the same extras as the CI `test` job.+"""+import argparse+import os+import subprocess+import sys++ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))++SCRIPT_SUITES = [+ "tests/test_router.py",+ "tests/test_criteria.py",+ "tests/test_batch.py",+ "tests/test_predict_long.py",+ "tests/test_attention_dynamic_shapes.py",+ "tests/test_option_order.py",+ "tests/test_state_budget.py",+ "tests/test_hooks.py",+ "tests/test_hooks_api.py",+ "tests/test_structured.py",+ "tests/test_structured_api.py",+ "tests/test_structured_docs.py",+ "tests/test_onnx_lang_parity.py",+ "tests/test_onnx_truncation_parity.py",+ "tests/test_onnx_batch.py",+ "tests/test_onnx_long.py",+ "tests/test_onnx_sort.py",+ "tests/test_download.py",+ "tests/test_verify_checkpoints.py",+ "tests/test_shortlist.py",+ "tests/test_decision_model.py",+ "tests/test_head_checkpointing.py",+ "tests/test_truncation.py",+ "tests/test_packaging.py",+ "tests/test_compose_env.py",+ "tests/test_audit_scope.py",+ "tests/test_doc_tables.py",+ "tests/test_env_docs.py",+ "tests/test_tokenizer_cache.py",+ "tests/test_tokenizer_concurrency.py",+ "tests/test_question_token_reuse.py",+ "tests/test_lazy_import.py",+ "tests/test_runtime_fixes.py",+ "tests/test_agent_gate.py",+ "tests/test_option_collapse.py",+ "tests/test_lang_guess.py",+ "tests/test_identifier_complexity.py",+ "tests/test_lang_stats.py",+ "tests/test_calibration_persistence.py",+ "tests/test_calibrate.py",+ "tests/test_context_manager.py",+ "tests/test_criteria_normalization.py",+ "tests/test_docker_entrypoint.py",+ "tests/test_modelscope_prefetch.py",+ "tests/test_empty_questions.py",+ "tests/test_router_memory.py",+ "tests/test_shortlist_cosine.py",+ "tests/test_temperature_loading.py",+ "tests/test_confidence.py",+ "tests/test_cli.py",+ "tests/test_cli_lang_guess.py",+ "tests/test_mcp.py",+ "tests/test_mcp_device.py",+ "tests/test_langchain.py",+ "tests/test_portability.py",+ "tests/test_training.py",+ "tests/test_example_server_limits.py",+ "tests/test_blank_lang_routing.py",+ "tests/test_export_onnx_safety.py",+ "tests/test_load_errors.py",+ "tests/test_revision_pinning.py",+ "tests/test_example_server_errors.py",+ "tests/test_example_server_per_call_controls.py",+ "tests/test_crewai.py",+ "tests/test_llamaindex.py",+ "tests/test_langchain_real_mode.py",+ "tests/test_email.py",+]++PYTEST_SUITES = [+ "tests/test_serve.py",+ "tests/test_router_batch.py",+ "tests/test_predict_batch.py",+ "tests/test_system_one_lang.py",+ "tests/test_audit_regressions.py",+ "tests/test_truncation_direction.py",+ "tests/test_compile.py",+]+++def suites():+ """Every suite in run order: the scripts, then one pytest invocation."""+ return list(SCRIPT_SUITES) + list(PYTEST_SUITES)+++def _commands():+ commands = [[sys.executable, path] for path in SCRIPT_SUITES]+ if PYTEST_SUITES:+ commands.append([sys.executable, "-m", "pytest", *PYTEST_SUITES, "-q"])+ return commands+++def main(argv=None):+ parser = argparse.ArgumentParser(description="Run the shared CI/release test suite list.")+ parser.add_argument("--list", action="store_true", help="print the suites and exit")+ args = parser.parse_args(argv)++ if args.list:+ for path in suites():+ print(path)+ return 0++ missing = [p for p in suites() if not os.path.exists(os.path.join(ROOT, p))]+ if missing:+ print("test_suites: suite file(s) missing: %s" % ", ".join(missing))+ return 1++ failed = []+ for command in _commands():+ label = " ".join(command[1:])+ print("\n===== %s =====" % label, flush=True)+ if subprocess.run(command, cwd=ROOT).returncode != 0:+ failed.append(label)++ if failed:+ print("\n%d suite invocation(s) failed:" % len(failed))+ for label in failed:+ print(" - %s" % label)+ return 1+ print("\nall %d suites passed" % len(suites()))+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tests/test_packaging.py b/tests/test_packaging.pyindex f7ab2abd..3ed6e055 100644--- a/tests/test_packaging.py+++ b/tests/test_packaging.py@@ -174,7 +174,16 @@ def _invoked_workflow_tests(yaml_text):docker_workflow = read(os.path.join(".github", "workflows", "docker.yml"))-registered_test_files = _invoked_workflow_tests(workflow) | _invoked_workflow_tests(docker_workflow)+release_workflow = read(os.path.join(".github", "workflows", "release.yml"))+# The Linux `test` job and the release gate both invoke `scripts/test_suites.py` instead of naming+# their suites inline, so that shared list is a source of suite names too. Without this the+# "every suite is wired" check would fail for every suite the workflows no longer name.+shared_suites = read(os.path.join("scripts", "test_suites.py"))+registered_test_files = (+ _invoked_workflow_tests(workflow)+ | _invoked_workflow_tests(docker_workflow)+ | set(re.findall(r"\btests/(test_[a-zA-Z0-9_]+\.py)\b", shared_suites))+)EXEMPT_TEST_SUITES = {"test_local_e2e.py": "Requires local checkpoints under ~/laya_models (AGENTS.md)",@@ -193,6 +202,10 @@ untested_suites = [if f not in EXEMPT_TEST_SUITES and f not in registered_test_files]check("ci/wires every non-exempt test suite", untested_suites, [])+# The point of the shared list is that CI and the publish gate run the same suites. Both invoking+# the script is the entire mechanism, so assert neither workflow dropped back to an inline list.+check_true("ci and the release gate share one suite list",+ "scripts/test_suites.py" in workflow and "scripts/test_suites.py" in release_workflow, "")# --------------------------------------------------------------- markdown links