Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
#!/usr/bin/env python3
"""The one list of test suites CI and the release gate run.
`ci.yml` used to name the script-style suites inline and `release.yml` kept its own copy of the
list; they drifted, and the release gate ("tests must pass before anything is published") named 33
fewer suites than CI, so a regression in an omitted one could ship to PyPI. Both workflows invoke
this module now, so there is a single list to keep and `tests/test_packaging.py` fails if either
workflow stops using it.
Two kinds of suite, because `python tests/<name>.py` does not execute a pytest suite -- it defines
the test functions and exits 0, having asserted nothing (#374):
* ``SCRIPT_SUITES`` print their own PASS/FAIL summary and exit non-zero on failure.
* ``PYTEST_SUITES`` are collected and run by pytest in a single invocation.
A suite that crashes, rather than failing, still has to leave evidence: children run with
``PYTHONUNBUFFERED`` (their stdout is a pipe here, and a process that dies in native code never
flushes it, so a buffered run loses every line it had already printed) and the failure summary
carries each child's exit status, which is the only thing that tells a crash apart from a failed
assertion.
``tests/test_langchain_real_mode.py`` is in ``SCRIPT_SUITES`` and fails (it does not skip) when
``langchain-core`` is missing, so a lane that runs this list needs the ``langchain`` extra -- which
is why the release build installs the same extras as the CI `test` job.
"""
import argparse
import os
import subprocess
import sys
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SCRIPT_SUITES = [
"tests/test_router.py",
"tests/test_criteria.py",
"tests/test_batch.py",
"tests/test_predict_long.py",
"tests/test_attention_dynamic_shapes.py",
"tests/test_option_order.py",
"tests/test_parallel_options.py",
"tests/test_state_budget.py",
"tests/test_hooks.py",
"tests/test_hooks_api.py",
"tests/test_option_stability.py",
"tests/test_structured.py",
"tests/test_structured_api.py",
"tests/test_structured_docs.py",
"tests/test_onnx_lang_parity.py",
"tests/test_onnx_truncation_parity.py",
"tests/test_onnx_batch.py",
"tests/test_onnx_long.py",
"tests/test_onnx_sort.py",
"tests/test_download.py",
"tests/test_verify_checkpoints.py",
"tests/test_shortlist.py",
"tests/test_decision_model.py",
"tests/test_head_checkpointing.py",
"tests/test_truncation.py",
"tests/test_packaging.py",
"tests/test_imports.py",
"tests/test_compose_env.py",
"tests/test_audit_scope.py",
"tests/test_doc_tables.py",
"tests/test_env_docs.py",
"tests/test_doc_citations.py",
"tests/test_tokenizer_cache.py",
"tests/test_tokenizer_concurrency.py",
"tests/test_question_token_reuse.py",
"tests/test_lazy_import.py",
"tests/test_runtime_fixes.py",
"tests/test_backends.py",
"tests/test_fast_head_partition.py",
"tests/test_agent_gate.py",
"tests/test_option_collapse.py",
"tests/test_lang_guess.py",
"tests/test_identifier_complexity.py",
"tests/test_lang_stats.py",
"tests/test_calibration_persistence.py",
"tests/test_calibrate.py",
"tests/test_context_manager.py",
"tests/test_criteria_normalization.py",
"tests/test_docker_entrypoint.py",
"tests/test_modelscope_prefetch.py",
"tests/test_empty_questions.py",
"tests/test_router_memory.py",
"tests/test_shortlist_cosine.py",
"tests/test_temperature_loading.py",
"tests/test_confidence.py",
"tests/test_conformal_abstention.py",
"tests/test_cli.py",
"tests/test_cli_lang_guess.py",
"tests/test_evidence.py",
"tests/test_mcp.py",
"tests/test_mcp_device.py",
"tests/test_langchain.py",
"tests/test_agno.py",
"tests/test_portability.py",
"tests/test_guard_document_781.py",
"tests/test_training.py",
"tests/test_train.py",
"tests/test_kaggle_ddp_convergence.py",
"tests/test_example_server_limits.py",
"tests/test_blank_lang_routing.py",
"tests/test_export_onnx_safety.py",
"tests/test_load_errors.py",
"tests/test_revision_pinning.py",
"tests/test_example_server_errors.py",
"tests/test_example_server_per_call_controls.py",
"tests/test_crewai.py",
"tests/test_llamaindex.py",
"tests/test_langchain_real_mode.py",
"tests/test_email.py",
"tests/test_tools.py",
]
PYTEST_SUITES = [
"tests/test_lang_diagnostics.py",
"tests/test_evals_agreement.py",
"tests/test_evals_tournament.py",
"tests/test_serve.py",
"tests/test_mcp_remote.py",
"tests/test_mcp_diagnostics.py",
"tests/test_router_batch.py",
"tests/test_predict_batch.py",
"tests/test_system_one_lang.py",
"tests/test_audit_regressions.py",
"tests/test_truncation_direction.py",
"tests/test_compile.py",
"tests/test_finetune_entrypoints.py",
"tests/test_vega.py",
"tests/test_router_vega.py",
]
def suites():
"""Every suite in run order: the scripts, then one pytest invocation."""
return list(SCRIPT_SUITES) + list(PYTEST_SUITES)
def _commands():
commands = [[sys.executable, path] for path in SCRIPT_SUITES]
if PYTEST_SUITES:
commands.append([sys.executable, "-m", "pytest", *PYTEST_SUITES, "-q"])
return commands
# A child's stdout here is a pipe, and CPython block-buffers a pipe: a suite that dies in native
# code never flushes it, so the log loses every line it had printed and is left naming a suite
# that said nothing. Unbuffered, the evidence reaches the log before the crash does.
CHILD_ENV = dict(os.environ, PYTHONUNBUFFERED="1")
def _death_note(returncode):
"""How a suite invocation ended, in the one number the child actually left behind.
A `FAILED`/`ERROR` line says *which* assertion did not hold; a signal (negative, POSIX) or an
NTSTATUS (Windows) says the suite never reached its own report. Hex goes alongside the decimal
for those because 3221225477 means nothing while 0xC0000005 is a name.
"""
if returncode < 0:
return "killed by signal %d" % -returncode
if returncode > 0xFF:
return "exit %d (0x%08X)" % (returncode, returncode)
return "exit %d" % returncode
def main(argv=None):
parser = argparse.ArgumentParser(description="Run the shared CI/release test suite list.")
parser.add_argument("--list", action="store_true", help="print the suites and exit")
args = parser.parse_args(argv)
if args.list:
for path in suites():
print(path)
return 0
missing = [p for p in suites() if not os.path.exists(os.path.join(ROOT, p))]
if missing:
print("test_suites: suite file(s) missing: %s" % ", ".join(missing))
return 1
failed = []
for command in _commands():
label = " ".join(command[1:])
print("\n===== %s =====" % label, flush=True)
returncode = subprocess.run(command, cwd=ROOT, env=CHILD_ENV).returncode
if returncode != 0:
failed.append((label, returncode))
if failed:
print("\n%d suite invocation(s) failed:" % len(failed))
for label, returncode in failed:
print(" - %s [%s]" % (label, _death_note(returncode)))
return 1
print("\nall %d suites passed" % len(suites()))
return 0
if __name__ == "__main__":
sys.exit(main())