Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

ci(suites): a suite that crashes leaves evidence and an exit status

Aashishcff12025cce3
diff --git a/scripts/test_suites.py b/scripts/test_suites.py
index 922ade5f..4af8f17a 100644
--- a/scripts/test_suites.py
+++ b/scripts/test_suites.py
@@ -13,6 +13,12 @@ the test functions and exits 0, having asserted nothing (#374):
* ``SCRIPT_SUITES`` print their own PASS/FAIL summary and exit non-zero on failure.
* ``PYTEST_SUITES`` are collected and run by pytest in a single invocation.
+A suite that crashes, rather than failing, still has to leave evidence: children run with
+``PYTHONUNBUFFERED`` (their stdout is a pipe here, and a process that dies in native code never
+flushes it, so a buffered run loses every line it had already printed) and the failure summary
+carries each child's exit status, which is the only thing that tells a crash apart from a failed
+assertion.
+
``tests/test_langchain_real_mode.py`` is in ``SCRIPT_SUITES`` and fails (it does not skip) when
``langchain-core`` is missing, so a lane that runs this list needs the ``langchain`` extra -- which
is why the release build installs the same extras as the CI `test` job.
@@ -130,6 +136,26 @@ def _commands():
return commands
+# A child's stdout here is a pipe, and CPython block-buffers a pipe: a suite that dies in native
+# code never flushes it, so the log loses every line it had printed and is left naming a suite
+# that said nothing. Unbuffered, the evidence reaches the log before the crash does.
+CHILD_ENV = dict(os.environ, PYTHONUNBUFFERED="1")
+
+
+def _death_note(returncode):
+ """How a suite invocation ended, in the one number the child actually left behind.
+
+ A `FAILED`/`ERROR` line says *which* assertion did not hold; a signal (negative, POSIX) or an
+ NTSTATUS (Windows) says the suite never reached its own report. Hex goes alongside the decimal
+ for those because 3221225477 means nothing while 0xC0000005 is a name.
+ """
+ if returncode < 0:
+ return "killed by signal %d" % -returncode
+ if returncode > 0xFF:
+ return "exit %d (0x%08X)" % (returncode, returncode)
+ return "exit %d" % returncode
+
+
def main(argv=None):
parser = argparse.ArgumentParser(description="Run the shared CI/release test suite list.")
parser.add_argument("--list", action="store_true", help="print the suites and exit")
@@ -149,13 +175,14 @@ def main(argv=None):
for command in _commands():
label = " ".join(command[1:])
print("\n===== %s =====" % label, flush=True)
- if subprocess.run(command, cwd=ROOT).returncode != 0:
- failed.append(label)
+ returncode = subprocess.run(command, cwd=ROOT, env=CHILD_ENV).returncode
+ if returncode != 0:
+ failed.append((label, returncode))
if failed:
print("\n%d suite invocation(s) failed:" % len(failed))
- for label in failed:
- print(" - %s" % label)
+ for label, returncode in failed:
+ print(" - %s [%s]" % (label, _death_note(returncode)))
return 1
print("\nall %d suites passed" % len(suites()))
return 0
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
index a341cc98..3018f1ae 100644
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@ -3,11 +3,16 @@
Text parsing, not tomllib: the floor is 3.10 and tomllib arrives in 3.11.
"""
import ast
+import contextlib
import fnmatch
+import importlib.util
+import io
import os
import re
import shlex
+import subprocess
import sys
+import types
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -341,6 +346,91 @@ for label, source, job in (("ci/linux", workflow, "test"),
"expected run: python scripts/test_suites.py in %s" % job)
+# -------------------------------------------------- the runner's own failure surface
+# A suite does not always get to report. `tests (windows)` has printed
+# `1 suite invocation(s) failed: tests/test_runtime_fixes.py` with not one line of that suite's
+# output above it, and no exit status below -- a child's stdout is a pipe, CPython block-buffers a
+# pipe, and a process that leaves by a native fault never flushes. So the log could not tell a
+# crash from a failed assertion, which is a red lane nobody can diagnose. Pinned on both halves:
+# the child runs unbuffered so its evidence reaches the log first, and the summary carries the exit
+# status.
+_runner_spec = importlib.util.spec_from_file_location(
+ "laya_test_suites", os.path.join(ROOT, "scripts", "test_suites.py"))
+suites_runner = importlib.util.module_from_spec(_runner_spec)
+_runner_spec.loader.exec_module(suites_runner)
+
+# Through `getattr`, so a runner that lacks the surface reports which check failed instead of
+# aborting the suite on an AttributeError -- the thing this section exists to make impossible.
+check("harness/the child runs unbuffered",
+ getattr(suites_runner, "CHILD_ENV", {}).get("PYTHONUNBUFFERED"), "1")
+# The banner labels stay the bare suite paths the CI log has always printed, so a reader who greps
+# for a suite by name still finds it after the runner's own command line changes.
+check("harness/a command still labels as its suite paths",
+ suites_runner._commands()[0][1:], [suites_runner.SCRIPT_SUITES[0]])
+
+# The mechanism itself, not just the constant that asks for it: a child that dies by `os._exit`
+# has already written its line when the runner's env is used, and loses it when it is not.
+_DEATH = "import os; print('EVIDENCE'); os._exit(7)"
+
+
+def _die(env):
+ proc = subprocess.run([sys.executable, "-c", _DEATH], stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, env=env)
+ return proc.stdout.decode(), proc.returncode
+
+
+_strip = {k: v for k, v in os.environ.items() if k != "PYTHONUNBUFFERED"}
+_buffered_out, _buffered_rc = _die(_strip)
+_unbuffered_out, _unbuffered_rc = _die(getattr(suites_runner, "CHILD_ENV", None) or _strip)
+check_true("harness/a buffered child loses its evidence when it dies",
+ "EVIDENCE" not in _buffered_out, repr(_buffered_out))
+check_true("harness/the unbuffered child has already written it",
+ "EVIDENCE" in _unbuffered_out, repr(_unbuffered_out))
+check("harness/and the exit status survives either way",
+ (_buffered_rc, _unbuffered_rc), (7, 7))
+
+_death_note = getattr(suites_runner, "_death_note", None)
+
+
+def _note(code):
+ return _death_note(code) if _death_note is not None else None
+
+
+check("harness/a plain failure says exit", _note(1), "exit 1")
+check("harness/a signal says what did it", _note(-11), "killed by signal 11")
+check("harness/an NTSTATUS prints the form a reader recognises",
+ _note(3221225477), "exit 3221225477 (0xC0000005)")
+
+# End to end, because the note and the env only matter if `main` uses them: every child faked to
+# crash the way the Windows lane does, with the call recorded. Nothing here runs a real suite.
+_real_run = suites_runner.subprocess.run
+_calls = []
+
+
+def _crash(*args, **kwargs):
+ _calls.append(kwargs)
+ return types.SimpleNamespace(returncode=3221225477)
+
+
+suites_runner.subprocess.run = _crash
+try:
+ _summary_buf = io.StringIO()
+ with contextlib.redirect_stdout(_summary_buf):
+ _main_rc = suites_runner.main([])
+finally:
+ suites_runner.subprocess.run = _real_run
+check_true("harness/every child is given the unbuffered env",
+ _calls and all((call.get("env") or {}).get("PYTHONUNBUFFERED") == "1" for call in _calls),
+ "%d children, %d of them unbuffered"
+ % (len(_calls), sum(1 for c in _calls if (c.get("env") or {}).get("PYTHONUNBUFFERED"))))
+_summary = _summary_buf.getvalue()
+_last_line = (_summary.strip().splitlines() or ["<empty>"])[-1]
+check("harness/a crashed suite still fails the run", _main_rc, 1)
+check_true("harness/the summary names the suite", "tests/test_router.py" in _summary, _last_line)
+check_true("harness/the summary says how it died",
+ "exit 3221225477 (0xC0000005)" in _summary, _last_line)
+
+
# -------------------------------------------------- CONTRIBUTING.md's "Running the tests" section
# That section is what a new contributor runs, and it had drifted in three ways at once: it sent
# readers to `.github/workflows/ci.yml` for "the full set" (ci.yml names 21 suite paths while the