Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Stop two CI guards failing for the machine they ran on (#11164)

Daniel Han9c5b54d6b1ae
diff --git a/studio/backend/tests/test_data_recipe_dataset_download.py b/studio/backend/tests/test_data_recipe_dataset_download.py
index b39a724e194..b3300e8d7a6 100644
--- a/studio/backend/tests/test_data_recipe_dataset_download.py
+++ b/studio/backend/tests/test_data_recipe_dataset_download.py
@@ -152,14 +152,29 @@ def test_build_dataset_download_leaves_no_temp_file_when_export_fails(tmp_path:
lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("export blew up")),
)
- before = set(Path(tempfile.gettempdir()).glob("*.jsonl"))
+ # A private system temp dir, not the shared one. The claim is "the failed export unlinked
+ # what it created", and reading that off $TMPDIR made it a claim about the whole box: the
+ # siblings in this file each leave a NamedTemporaryFile(suffix = ".jsonl") in flight until
+ # their own finally runs, and CI shards this suite with `-n 4 --dist loadgroup`, so a
+ # sibling on another worker could put a .jsonl there between the two reads and fail this
+ # test for something it does not measure. Pointing tempfile.tempdir at a directory only
+ # this test can reach makes the reading exact instead of merely usually-quiet, and lets
+ # the assertion be "nothing at all" rather than "the same set as a moment ago".
+ system_temp = tmp_path / "system-temp"
+ system_temp.mkdir()
+ monkeypatch.setattr(tempfile, "tempdir", str(system_temp))
+ assert Path(tempfile.gettempdir()) == system_temp, "the export must allocate under our dir"
+
with pytest.raises(RuntimeError):
build_dataset_download(
artifact_path = str(dataset_path),
export_format = "jsonl",
filename_stem = "leaky",
)
- assert set(Path(tempfile.gettempdir()).glob("*.jsonl")) == before
+ assert sorted(system_temp.iterdir()) == [], (
+ "the failed export left its temporary file behind: "
+ f"{sorted(p.name for p in system_temp.iterdir())}"
+ )
def test_build_dataset_download_parquet_zip_includes_images(tmp_path: Path, monkeypatch):
diff --git a/tests/_shared/growth.py b/tests/_shared/growth.py
index 394f9e0e186..ee8cc6f9f7f 100644
--- a/tests/_shared/growth.py
+++ b/tests/_shared/growth.py
@@ -27,11 +27,32 @@ def growth(
factor: int = 4,
repeats: int = 3,
abort_over_s: float = None,
+ budget_s: float = None,
+ clock = None,
):
- """How much more `factor` times the input costs. Returns (ratio, big_seconds, big_result).
+ """How much more `factor` times the input costs.
+
+ Returns (ratio, best_big_seconds, big_result, pairs_completed). `pairs_completed` is
+ less than `repeats` only when `abort_over_s` or `budget_s` cut the loop short, and a
+ caller that treats the ratio as a verdict has to look at it: one aborted pair whose
+ small leg was also slow can report a ratio under the bar from a sample that never
+ finished.
+
+ `abort_over_s` bounds ONE BIG LEG; `budget_s` bounds THE WHOLE CALL. They are not the
+ same guard and neither implies the other: `repeats` legs each just under `abort_over_s`
+ is `repeats` times the cost that one leg was allowed, which is how a sample sized from a
+ previous reading still overran the runner's per-test timeout. `budget_s` is measured
+ rather than predicted, and it is asked BEFORE each pair rather than after, reserving
+ room at the cost of the worst pair seen so far -- a bound tested only once a pair is
+ already home is not a bound on that pair.
`build(n)` makes an input of size n and `run(text)` is the thing being measured.
+ `clock` is `time.perf_counter` unless given. It exists so this module's OWN tests can
+ state a timing shape exactly instead of sleeping for it: a test of a flakiness guard
+ that needs the scheduler to cooperate is the thing being fixed here, not a way to
+ check it. Nothing in the repo passes it outside those tests.
+
The MEDIAN of `repeats` PAIRED ratios: each pair times the small input and then the big
one back to back, and the ratio is formed inside the pair before anything is aggregated.
@@ -68,16 +89,30 @@ def growth(
import statistics as _statistics
import time as _time
+ read_clock = _time.perf_counter if clock is None else clock
+
def once(text):
- start = _time.perf_counter()
+ start = read_clock()
result = run(text)
- return _time.perf_counter() - start, result
+ return read_clock() - start, result
small_text, big_text = build(units), build(units * factor)
ratios, big, result = [], None, None
+ started, pair_costs = read_clock(), []
for _ in range(repeats):
+ # Asked BEFORE the pair, not after it. A check that only fires once both legs are
+ # home cannot stop the pair that overran: three 59-second pairs under a 120-second
+ # bound run to 177, because each one is inside the bound at the moment it starts.
+ # So the loop reserves room for another pair at the cost of the WORST it has seen,
+ # which is the conservative reading -- contention only adds -- and the estimate
+ # comes from this sample rather than from a previous one.
+ if budget_s is not None and pair_costs:
+ if read_clock() - started + max(pair_costs) > budget_s:
+ break
+ pair_started = read_clock()
small_elapsed, _ = once(small_text)
big_elapsed, result = once(big_text)
+ pair_costs.append(read_clock() - pair_started)
# The backstop below reads the BEST big time, not the worst. Contention only adds, so
# the minimum is the closest this size got to its own cost, and the backstop should
# fire on a path that is genuinely too slow rather than on a runner that stalled once.
@@ -90,7 +125,7 @@ def growth(
if abort_over_s is not None and big_elapsed > abort_over_s:
break
- return _statistics.median(ratios), big, result
+ return _statistics.median(ratios), big, result, len(ratios)
def assert_linear(
@@ -101,6 +136,9 @@ def assert_linear(
*,
factor: int = 4,
tolerance: float = 6.0,
+ repeats_on_retry: int = 7,
+ total_budget_s: float = 240.0,
+ clock = None,
):
"""`run(build(n))` must cost ~`factor`x, not ~`factor ** 2`x, for `factor`x the input.
@@ -110,16 +148,116 @@ def assert_linear(
being guarded against that leg is `factor ** 2`x slower again. A guard whose broken case
takes a minute at the old size would then take a quarter of an hour, and the job's own
timeout kills it before the ratio below can say why.
+
+ `total_budget_s` caps what this call may SPEND IN TOTAL, for the same reason the paragraph
+ above exists: a red run has to arrive as this function's message and not as the runner's
+ timeout, or nobody learns which shape was measured. Everything is inside it, first sample
+ included -- the first sample has no bound of its own beyond the 60s per big leg, so a
+ scheduler stall in one of its small legs can spend most of the runner's patience before
+ the re-measurement is even considered, and a retry budget counted fresh from that point
+ overruns whatever was left. 240s against the `--timeout=330` those CI invocations pass,
+ leaving margin for collection and the rest of the test body.
"""
+ import time as _time
+
budget = 60.0
+ read_clock = _time.perf_counter if clock is None else clock
+ started = read_clock()
# Passed down rather than checked here: on a path slow enough to trip it, every repeat
# is another minute spent measuring something already known to be too slow.
- ratio, big, result = growth(run, build, units, factor, abort_over_s = budget)
+ ratio, big, result, first_pairs = growth(
+ run,
+ build,
+ units,
+ factor,
+ abort_over_s = budget,
+ budget_s = total_budget_s,
+ clock = clock,
+ )
# Backstop: a regression bad enough to make the ratio unmeasurable still has to fail, and
# fail quickly, rather than run until the job's own timeout kills it with no explanation.
assert big < budget, f"{label} path took {big:.1f}s on {units * factor} units"
- assert ratio < tolerance, (
- f"{label} path is not linear: {factor}x the input cost {ratio:.1f}x the time "
- f"(linear is ~{factor}, quadratic is ~{factor ** 2})"
+ # The budget covers THIS sample too, and an early stop is not a verdict. Contention that
+ # lands in the SMALL legs is the case: their ratios stay under the tolerance, so nothing
+ # reaches the retry branch below where the budget used to be the only one enforced, and
+ # three pairs of a slow-but-linear-looking path ran past the runner's patience and passed.
+ assert first_pairs == 3, (
+ f"{label} path: the first sample stopped after {first_pairs} of 3 pairs, on the "
+ f"{total_budget_s:.0f}s this call is allowed, so its ratio is not a reading of anything"
)
+ if ratio >= tolerance:
+ # RE-MEASURE rather than loosen. Pairing divides most contention out, but not all of
+ # it: the big leg runs `factor`x longer than the small one, so a scheduler stall that
+ # lands inside a run is `factor`x more likely to land in the big half, which biases a
+ # pair upward and never downward. Three pairs is few enough that two unlucky ones move
+ # the median, which is how this reported 7.2x on unslothai/unsloth#11152, a branch that
+ # touches none of this code.
+ #
+ # A second, larger sample is the honest answer, and it is not a second chance: a path
+ # that is genuinely quadratic measures ~`factor ** 2` in EVERY pair, so its second
+ # median comes back over the bar as surely as its first, while a contention spike does
+ # not survive being asked again on a bigger sample. The cost is paid only on the
+ # reading that would otherwise have failed, so a green run still takes three pairs.
+ #
+ # How many pairs it can AFFORD is a separate question from how many it wants. The
+ # per-leg backstop above bounds one leg, not the sample: a regression that holds each
+ # big leg just under 60s still costs ~7 * 60s here, and the four invocations in
+ # .github/workflows/studio-backend-ci.yml pass `--timeout=330`, so the worker would be
+ # killed mid-confirmation and report a bare timeout instead of the linearity failure
+ # this guard exists to name. Contention is the same size in every pair, so a shorter
+ # confirmation is a weaker vote but still a reading; the small leg is estimated from
+ # the ratio already measured, which is the only reading of it available here.
+ #
+ # The sizing below is an ESTIMATE and is deliberately not the guard. It is built from
+ # the first sample's BEST big leg and MEDIAN ratio, which are readings of different
+ # pairs, so a sample of 59s, 59s and 1s big legs offers a 1s pair cost and authorises
+ # all seven -- and if the confirmation's legs then come in at 40s it overruns anyway.
+ # The estimate exists to refuse a retry that obviously cannot fit and to keep a green
+ # run cheap; `budget_s` below is what actually stops the spending, because it is
+ # measured while the sample runs rather than predicted before it starts.
+ #
+ # What is left, not a fresh allowance. The first sample has no bound of its own past
+ # the 60s per big leg, so a scheduler stall in one of its small legs can spend most of
+ # the runner's patience before this point is reached, and a confirmation counted fresh
+ # from here overruns whatever remained.
+ remaining = total_budget_s - (read_clock() - started)
+ pair_cost = big * (1.0 + 1.0 / max(ratio, 1.0))
+ affordable = repeats_on_retry if pair_cost <= 0 else int(remaining // pair_cost)
+ # Below three pairs a median is not outvoting anything, so there is nothing worth
+ # spending the time on: the sample already taken is the verdict, and it says so.
+ assert affordable >= 3, (
+ f"{label} path is not linear: {factor}x the input cost {ratio:.1f}x the time over 3 "
+ f"pairs (linear is ~{factor}, quadratic is ~{factor ** 2}), and at {big:.1f}s per big "
+ f"leg a second sample does not fit in the {remaining:.0f}s left of {total_budget_s:.0f}s, "
+ "so this reading stands"
+ )
+ pairs_wanted = min(repeats_on_retry, affordable)
+ confirm, big_again, result, pairs = growth(
+ run,
+ build,
+ units,
+ factor,
+ repeats = pairs_wanted,
+ abort_over_s = budget,
+ budget_s = remaining,
+ clock = clock,
+ )
+ # The confirmation stands on its OWN reading, not on min(big, big_again). Taking the
+ # better of the two samples hid the case that matters: the retry aborts after one pair
+ # because its big leg blew the budget, and that same pair's small leg was slow enough
+ # to put the ratio under the bar, so a superlinear path passed on a sample that never
+ # finished. Both halves are now required of the confirmation itself.
+ assert (
+ big_again < budget
+ ), f"{label} path took {big_again:.1f}s on {units * factor} units while re-measuring"
+ assert pairs == pairs_wanted, (
+ f"{label} path: the confirmation stopped after {pairs} of {pairs_wanted} pairs, "
+ f"either on a big leg over {budget:.0f}s or on the {remaining:.0f}s the whole "
+ "sample is allowed, so its ratio is not a reading of anything"
+ )
+ assert confirm < tolerance, (
+ f"{label} path is not linear: {factor}x the input cost {confirm:.1f}x the time "
+ f"over {pairs_wanted} pairs, after {ratio:.1f}x over 3 "
+ f"(linear is ~{factor}, quadratic is ~{factor ** 2})"
+ )
return result
diff --git a/tests/test_growth_ratio_guard.py b/tests/test_growth_ratio_guard.py
new file mode 100644
index 00000000000..b877e0ec178
--- /dev/null
+++ b/tests/test_growth_ratio_guard.py
@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""The linearity guard four tests hand their verdict to, checked on paths whose cost is known.
+
+`assert_linear` decides `test_update_release_notes.py`, `test_tool_xml_strip.py`,
+`test_rag_store.py` and `test_pr5624_regressions.py`. It is a timing guard, so both of its
+failure modes are silent: too loose and a quadratic path ships, too tight and an unrelated
+branch goes red for someone else's load. tests/_shared/growth.py's own docstring records the
+second one happening three times, and it happened again on unslothai/unsloth#11152 at 7.2x
+after the paired-median form was supposed to have ended it.
+
+So the predicate gets rows of its own, the same way `test_cache_budget_discipline.py` and
+`test_windows_amd_gpu_scan_fallback.py` test the readers their guards hand the verdict to.
+
+Cost is DECLARED, not spent. Every row drives `growth` with a fake clock that advances by
+exactly what the shape says, so a "quadratic" path here is quadratic to the last digit and
+nothing depends on the scheduler. Sleeping for these shapes instead would have made a test
+of a flakiness guard flaky in the same way the guard was, which is the failure being fixed
+rather than a way to check it: a nominal 4 ms leg that the runner stretches to 7 ms moves
+the ratio across the bar and the row means nothing.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from growth import assert_linear, growth # tests/_shared, on sys.path via tests/conftest.py
+
+
+class FakeClock:
+ """A `perf_counter` that only moves when something says how much time it took.
+
+ `growth` reads the clock either side of `run`, so a `run` that advances this by the
+ cost of its input makes the measured elapsed exactly that cost.
+ """
+
+ def __init__(self):
+ self.now = 1000.0
+
+ def __call__(self) -> float:
+ return self.now
+
+ def advance(self, seconds: float) -> None:
+ self.now += seconds
+
+
+#: Seconds per unit of cost. Small enough that a quadratic row stays well under the 60s
+#: backstop `assert_linear` applies, so the quadratic rows fail on the RATIO, which is what
+#: they are about, rather than on the backstop, which has its own row.
+_UNIT = 0.001
+
+
+def _shaped(exponent: float, *, stalls: dict[int, float] = None):
+ """A `run` costing `_UNIT * len(text) ** exponent` seconds, plus a stall on chosen calls.
+
+ `stalls` maps a call index (0-based, counting every leg in order) to extra seconds, which
+ is how contention is stated exactly: the real thing lands inside one leg of one pair, and
+ because the big leg runs `factor`x longer it is the one that usually catches it.
+ """
+ stalls = stalls or {}
+ clock = FakeClock()
+ calls = {"n": 0}
+
+ def run(text: str) -> str:
+ index = calls["n"]
+ calls["n"] += 1
+ clock.advance(_UNIT * len(text) ** exponent + stalls.get(index, 0.0))
+ return text
+
+ return run, clock, calls
+
+
+def _build(n: int) -> str:
+ return "x" * n
+
+
+def test_a_linear_path_passes():
+ """The negative control. Without it every row below could pass by the guard never accepting."""
+ run, clock, _ = _shaped(1.0)
+ assert assert_linear(run, _build, "linear", 2, clock = clock) == _build(8)
+
+
+def test_a_quadratic_path_fails():
+ """The positive control: the shape these guards exist to catch."""
+ run, clock, _ = _shaped(2.0)
+ with pytest.raises(AssertionError, match = "is not linear"):
+ assert_linear(run, _build, "quadratic", 2, clock = clock)
+
+
+def test_a_stall_in_the_first_sample_does_not_fail_a_linear_path():
+ """#11152's failure, reproduced exactly and then not reproduced.
+
+ Three pairs is few enough that two stalled big legs move the median over the bar. The
+ stalls are placed on call indices 1 and 3, the big legs of the first two pairs, and are
+ sized so the first median reads well above 6. A linear path must still pass, because
+ the re-measurement is taken on a sample with no stall in it.
+ """
+ run, clock, calls = _shaped(1.0, stalls = {1: 40.0, 3: 40.0})
+ assert assert_linear(run, _build, "stalled but linear", 2, clock = clock) == _build(8)
+ # It really did take the retry: three pairs is six legs, so anything past six is the
+ # second sample. A row that passed on the first reading would prove nothing about it.
+ assert calls["n"] > 6, "the first sample did not trip the bar, so the retry was not exercised"
+
+
+def test_the_retry_is_not_a_second_chance_for_a_quadratic_path():
+ """The half that must NOT be weakened: re-measuring may only forgive noise, not a shape.
+
+ Same stalls as the row above, on a genuinely quadratic path. Every pair of a quadratic
+ path measures ~`factor ** 2`, so the larger second sample has to come back over the bar
+ too. If it does not, the retry has turned the guard off.
+ """
+ run, clock, _ = _shaped(2.0, stalls = {1: 40.0, 3: 40.0})
+ with pytest.raises(AssertionError, match = "is not linear"):
+ assert_linear(run, _build, "stalled and quadratic", 2, clock = clock)
+
+
+def test_the_failure_message_names_both_samples():
+ """A red run has to say it was measured twice, or the next person re-litigates the retry."""
+ run, clock, _ = _shaped(2.0)
+ with pytest.raises(AssertionError) as excinfo:
+ assert_linear(run, _build, "quadratic", 2, clock = clock)
+ message = str(excinfo.value)
+ assert "pairs, after" in message, message
+ assert "quadratic is ~16" in message, message
+
+
+def test_growth_reports_the_best_big_time_not_the_worst():
+ """`assert_linear`'s 60s backstop reads this, and one stalled run must not trip it.
+
+ Contention only adds, so the minimum big leg is the closest that size got to its own
+ cost. A backstop reading the worst would fire on a runner that stalled once.
+ """
+ run, clock, _ = _shaped(1.0, stalls = {1: 50.0})
+ _, big, _, _ = growth(run, _build, 2, repeats = 3, clock = clock)
+ assert big == pytest.approx(_UNIT * 8), f"the stalled leg was the big time: {big}"
+
+
+def test_a_path_slow_enough_to_trip_the_backstop_fails_on_the_backstop():
+ """The other arm of the backstop: too slow to measure still has to fail, and say so."""
+ run, clock, _ = _shaped(1.0, stalls = {1: 120.0})
+ with pytest.raises(AssertionError, match = "path took"):
+ assert_linear(run, _build, "glacial", 2, clock = clock)
+
+
+def test_the_first_sample_is_inside_the_budget_too():
+ """The budget covers the whole call, and the first sample is part of it.
+
+ The case is contention landing in the SMALL legs: 50s small against 55s big reads as a
+ ratio of 1.1, comfortably under the tolerance, so nothing reaches the retry branch where
+ the budget used to be the only one enforced. Three of those pairs is 315s, past the
+ `--timeout=330` these CI invocations pass, and the old form returned a cheerful pass
+ from a sample that had eaten the whole run.
+
+ Stopping is not passing either: two pairs is not the reading the caller asked for, so it
+ fails saying so rather than reporting a median of whatever it managed.
+ """
+ stalls = {}
+ for index in range(6):
+ stalls[index] = 49.998 if index % 2 == 0 else 54.992
+ run, clock, calls = _shaped(1.0, stalls = stalls)
+ with pytest.raises(AssertionError, match = "first sample stopped after 2 of 3"):
+ assert_linear(run, _build, "slow on both legs", 2, clock = clock)
+ assert calls["n"] == 4, f"a pair that could not fit was started anyway: {calls['n']}"
+ assert clock.now - 1000.0 < 330.0, f"the call outlasted the runner: {clock.now - 1000.0}"
+
+
+def test_a_confirmation_that_would_outlast_the_job_is_not_started():
+ """The retry must not cost more than the runner will wait, or the message is a timeout.
+
+ Seven pairs of a big leg just under the 60s backstop is ~420s, and the four pytest
+ invocations in .github/workflows/studio-backend-ci.yml pass `--timeout=330`: the worker
+ is killed mid-confirmation and reports a bare timeout, so the shape that was measured
+ never reaches anyone. Shaped here as a big leg of 40s with a ratio over the bar, which
+ is the regression this describes and not a contended runner. The row pins BOTH halves:
+ it fails as a linearity failure naming the first sample, and it does not pay for a
+ second one -- three pairs is six legs, and nothing past six may run.
+ """
+ run, clock, calls = _shaped(1.0, stalls = {1: 40.0, 3: 40.0, 5: 40.0})
+ with pytest.raises(AssertionError, match = "does not fit in"):
+ assert_linear(run, _build, "slow and superlinear", 2, clock = clock)
+ assert calls["n"] == 6, f"a confirmation it cannot afford was started anyway: {calls['n']}"
+
+
+def test_a_confirmation_it_can_only_partly_afford_is_still_taken():
+ """The other arm: short of seven pairs is a weaker vote, not a reason to skip the retry.
+
+ Big legs of 30s put 90s of the 240s total into the first sample, so 150s is left and a
+ 30s pair affords four of them, not seven. The path is linear once the stalls stop, so it
+ must still pass -- on four pairs, rather than on the ratio being forgiven. Without this
+ the row above could be satisfied by refusing every confirmation that is not free.
+ """
+ run, clock, calls = _shaped(1.0, stalls = {1: 30.0, 3: 30.0, 5: 30.0})
+ assert assert_linear(run, _build, "affordable in part", 2, clock = clock) == _build(8)
+ # Six legs of the first sample, then four pairs rather than seven.
+ assert calls["n"] == 6 + 8, f"the confirmation ran {(calls['n'] - 6) // 2} pairs"
+
+
+def test_a_confirmation_sized_from_a_mixed_estimate_still_stops_at_the_budget():
+ """Sizing the retry from the first sample is an estimate; the budget has to be measured.
+
+ The estimate reads the BEST big leg and the MEDIAN ratio, and those come from different
+ pairs. A first sample of 59s, 59s and 1s big legs against 1s small legs therefore offers
+ a one-second pair cost alongside a ratio of 59, and authorises all seven pairs. If the
+ confirmation's own legs then arrive at 40s, seven of them is about 290s, which on top of
+ the first sample is past the `--timeout=330` those CI invocations pass.
+
+ So the elapsed deadline inside the sample is what stops it, not the prediction made
+ before it started: the loop gives up after two of the seven pairs it was authorised and
+ the run fails with a sentence about an unfinished confirmation rather than with the
+ runner's timeout.
+ """
+ stalls = {0: 0.998, 1: 58.992, 2: 0.998, 3: 58.992, 4: 0.998, 5: 0.992}
+ for index in range(6, 6 + 2 * 7):
+ stalls[index] = 0.998 if index % 2 == 0 else 39.992
+ run, clock, calls = _shaped(1.0, stalls = stalls)
+ with pytest.raises(AssertionError, match = "confirmation stopped after"):
+ assert_linear(run, _build, "mixed estimate", 2, clock = clock)
+ assert calls["n"] == 6 + 4, f"the confirmation ran on past its budget: {calls['n']}"
+
+
+def test_the_budget_is_asked_before_a_pair_rather_than_after_it():
+ """A bound tested once a pair is already home is not a bound on that pair.
+
+ First-sample big legs of 13s, 20s and 20s against 1s small legs cost 56s, so 184s of the
+ 240s total is left, and at the sizing's estimated pair cost that authorises all seven.
+ The confirmation's own legs then arrive at 59s, just inside the per-leg backstop.
+
+ Asked AFTER each pair, the fourth one runs: 177s elapsed is under 184 at the moment the
+ third finishes, so the sample reaches 236s and blows a bound it was never tested against
+ while inside it. Reserving room for one more pair at the cost of the worst seen stops it
+ at three, for 177s, and the whole test lands at 233s.
+ """
+ stalls = {0: 0.998, 1: 12.992, 2: 0.998, 3: 19.992, 4: 0.998, 5: 19.992}
+ for index in range(6, 6 + 2 * 7):
+ stalls[index] = 0.0 if index % 2 == 0 else 58.992
+ run, clock, calls = _shaped(1.0, stalls = stalls)
+ with pytest.raises(AssertionError, match = "confirmation stopped after 3 of 7"):
+ assert_linear(run, _build, "pair that would overrun", 2, clock = clock)
+ assert calls["n"] == 6 + 6, f"a pair that could not fit was started anyway: {calls['n']}"
+ assert (
+ clock.now - 1000.0 < 330.0
+ ), f"the whole test would outlast the runner: {clock.now - 1000.0}"
+
+
+def test_an_aborted_confirmation_is_not_accepted_as_one():
+ """A confirmation that stopped after one pair is not a reading, however low its ratio.
+
+ The shape: the first sample is over the bar, then the retry's first pair has BOTH legs
+ stalled -- the big one past the 60s backstop, which cuts the loop, and the small one
+ enough to drag that pair's ratio under the tolerance. Reading the median of that one
+ ratio, and taking the better of the two samples' big times, accepted a superlinear path
+ on a sample that never finished. The stalls here are sized so the surviving ratio really
+ is under 6, so the row fails for the abort and not for the ratio.
+ """
+ run, clock, _ = _shaped(2.0, stalls = {1: 40.0, 3: 40.0, 6: 30.0, 7: 70.0})
+ with pytest.raises(AssertionError, match = "while re-measuring|confirmation stopped after"):
+ assert_linear(run, _build, "aborted confirmation", 2, clock = clock)