Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
21.8 KB
Raw Blame
"""Tuning job contract tests.

The measurements are injected, which is the point: these tests are about the *contract* — which
dimensions a backend may search, what happens to a candidate that fails, how a winner is chosen,
and what the job refuses to do. A GPU is not needed and, more importantly, a hardcoded benchmark
number can never leak into the module under test.

The negative cases carry the weight: a sweep must never promote a candidate the engine could not
run, never pick a winner from a machine that drifted, and never recommend a change on evidence it
does not have.
"""

from __future__ import annotations

import json
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from strata_control import tuning  # noqa: E402


class TuningCase(unittest.TestCase):
    def setUp(self):
        self.tmp = Path(tempfile.mkdtemp(prefix="strata-tune-"))

    def tearDown(self):
        shutil.rmtree(self.tmp, ignore_errors=True)

    def plan(self, backend="cuda", objective="generation", hold=(), max_trials=16, repeats=3,
             flags=None, dimensions=()):
        return tuning.TuningPlan(
            id="job1", backend=backend, engine="/nonexistent/strata",
            engine_flags_known=flags is not None, model="qwen3.8-flash-next",
            profile_revision="abc123", objective=objective,
            workload={"prompt_tokens": 512, "max_new": 200, "seed": 1},
            dimensions=[{"id": d} for d in dimensions], excluded=[], fixed={},
            max_trials=max_trials, total_time_s=60.0,
            repeats=repeats, devices=["0"],
        )

    def run_job(self, plan, measure):
        tuner = tuning.Tuner(self.tmp / "runs", plan, measure)
        return tuner, tuner.run()


class TestDimensionValidation(TuningCase):
    """A backend is offered only the levers its engine can actually accept."""

    def test_metal_is_not_offered_nvidia_only_dimensions(self):
        offered, excluded = tuning.validate_dimensions("metal")
        ids = {d.id for d in offered}
        self.assertNotIn("vram_reserve", ids)
        self.assertNotIn("pcie_frac", ids)
        reasons = {e["id"]: e["reason"] for e in excluded}
        self.assertIn("vram_reserve", reasons)
        self.assertIn("NVIDIA/AMD", reasons["vram_reserve"])

    def test_cuda_is_offered_the_nvidia_dimensions(self):
        offered, _ = tuning.validate_dimensions("cuda")
        ids = {d.id for d in offered}
        self.assertTrue({"vram_reserve", "pcie_frac", "expert_cache"} <= ids)

    def test_a_flag_the_binary_does_not_accept_excludes_the_dimension(self):
        """The static table is not trusted: the actual binary's flag list decides."""
        offered, excluded = tuning.validate_dimensions("cuda", engine_flags=["--kv", "--ple-io"])
        ids = {d.id for d in offered}
        self.assertNotIn("vram_reserve", ids, "a dimension whose flag the binary lacks must be excluded")
        reasons = {e["id"]: e["reason"] for e in excluded}
        self.assertIn("does not accept --vram-reserve-mib", reasons["vram_reserve"])

    def test_a_flag_the_binary_does_accept_keeps_the_dimension(self):
        offered, _ = tuning.validate_dimensions("cuda", engine_flags=["--kv", "--vram-reserve-mib"])
        self.assertIn("vram_reserve", {d.id for d in offered})

    def test_benchmark_only_dimension_is_never_exported(self):
        tuner = tuning.Tuner(self.tmp / "runs", self.plan(), lambda c: {"tokens_per_s": 1.0})
        args = tuner._args_for({"prompt_cache_tail": True, "kv": "int8"})
        self.assertNotIn("--prompt-cache-tail", args,
                         "a flag the production interface does not accept must not be exported")
        self.assertEqual(args["--kv"], "int8")

    def test_flag_inventory_returns_none_for_a_missing_engine(self):
        self.assertIsNone(tuning.engine_flag_inventory(Path("/nonexistent/strata")))

    def test_flag_inventory_reads_stderr_as_well_as_stdout(self):
        """The Strata engine writes its usage to stderr.

        A stdout-only read reports "no flags" for a working engine, and because every dimension is
        then excluded the sweep silently searches nothing — safe, but useless and very hard to
        notice. This test uses a stand-in that prints usage to stderr only.
        """
        fake = self.tmp / "fake-engine.py"      # run as [sys.executable, fake]: works on every OS
        fake.write_text("import sys\nfor line in ('usage:', '  --kv KV', '  --ple-io MODE'):\n    print(line, file=sys.stderr)\n")
        flags = tuning.engine_flag_inventory(fake)
        self.assertIsNotNone(flags, "usage on stderr must still be read")
        self.assertIn("--kv", flags)
        self.assertIn("--ple-io", flags)

    def test_flag_inventory_reads_stdout_too(self):
        fake = self.tmp / "fake-engine-stdout.py"
        fake.write_text("print('  --vram-reserve-mib MIB')\n")
        self.assertIn("--vram-reserve-mib", tuning.engine_flag_inventory(fake))

    def test_a_backend_with_dimensions_that_all_need_missing_flags_searches_nothing(self):
        """The safe outcome, asserted so it cannot silently become an unsafe one.

        Every dimension has *some* reason: either its flags are not this backend's concept, or the
        binary does not accept them. What must never happen is a dimension with no reason being
        dropped silently.
        """
        offered, excluded = tuning.validate_dimensions("metal", engine_flags=["--unrelated"])
        self.assertEqual(offered, [])
        self.assertEqual(len(excluded), len(tuning.DIMENSIONS))
        for e in excluded:
            self.assertTrue(e["reason"].strip(), f"{e['id']} was excluded with no reason given")
        # The dimensions that ARE Metal concepts are excluded specifically because the binary lacks
        # their flags, which is the check that matters.
        by_id = {e["id"]: e["reason"] for e in excluded}
        for dim_id in ("kv", "ple_io", "kv_grow", "spec_min_p"):
            self.assertIn("does not accept", by_id[dim_id])


class TestBaselineAndSelection(TuningCase):
    def test_a_clear_winner_is_recommended(self):
        calls = {"n": 0}

        def measure(candidate):
            calls["n"] += 1
            # The baseline (no settings) is slow; any candidate is faster.
            return {"tokens_per_s": 50.0 if not candidate.settings else 80.0, "correct": True}

        plan = self.plan(max_trials=2, dimensions=("kv",))
        tuner, result = self.run_job(plan, measure)
        rep = result["report"]
        self.assertTrue(rep["recommendation_available"], rep)
        self.assertEqual(rep["baseline"]["median"], 50.0)
        self.assertEqual(rep["recommendation"]["median"], 80.0)

    def test_a_faster_candidate_inside_the_baseline_spread_is_not_recommended(self):
        """Promoting noise is the classic tuning failure, so the gate compares worst-against-worst."""
        state = {"phase": 0}

        def measure(candidate):
            if candidate.is_baseline:
                # A wide baseline spread: worst 40, best 90.
                state["phase"] += 1
                return {"tokens_per_s": [40.0, 90.0, 65.0][state["phase"] % 3]}
            # The candidate's median is higher but its worst run is below the baseline's worst.
            return {"tokens_per_s": [45.0, 60.0, 60.0][state["phase"] % 3]}

        plan = self.plan(max_trials=1, repeats=3, dimensions=("kv",))
        _, result = self.run_job(plan, measure)
        rep = result["report"]
        self.assertFalse(rep["recommendation_available"],
                         "a candidate whose worst run is below the baseline's worst must not win")
        self.assertIn("spread", rep["reason"])

    def test_no_candidate_better_means_recommend_no_change(self):
        def measure(candidate):
            return {"tokens_per_s": 60.0 if not candidate.settings else 55.0}

        _, result = self.run_job(self.plan(max_trials=2, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertFalse(rep["recommendation_available"])
        self.assertIn("keeping the current configuration", rep["reason"])

    def test_drift_is_reported(self):
        """The baseline is re-measured last, so a drifting machine is visible instead of silent.

        The baseline is measured once as a warmup plus `repeats` times, and then again after the
        sweep. This case makes the measured value a function of *which* baseline measurement it is,
        rather than of the call count, so the expectation does not depend on how repeats are
        scheduled.
        """
        state = {"baseline_measurements": 0}

        def measure(candidate):
            if candidate.kind == "baseline":
                state["baseline_measurements"] += 1
                # The first measurement is the warmup; the three counted repeats are 50, and the
                # recheck afterwards is 40 (below), simulating a machine that got slower.
                return {"tokens_per_s": 50.0}
            if candidate.kind == "recheck":
                return {"tokens_per_s": 40.0}
            return {"tokens_per_s": 45.0}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertEqual(rep["baseline"]["median"], 50.0)
        self.assertIsNotNone(rep["recheck"], "the baseline must be re-measured after the sweep")
        self.assertEqual(rep["recheck"]["median"], 40.0)
        self.assertIsNotNone(rep["drift"], "the drift between the two baseline measurements is reported")
        self.assertLess(rep["drift"]["relative"], 0,
                        "the report must show that the machine slowed, not hide it")

    def test_fewer_repeats_than_required_disqualifies_a_candidate(self):
        def measure(candidate):
            return {"tokens_per_s": 90.0 if candidate.settings else 50.0}

        plan = self.plan(max_trials=1, repeats=5, dimensions=("kv",))
        _, result = self.run_job(plan, measure)
        best = [c for c in result["report"]["candidates"] if c["kind"] == "trial"][0]
        self.assertGreaterEqual(len(best["metrics"]), 5)


class TestFailuresAreRecorded(TuningCase):
    """A candidate that could not run is evidence about the configuration, never a slow winner."""

    def test_a_boot_failure_is_recorded_and_excluded(self):
        def measure(candidate):
            if candidate.settings:
                raise RuntimeError("the engine failed to boot with these flags")
            return {"tokens_per_s": 50.0}

        _, result = self.run_job(self.plan(max_trials=2, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertEqual(rep["recommendation_available"], False)
        self.assertTrue(rep["failures"])
        self.assertEqual(rep["failures"][0]["status"], "FAIL_BOOT")
        self.assertIn("failed to boot", rep["failures"][0]["reason"])

    def test_a_timeout_is_recorded_as_a_timeout(self):
        def measure(candidate):
            if candidate.settings:
                raise RuntimeError("the measurement timed out after 120s")
            return {"tokens_per_s": 50.0}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        self.assertEqual(result["report"]["failures"][0]["status"], "TIMEOUT")

    def test_no_tokens_is_not_a_slow_configuration(self):
        def measure(candidate):
            return {"tokens_per_s": 0.0}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertFalse(rep["recommendation_available"])
        statuses = [c["status"] for c in rep["candidates"]]
        self.assertIn("NO_TOKENS", statuses)

    def test_a_correctness_failure_disqualifies_a_fast_candidate(self):
        """tokens/s alone is not acceptance."""
        def measure(candidate):
            if candidate.settings:
                return {"tokens_per_s": 500.0, "correct": False}
            return {"tokens_per_s": 50.0, "correct": True}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertFalse(rep["recommendation_available"],
                         "a fast but incorrect configuration must never be recommended")
        self.assertTrue(any(f["status"] == "INVALID" for f in rep["failures"]), rep["failures"])

    def test_a_baseline_that_cannot_be_measured_stops_the_sweep(self):
        def measure(candidate):
            raise RuntimeError("no engine here")

        _, result = self.run_job(self.plan(max_trials=5, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertFalse(rep["recommendation_available"])
        self.assertIsNone(rep["baseline"])
        self.assertIn("could not be measured", rep["reason"])


class TestBounds(TuningCase):
    def test_the_trial_limit_is_respected(self):
        # Count *trials*, not measurement calls: each candidate is measured once as a warmup plus
        # `repeats` times, so counting calls would measure the wrong thing entirely.
        plan = self.plan(max_trials=3, dimensions=("ple_io", "kv"))
        _, result = self.run_job(plan, lambda c: {"tokens_per_s": 60.0})
        self.assertEqual(result["report"]["trials_run"], 3)
        self.assertEqual(len([c for c in result["report"]["candidates"]
                              if c["kind"] == "trial"]), 3)
        self.assertLessEqual(result["report"]["baseline"] is not None, True)

    def test_the_time_bound_stops_new_trials(self):
        clock = {"t": 0.0}

        def measure(candidate):
            clock["t"] += 30.0
            return {"tokens_per_s": 60.0}

        plan = self.plan(max_trials=100, dimensions=("kv",))
        plan.total_time_s = 45.0
        tuner = tuning.Tuner(self.tmp / "runs", plan, measure, clock=lambda: clock["t"])
        result = tuner.run()
        self.assertTrue(any(e["state"] == "time_limit" for e in result["events"]),
                        "the displayed time bound must actually stop the sweep")

    def test_cancellation_keeps_evidence_and_recommends_nothing(self):
        def measure(candidate):
            if candidate.settings:
                tuner.request_cancel()
            return {"tokens_per_s": 90.0 if candidate.settings else 50.0}

        plan = self.plan(max_trials=10, dimensions=("kv",))
        tuner = tuning.Tuner(self.tmp / "runs", plan, measure)
        result = tuner.run()
        self.assertEqual(result["state"], "cancelled")
        self.assertFalse(result["report"]["recommendation_available"],
                         "a cancelled sweep must not produce a winner")
        self.assertTrue(result["report"]["candidates"], "evidence collected before the cancel is kept")


class TestSearchSpaceIsDisclosed(TuningCase):
    """A bounded sweep must not look like a complete one.

    This is the honesty property that decides whether "no candidate beat the baseline" can be read
    as a conclusion. With four trials out of ninety-six combinations, it cannot — and the report has
    to say so, or a reader will take a partial search for a full one.
    """

    def test_a_bounded_sweep_reports_the_size_of_the_space_it_sampled(self):
        plan = self.plan(max_trials=2, dimensions=("kv",))
        _, result = self.run_job(plan, lambda c: {"tokens_per_s": 60.0})
        rep = result["report"]
        self.assertEqual(rep["trials_run"], 2)
        # `kv` has four values, so the space is four and two trials is a prefix of it.
        self.assertEqual(rep["combinations_total"], 4)
        self.assertTrue(rep["trials_are_a_prefix"])

    def test_a_complete_sweep_is_not_labelled_a_prefix(self):
        plan = self.plan(max_trials=10, dimensions=("kv",))
        _, result = self.run_job(plan, lambda c: {"tokens_per_s": 60.0})
        rep = result["report"]
        self.assertEqual(rep["combinations_total"], 4)
        self.assertEqual(rep["trials_run"], 4, "the whole four-value space is enumerated")
        self.assertFalse(rep["trials_are_a_prefix"])

    def test_the_space_size_accounts_for_every_offered_dimension(self):
        plan = self.plan(max_trials=1, dimensions=("kv", "kv_grow"))
        _, result = self.run_job(plan, lambda c: {"tokens_per_s": 60.0})
        # kv has 4 values, kv_grow has 2 -> 8 combinations.
        self.assertEqual(result["report"]["combinations_total"], 8)

    def test_a_plan_with_no_dimensions_reports_no_space(self):
        _, result = self.run_job(self.plan(max_trials=1), lambda c: {"tokens_per_s": 60.0})
        self.assertEqual(result["report"]["combinations_total"], 0)


class TestDriftIsGraded(TuningCase):
    """A drift note is only warranted when the drift is material.

    Printing "the comparison is weaker than the repeat count suggests" at +0.0% trains the reader to
    ignore the warning, which is worse than not printing it.
    """

    def test_negligible_drift_is_not_flagged_as_material(self):
        def measure(candidate):
            return {"tokens_per_s": 50.0}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertIsNotNone(rep["drift"])
        self.assertAlmostEqual(rep["drift"]["relative"], 0.0)
        self.assertFalse(rep["drift_material"])

    def test_material_drift_is_flagged(self):
        def measure(candidate):
            return {"tokens_per_s": 40.0 if candidate.kind == "recheck" else 50.0}

        _, result = self.run_job(self.plan(max_trials=1, dimensions=("kv",)), measure)
        rep = result["report"]
        self.assertAlmostEqual(rep["drift"]["relative"], -0.2)
        self.assertTrue(rep["drift_material"])


class TestExclusiveLease(TuningCase):
    """Tuning must own its devices: a second job, or a serving load, must not share them."""

    def test_a_second_job_is_refused_while_the_lease_is_held(self):
        first = tuning.Tuner(self.tmp / "runs", self.plan(), lambda c: {"tokens_per_s": 50.0})
        first._acquire_lease()
        try:
            second = tuning.Tuner(self.tmp / "runs", self.plan(), lambda c: {"tokens_per_s": 50.0})
            with self.assertRaises(RuntimeError) as ctx:
                second._acquire_lease()
            self.assertIn("exclusive device lease", str(ctx.exception))
        finally:
            first._release_lease()

    def test_the_lease_is_released_even_when_the_job_fails(self):
        plan = self.plan()
        tuner = tuning.Tuner(self.tmp / "runs", plan, lambda c: (_ for _ in ()).throw(RuntimeError("boom")))
        tuner.run()
        self.assertFalse((tuner.root / "tuning.lease").exists(),
                         "a failed job must not leave the devices leased")

    def test_a_stale_lease_is_reported_rather_than_ignored(self):
        stale = self.tmp / "runs" / "job1"
        stale.mkdir(parents=True)
        (stale / "tuning.lease").write_text(json.dumps({"id": "dead", "pid": 1}))
        tuner = tuning.Tuner(self.tmp / "runs", self.plan(), lambda c: {"tokens_per_s": 50.0})
        with self.assertRaises(RuntimeError) as ctx:
            tuner._acquire_lease()
        self.assertIn("dead", str(ctx.exception))


class TestExportedRetest(TuningCase):
    """A winner is not qualified until the *exported* configuration is measured again."""

    def test_a_configuration_that_loses_its_advantage_on_export_is_withdrawn(self):
        def measure(args):
            # The benchmark-only lever carried the gain; without it the engine is at the baseline.
            return {"tokens_per_s": 90.0 if "--prompt-cache-tail" in args else 50.0}

        outcome = tuning.verify_exported(Path("/engine"), {"--kv": "int8"}, measure)
        self.assertTrue(outcome["verified"])
        self.assertEqual(outcome["median"], 50.0)

    def test_an_exported_configuration_that_fails_to_boot_is_not_verified(self):
        def measure(args):
            raise RuntimeError("the engine failed to boot")

        outcome = tuning.verify_exported(Path("/engine"), {}, measure)
        self.assertFalse(outcome["verified"])
        self.assertIn("failed to run", outcome["reason"])

    def test_an_incorrect_exported_configuration_is_not_verified(self):
        outcome = tuning.verify_exported(Path("/engine"), {}, lambda a: {"tokens_per_s": 100.0,
                                                                        "correct": False})
        self.assertFalse(outcome["verified"])
        self.assertIn("correctness", outcome["reason"])


class TestPersistenceAndReporting(TuningCase):
    def test_the_job_is_reloadable_with_its_report(self):
        def measure(candidate):
            return {"tokens_per_s": 70.0 if candidate.settings else 50.0}

        plan = self.plan(max_trials=1, dimensions=("kv",))
        _, result = self.run_job(plan, measure)
        saved = json.loads((self.tmp / "runs" / "job1" / "job.json").read_text())
        self.assertEqual(saved["state"], "completed")
        self.assertEqual(saved["id"], "job1")
        self.assertTrue(saved["report"]["candidates"])
        self.assertIn("excluded_dimensions", saved["report"])
        self.assertIn("machine", json.dumps(tuning.machine_identity()))

    def test_the_report_names_what_was_excluded_and_why(self):
        plan = tuning.plan_tuning(Path("/nonexistent/strata"), "metal", "m", "generation",
                                  {"prompt_tokens": 128})
        self.assertIn("vram_reserve", {e["id"] for e in plan.excluded})
        self.assertFalse(plan.engine_flags_known,
                         "a plan must record that the engine could not be interrogated")

    def test_objective_selects_the_metric(self):
        self.assertEqual(self.plan(objective="generation").__class__.__name__, "TuningPlan")
        tuner = tuning.Tuner(self.tmp / "runs", self.plan(objective="prefill"),
                             lambda c: {"tokens_per_s": 1.0})
        self.assertEqual(tuner._objective_metric(), "prefill_tokens_per_s")
        tuner2 = tuning.Tuner(self.tmp / "runs2", self.plan(objective="latency"),
                              lambda c: {"tokens_per_s": 1.0})
        self.assertEqual(tuner2._objective_metric(), "ttft_ms")


if __name__ == "__main__":
    unittest.main()