Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++
11.7 KB
Raw Blame
"""Tuning history tests (checklist T06).

The requirement is that an engine, model, driver, workload or context change invalidates incompatible
history, and that model identity is more than a name. The risk being tested is specific and quiet: a
report that mixes two configurations' numbers produces a confident recommendation from incomparable
measurements, and nothing about the output looks wrong.

So each element of the comparison key gets its own test, and the *reason* is asserted, not just the
boolean — a report has to be able to say "the model changed", not merely "no".
"""

from __future__ import annotations

import sys
import tempfile
import textwrap
import unittest
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from strata_control import tune_history as th  # noqa: E402


class HistoryCase(unittest.TestCase):
    def setUp(self):
        self.tmp = Path(tempfile.mkdtemp(prefix="strata-history-"))

    def tearDown(self):
        import shutil
        shutil.rmtree(self.tmp, ignore_errors=True)

    def store(self) -> th.HistoryStore:
        return th.HistoryStore(self.tmp / "history")

    def file(self, name: str, content: bytes) -> Path:
        p = self.tmp / name
        p.write_bytes(content)
        return p

    def key(self, **over):
        base = th.comparison_key(
            engine={"digest": "e" * 64, "bytes": 100, "exists": True, "method": th.FULL, "path": "/e"},
            model={"digest": "m" * 64, "bytes": 200, "exists": True, "method": th.SAMPLED, "path": "/m"},
            backend="metal",
            workload={"prompt_tokens": 512, "max_new": 128, "seed": 1, "temperature": 0.0},
            context=32768,
            machine={"system": "Darwin", "machine": "arm64", "cpu_count": 10},
        )
        base.update(over)
        return base


class TestFileIdentity(HistoryCase):
    """Identity is a content hash, not a label."""

    def test_a_different_file_with_the_same_name_is_a_different_identity(self):
        """The whole point: a re-quantised model keeps its name and changes its behaviour."""
        a = self.file("model.gguf", b"x" * 5000)
        first = th.file_identity(a)["digest"]
        a.write_bytes(b"y" * 5000)          # same path, same size, different bytes
        self.assertNotEqual(th.file_identity(a)["digest"], first)

    def test_a_truncated_file_is_a_different_identity(self):
        a = self.file("model.gguf", b"x" * 5000)
        full = th.file_identity(a, full=True)["digest"]
        a.write_bytes(b"x" * 4000)
        self.assertNotEqual(th.file_identity(a, full=True)["digest"], full)

    def test_the_identity_method_travels_with_the_digest(self):
        a = self.file("model.gguf", b"x" * 5000)
        self.assertEqual(th.file_identity(a)["method"], th.SAMPLED)
        self.assertEqual(th.file_identity(a, full=True)["method"], th.FULL)

    def test_a_large_file_is_sampled_without_reading_all_of_it(self):
        # The point of sampling is cost. This asserts the digest does not depend on every byte, so a
        # change strictly between the sampled regions is (correctly) not claimed as detection.
        big = self.tmp / "big.gguf"
        with big.open("wb") as fh:
            fh.write(b"a" * th.SAMPLE_BYTES)
            fh.write(b"b" * (4 * th.SAMPLE_BYTES))   # middle lands here
            fh.write(b"c" * th.SAMPLE_BYTES)
        first = th.file_identity(big)
        self.assertEqual(first["bytes"], 6 * th.SAMPLE_BYTES)
        self.assertIsNotNone(first["digest"])

    def test_a_missing_file_is_explicitly_absent_rather_than_hashed_as_empty(self):
        ident = th.file_identity(self.tmp / "nope")
        self.assertFalse(ident["exists"])
        self.assertIsNone(ident["digest"])
        self.assertIsNone(ident["method"])


class TestComparisonKey(HistoryCase):
    """Every element that makes two measurements incomparable is in the key."""

    def test_each_element_produces_its_own_reason(self):
        base = self.key()
        cases = {
            "engine_digest": "a different engine build",
            "model_digest": "different model bytes",
            "backend": "a different backend",
            "workload": "a different workload",
            "context": "a different context",
            "machine": "a different machine or driver",
        }
        for field, expected in cases.items():
            other = dict(base)
            other[field] = "changed" if not isinstance(base[field], (dict, int)) else 1
            reasons = th.incomparable_reasons(base, other)
            self.assertTrue(reasons, f"{field} must make the records incomparable")
            self.assertTrue(any(expected in r for r in reasons),
                            f"{field}: expected a reason mentioning {expected!r}, got {reasons}")

    def test_a_different_digest_method_is_incomparable(self):
        """Two records hashed differently cannot be compared as though they were the same."""
        base = self.key()
        other = dict(base, digest_method=th.FULL)
        reasons = th.incomparable_reasons(base, other)
        self.assertTrue(any("computed differently" in r for r in reasons), reasons)

    def test_an_unknown_identity_is_not_treated_as_equal(self):
        """An unknown is not evidence of sameness."""
        known = self.key()
        unknown = self.key(model_digest=None)
        self.assertTrue(th.incomparable_reasons(known, unknown))

    def test_identical_keys_are_comparable(self):
        self.assertEqual(th.incomparable_reasons(self.key(), self.key()), [])

    def test_a_different_hostname_is_the_same_machine(self):
        """A renamed machine is the same machine; the hostname is deliberately not in the key."""
        self.assertNotIn("node", th.machine_identity())
        self.assertNotIn("hostname", th.machine_identity())


class TestHistoryStore(HistoryCase):
    def test_a_recorded_sweep_is_comparable_to_itself(self):
        store = self.store()
        store.record(key=self.key(), outcome="accepted", recommendation_args={"--kv": "int8"},
                     median=24.0, baseline_median=20.0, trials_run=4, combinations_total=4)
        usable, unusable = store.comparable(self.key())
        self.assertEqual(len(usable), 1)
        self.assertEqual(usable[0].recommendation_args, {"--kv": "int8"})
        self.assertEqual(unusable, [])

    def test_a_changed_model_makes_the_history_incomparable_with_a_reason(self):
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        changed = self.key(model_digest="z" * 64)
        usable, unusable = store.comparable(changed)
        self.assertEqual(usable, [], "a different model's numbers must not be reused")
        self.assertEqual(len(unusable), 1)
        self.assertIn("different model bytes", unusable[0]["reasons"])

    def test_a_changed_engine_makes_the_history_incomparable(self):
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        usable, unusable = store.comparable(self.key(engine_digest="d" * 64))
        self.assertEqual(usable, [])
        self.assertIn("a different engine build", unusable[0]["reasons"])

    def test_a_changed_context_makes_the_history_incomparable(self):
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        usable, _ = store.comparable(self.key(context=131072))
        self.assertEqual(usable, [], "a different context is a different measurement")

    def test_a_changed_workload_makes_the_history_incomparable(self):
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        usable, _ = store.comparable(self.key(
            workload={"prompt_tokens": 1024, "max_new": 128, "seed": 1, "temperature": 0.0}))
        self.assertEqual(usable, [])

    def test_incomparable_history_is_retained_not_deleted(self):
        """A previous sweep may still answer a question, so the store does not discard it."""
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        store.record(key=self.key(model_digest="z" * 64), outcome="accepted")
        usable, unusable = store.comparable(self.key())
        self.assertEqual(len(usable), 1)
        self.assertEqual(len(unusable), 1)
        self.assertEqual(len(store.records()), 2, "both records are still there")

    def test_a_damaged_line_is_skipped_rather_than_fatal(self):
        """The store is a text file a human may have edited; one bad line must not lose the rest."""
        store = self.store()
        store.record(key=self.key(), outcome="accepted")
        with store.index.open("a", encoding="utf-8") as fh:
            fh.write("{ this is not json\n")
        store.record(key=self.key(), outcome="inconclusive")
        records = store.records()
        self.assertEqual(len(records), 2)
        self.assertEqual([r.outcome for r in records], ["accepted", "inconclusive"])

    def test_a_record_from_another_schema_version_is_ignored(self):
        store = self.store()
        store.root.mkdir(parents=True, exist_ok=True)
        store.index.write_text('{"schema_version": 99, "id": "old", "recorded_at": 1, "key": {}}\n',
                               encoding="utf-8")
        self.assertEqual(store.records(), [])

    def test_the_summary_says_what_the_numbers_sit_next_to(self):
        store = self.store()
        empty = th.summarise_history(store, self.key())
        self.assertEqual(empty["comparable_records"], 0)
        self.assertIn("no earlier sweep", empty["note"])

        store.record(key=self.key(), outcome="accepted")
        store.record(key=self.key(model_digest="z" * 64), outcome="accepted")
        summary = th.summarise_history(store, self.key())
        self.assertEqual(summary["comparable_records"], 1)
        self.assertEqual(summary["incomparable_records"], 1)
        self.assertTrue(summary["incomparable"][0]["reasons"])
        self.assertIn("this exact engine", summary["note"])


class TestPlanCarriesIdentity(HistoryCase):
    """The plan records what it measured, so a later sweep can compare without re-deriving it."""

    def test_plan_tuning_records_a_content_identity(self):
        from strata_control import tuning
        engine = self.file("strata", b"#!/bin/sh\necho 'x' >&2\n")
        model = self.file("model.gguf", b"weights" * 1000)
        plan = tuning.plan_tuning(
            engine, "metal", "fixture", "generation",
            {"prompt_tokens": 16, "max_new": 4, "seed": 1, "temperature": 0.0},
            hold=("ple_io", "kv", "kv_grow", "spec_min_p"), model_path=model, context=32768)
        ident = plan.identity
        self.assertEqual(ident["backend"], "metal")
        self.assertEqual(ident["context"], 32768)
        self.assertIsNotNone(ident["engine_digest"])
        self.assertIsNotNone(ident["model_digest"])
        self.assertEqual(ident["model_bytes"], len(b"weights" * 1000))

    def test_a_plan_without_a_model_path_is_incomparable_rather_than_falsely_comparable(self):
        """Not knowing the weights must be recorded as unknown, not as a match."""
        from strata_control import tuning
        engine = self.file("strata", b"#!/bin/sh\necho 'x' >&2\n")
        plan = tuning.plan_tuning(
            engine, "metal", "fixture", "generation",
            {"prompt_tokens": 16, "max_new": 4, "seed": 1, "temperature": 0.0},
            hold=("ple_io", "kv", "kv_grow", "spec_min_p"))
        self.assertIsNone(plan.identity["model_digest"])
        # And such a key matches nothing, including another key with the same unknown model.
        other = dict(plan.identity)
        self.assertTrue(th.incomparable_reasons(plan.identity, other) == [],
                        "two records with the same unknowns are consistently comparable to each other")
        self.assertTrue(th.incomparable_reasons(
            plan.identity, dict(plan.identity, model_digest="m" * 64)))


if __name__ == "__main__":
    unittest.main()