Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

30.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Snapshot/verify guard for tool-call parsing: AST inventory, golden outputs, patch-target routing."""

from __future__ import annotations

import ast
import hashlib
import importlib
import inspect
import json
import random
import re
import sys
from collections import Counter
from pathlib import Path

BACKEND_ROOT = Path(__file__).resolve().parents[2]
if str(BACKEND_ROOT) not in sys.path:
    sys.path.insert(0, str(BACKEND_ROOT))

BASELINE_DIR = BACKEND_ROOT / "tests" / "data" / "refactor_guard"

GUARDED_MODULES = {
    "core.tool_healing": BACKEND_ROOT / "core" / "tool_healing.py",
    "core.inference.tool_call_parser": BACKEND_ROOT / "core" / "inference" / "tool_call_parser.py",
    "core.inference.llama_cpp": BACKEND_ROOT / "core" / "inference" / "llama_cpp.py",
    "core.inference.inference": BACKEND_ROOT / "core" / "inference" / "inference.py",
    "core.inference.safetensors_agentic": (
        BACKEND_ROOT / "core" / "inference" / "safetensors_agentic.py"
    ),
}

# Must not import the inference stack, so llama_cpp is deliberately absent.
BEHAVIOUR_MODULES = ("core.tool_healing", "core.inference.tool_call_parser")


# Every re entry point carrying a pattern, not just compile.
_RE_CALLS = frozenset(
    {"compile", "match", "search", "fullmatch", "sub", "subn", "split", "findall", "finditer"}
)


def _compiled_patterns(mod_name: str) -> dict:
    """Read `.pattern` from live objects: AST call source pins only a constant's name, not its text."""
    module = importlib.import_module(mod_name)
    out = {}
    for name in dir(module):
        obj = getattr(module, name, None)
        for index, item in enumerate(obj if isinstance(obj, (list, tuple)) else [obj]):
            if isinstance(item, re.Pattern):
                key = name if not isinstance(obj, (list, tuple)) else f"{name}[{index}]"
                out[key] = f"{item.pattern!r} flags={item.flags}"
    return out


def _pattern_literals(tree: ast.Module) -> dict:
    """Regex literals are pinned on their own: a rewrite is the easiest silent change to stripping."""
    calls = []
    for node in ast.walk(tree):
        if (
            isinstance(node, ast.Call)
            and isinstance(node.func, ast.Attribute)
            and node.func.attr in _RE_CALLS
            and isinstance(node.func.value, ast.Name)
            and node.func.value.id == "re"
        ):
            calls.append(ast.unparse(node))
    counts = {}
    for call in calls:
        counts[call] = counts.get(call, 0) + 1
    return counts


def ast_inventory() -> dict:
    """Only two restructured modules pin signatures; the rest pin name and kind, to avoid churn."""
    inventory = {}
    for mod_name, path in GUARDED_MODULES.items():
        detailed = mod_name in BEHAVIOUR_MODULES
        tree = ast.parse(path.read_text(encoding = "utf-8"))
        symbols = {}
        for node in tree.body:
            if not detailed:
                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
                    symbols[node.name] = {
                        "kind": "class" if isinstance(node, ast.ClassDef) else "def"
                    }
                elif isinstance(node, ast.Assign):
                    for target in node.targets:
                        if isinstance(target, ast.Name):
                            symbols[target.id] = {"kind": "assign"}
                # An annotated global is an AnnAssign, not an Assign.
                elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
                    symbols[node.target.id] = {"kind": "assign"}
                continue
            if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
                symbols[node.name] = {
                    "kind": "async def" if isinstance(node, ast.AsyncFunctionDef) else "def",
                    "signature": ast.unparse(node.args),
                    "decorators": [ast.unparse(d) for d in node.decorator_list],
                }
            elif isinstance(node, ast.ClassDef):
                methods = {}
                for sub in node.body:
                    if isinstance(sub, (ast.FunctionDef, ast.AsyncFunctionDef)):
                        methods[sub.name] = {
                            "signature": ast.unparse(sub.args),
                            "decorators": [ast.unparse(d) for d in sub.decorator_list],
                        }
                symbols[node.name] = {
                    "kind": "class",
                    "bases": [ast.unparse(b) for b in node.bases],
                    "decorators": [ast.unparse(d) for d in node.decorator_list],
                    "methods": methods,
                }
            elif isinstance(node, ast.Assign):
                for target in node.targets:
                    if isinstance(target, ast.Name):
                        symbols[target.id] = {"kind": "assign"}
            elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name):
                symbols[node.target.id] = {"kind": "assign"}
        entry = {"symbols": symbols}
        if detailed:
            entry["regexes"] = _pattern_literals(tree)
            entry["compiled"] = _compiled_patterns(mod_name)
        inventory[mod_name] = entry
    return inventory


def runtime_inventory() -> dict:
    """``dir()`` plus signatures, which catches re-export aliasing the AST cannot see."""
    out = {}
    for mod_name in BEHAVIOUR_MODULES:
        module = importlib.import_module(mod_name)
        entry = {"dir": sorted(n for n in dir(module) if not n.startswith("__"))}
        signatures = {}
        for name in entry["dir"]:
            obj = getattr(module, name)
            if inspect.isfunction(obj):
                signatures[name] = f"{obj.__module__}.{obj.__qualname__}{inspect.signature(obj)}"
        entry["functions"] = signatures
        out[mod_name] = entry
    return out


_FRAGMENTS = (
    '<tool_call>{"name": "get_weather", "arguments": {"city": "Paris"}}</tool_call>',
    '<tool_call>{"name": "search", "arguments": {"q": "</tool_call> literal"}}</tool_call>',
    '<tool_call>{"name": "trunc", "arguments": {"a": ',
    "<function=get_weather><parameter=city>Paris</parameter></function>",
    '<function name="get_weather"><parameter=city>Paris</parameter></function>',
    "<function=broken><parameter=city>Paris",
    '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}<tool_call|>',
    "<|tool_call>call:get_weather{city:Paris",
    'call:get_weather{city:<|"|>Paris<|"|>}',
    '[TOOL_CALLS]get_weather{"city": "Paris"}',
    '[TOOL_CALLS] [{"name": "get_weather", "arguments": {"city": "Paris"}}]',
    '[TOOL_CALLS]get_weather[ARGS]{"city": "Paris"}',
    'get_weather[ARGS]{"city": "Paris"}',
    "get_weather[ARGS]{",
    "get_weather[ARGS]",
    'unlisted_tool[ARGS]{"city": "Paris"}',
    # Execution-class forms: without them every guard-gated function is constant.
    'terminal[ARGS]{"command": "id"}',
    'call:terminal{command:<|"|>id<|"|>}',
    '{"name": "terminal", "parameters": {"command": "id"}};'
    '{"name": "get_weather", "parameters": {"city": "Paris"}}',
    # Chunk boundaries mid-call, so the streaming holds are driven.
    "Here is prose call:get_weather{city:",
    "Here is prose cal",
    '{"name": "terminal", "parameters": {"command": "id"}}; call:get_weather{city:1}',
    '<|python_tag|>{"name": "get_weather", "parameters": {"city": "Paris"}}',
    '<|python_tag|>get_weather.call(city="Paris")',
    '{"name": "get_weather", "parameters": {"city": "Paris"}}',
    "<|message_model|>get_weather<|content_invoke_tool_json|>"
    '{"name": "get_weather", "args": {"city": "Paris"}}<|end_message|>',
    "<|tool▁calls▁begin|><|tool▁call▁begin|>get_weather<|tool▁sep|>"
    '{"city": "Paris"}<|tool▁call▁end|><|tool▁calls▁end|>',
    "<|tool▁calls▁begin|>function<|tool▁sep|>get_weather\n"
    '```json\n{"city": "Paris"}\n```<|tool▁calls▁end|>',
    "<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0"
    '<|tool_call_argument_begin|>{"city": "Paris"}<|tool_call_end|>'
    "<|tool_calls_section_end|>",
    "<tool_call>get_weather\n<arg_key>city</arg_key>\n<arg_value>Paris</arg_value>\n</tool_call>",
    '<minimax:tool_call>\n<invoke name="get_weather">\n<parameter name="city">Paris</parameter>\n'
    "</invoke>\n</minimax:tool_call>",
    '<minimax:tool_call>\n<invoke name="get_weather">\n<parameter name="city">Par',
    # Cross-format markup, so an arm swap inside strip_segment is visible.
    "<function=x><tool_call> txt <arg_key>c</arg_key></function><parameter=p>",
    '<tool_call>{"name": "search", "arguments": {"q": "<function=get_weather>"}}</tool_call>',
    '<tool_call>{"name": "search", "arguments": {"q": "[TOOL_CALLS]other[ARGS]{}"}}</tool_call>',
    "<function=get_weather><parameter=city><|tool_call>call:search{q:1}<tool_call|></parameter></function>",
    '[TOOL_CALLS]search[ARGS]{"q": "<tool_call>{}</tool_call>"}',
    '<|tool_call>call:search{q:<|"|><think>x</think><|"|>}<tool_call|>',
    '<function=x><parameter=p>get_weather[ARGS]{"a": 1}</parameter></function> tail',
    "<tool_call>get_weather\n<arg_key>c</arg_key>\n<arg_value><function=y></arg_value>\n</tool_call>",
    "<think>I should call get_weather[ARGS]{} but not really</think>",
    "<think>unclosed reasoning",
    '```\nget_weather[ARGS]{"city": "Paris"}\n```',
    "`get_weather[ARGS]{}`",
    "Sure, let me look that up for you.",
    "The weather in Paris is 18 degrees.",
    "",
    "{",
    "}",
    "[",
    "]",
    '[{"a": 1}]',
    "[}]",
    '["a}b"]',
    "\\",
    '{"a": "\\""}',
)

# `terminal` enables the execution guard, or gated functions are constant here.
_ENABLED_NAMES = ("get_weather", "search", "trunc", "broken", "terminal")


def build_corpus(seed: int = 20260811, count: int = 600) -> list:
    """Deterministic corpus: every fragment alone, every ordered pair, then random splices."""
    corpus = list(_FRAGMENTS)
    for left in _FRAGMENTS:
        for right in _FRAGMENTS:
            corpus.append(left + right)
    rng = random.Random(seed)
    joiners = ("", " ", "\n", "\n\n", " and then ", "\n```\n")
    for _ in range(count):
        parts = [rng.choice(_FRAGMENTS) for _ in range(rng.randint(2, 5))]
        corpus.append(rng.choice(joiners).join(parts))
    return list(dict.fromkeys(corpus))


# Offsets come from the text so scanners start on a real delimiter, else <undrivable>.
_ARG_FIXTURES = {
    "brace_start": lambda text: max(text.find("{"), 0),
    "brace_pos": lambda text: max(text.find("{"), 0),
    "paren_start": lambda text: max(text.find("("), 0),
    "start": lambda text: max(text.find("["), 0),
    "pos": lambda text: len(text),
    "body_start": lambda text: max(text.find("[") + 1, 0),
    "body_end": lambda text: len(text),
    "end": lambda text: len(text),
    "bodies": lambda text: [],
    "body": lambda text: text,
    "hard_stop": lambda text: len(text),
    "i": lambda text: 0,
    "idx": lambda text: 0,
    "floor": lambda text: 0,
    "p": lambda text: 0,
    "n": lambda text: len(text),
    "vs": lambda text: 0,
    "needle": lambda text: "[",
    # _safe_cut needs the real first-sentinel offset; at 0 it pins nothing.
    "first": lambda text: _parser_first_sentinel(text),
    "found": lambda text: max(_parser_first_sentinel(text), 0),
    # Offset of the first FOREIGN signal, as callers pass; 0 would pin nothing.
    "signal": lambda text: max(_parser_first_foreign_signal(text) or 0, 0),
    "out": lambda text: [],
    "result": lambda text: text,
    "last_result": lambda text: text,
    # Above _MAX_IDENTICAL_TOOL_RESULTS so the plural branch renders.
    "times": lambda text: 3,
    "previous": lambda text: text,
    "markers": lambda text: _tool_healing_build_markers(text),
    "patterns": lambda text: _tool_healing_all_pats(),
    "strip_segment": lambda text: (lambda segment, is_last: segment),
    # The DeepSeek opener match the envelope parser is handed; None (no opener) pins the raise.
    "begin": lambda text: _parser_deepseek_begin(text),
}


_SWEEP_PARAMS = frozenset({"pos"})

_VARIANTS_KEY = "@variants"

_BOTH_WAYS = ("final", "seg_final", "with_spans", "allow_incomplete", "gemma_quotes")

_TEXT_ADAPTERS = {
    "_gemma_arguments_to_json": lambda text: _gemma_argument_body(text),
    # The corpus holds no backticked tags; backtick each one so the spans vary with the text.
    "tag_mention_spans": lambda text: re.sub(r"<[^<>`\n]+>|\[/?TOOL_CALLS\]", r"`\g<0>`", text),
}


def _sweep_offsets(text: str):
    n = len(text)
    return sorted({0, n // 4, n // 2, (3 * n) // 4, n})


def _gemma_argument_body(text: str) -> str:
    """The parser takes a raw argument body; a whole corpus entry makes it raise JSONDecodeError."""
    from core import tool_healing

    brace = text.find("{")
    if brace >= 0:
        end = tool_healing._balanced_brace_end(text, brace)
        if end is not None:
            return text[brace + 1 : end]
    return "city:Paris, n:2, tags:[a, b]"


def _parser_first_sentinel(text: str):
    from core.inference import tool_call_parser
    return tool_call_parser._first_sentinel(text, 0)


def _parser_first_foreign_signal(text: str):
    from core.inference import tool_call_parser
    return tool_call_parser._first_foreign_tool_signal(text)


def _parser_deepseek_begin(text: str):
    from core.inference import tool_call_parser
    return tool_call_parser._DEEPSEEK_BEGIN_RE.search(text)


def _tool_healing_build_markers(text: str):
    from core import tool_healing
    return tool_healing._build_markers(text)


def _tool_healing_all_pats():
    from core import tool_healing
    return tool_healing._TOOL_ALL_PATS


def _drive(func, text: str):
    """Returns a marker string when func raises; a refactor that starts or stops raising is a change."""
    params = inspect.signature(func).parameters
    names = list(params)
    kwargs = {}
    if "enabled_tool_names" in params:
        kwargs["enabled_tool_names"] = set(_ENABLED_NAMES)
    combos = [{}]
    for flag in _BOTH_WAYS:
        if flag in params:
            combos = [dict(combo, **{flag: value}) for combo in combos for value in (True, False)]
    if "id_offset" in params:
        kwargs["id_offset"] = 0

    adapter = _TEXT_ADAPTERS.get(getattr(func, "__name__", ""))
    args = [adapter(text) if adapter else text]
    sweep = None
    for extra in names[1:]:
        if extra in kwargs or params[extra].kind == inspect.Parameter.KEYWORD_ONLY:
            continue
        if params[extra].default is not inspect.Parameter.empty:
            continue
        fixture = _ARG_FIXTURES.get(extra)
        if fixture is None:
            return "<undrivable>"
        if extra in _SWEEP_PARAMS:
            sweep = extra
            args.append(None)
            continue
        args.append(fixture(text))

    def _call(extra):
        merged = dict(kwargs, **extra)
        if sweep is not None:
            slot = names.index(sweep)
            results = []
            for offset in _sweep_offsets(text):
                args[slot] = offset
                try:
                    results.append(_jsonable(func(*args, **merged)))
                except Exception as exc:  # noqa: BLE001
                    results.append(f"<raised {type(exc).__name__}>")
            return results
        try:
            outcome = func(*args, **merged)
        except Exception as exc:  # noqa: BLE001 - the exception type is the pinned value
            return f"<raised {type(exc).__name__}>"
        if "out" in names and args[names.index("out")]:
            return [_jsonable(outcome), _jsonable(args[names.index("out")])]
        return _jsonable(outcome)

    if len(combos) > 1:
        return {_VARIANTS_KEY: {json.dumps(c, sort_keys = True): _call(c) for c in combos}}
    return _call(combos[0])


def _jsonable(value):
    if isinstance(value, (str, int, float, bool)) or value is None:
        return value
    if isinstance(value, (list, tuple)):
        return [_jsonable(v) for v in value]
    if isinstance(value, (set, frozenset)):
        return sorted(_jsonable(v) for v in value)
    if isinstance(value, dict):
        return {str(k): _jsonable(v) for k, v in value.items()}
    if inspect.isgenerator(value):
        # A generator repr carries its address; the yielded values are the behaviour.
        try:
            return [_jsonable(item) for item in value]
        except Exception as exc:  # noqa: BLE001 - raising mid-iteration is pinned too
            return f"<raised {type(exc).__name__}>"
    return repr(value)


def _guarded_functions(mod_name):
    """Single-text-argument functions in a behaviour module, in a stable order."""
    module = importlib.import_module(mod_name)
    out = []
    for name in sorted(dir(module)):
        if name.startswith("__"):
            continue
        obj = getattr(module, name)
        if not inspect.isfunction(obj) or obj.__module__ != mod_name:
            continue
        params = list(inspect.signature(obj).parameters.values())
        if not params:
            continue
        if params[0].annotation not in (str, "str"):
            continue
        out.append((name, obj))
    return out


def golden_outputs(corpus) -> dict:
    """Stores one digest per function, plus one for the corpus, so a corpus edit reports as one change."""
    out = {"corpus": _digest([len(corpus), corpus])}
    for mod_name in BEHAVIOUR_MODULES:
        per_module = {}
        for name, func in _guarded_functions(mod_name):
            per_module[name] = _digest([_drive(func, text) for text in corpus])
        out[mod_name] = per_module
    return out


def _digest(value) -> str:
    payload = json.dumps(value, sort_keys = True, ensure_ascii = False, default = repr)
    return hashlib.sha256(payload.encode()).hexdigest()


def first_divergence(corpus, mod_name, func_name):
    """Outputs per corpus input, to localise a digest change; run on each side of the revision."""
    module = importlib.import_module(mod_name)
    func = getattr(module, func_name)
    return [{"input": text, "output": _drive(func, text)} for text in corpus]


def _variants(result):
    """Detect variants by the tag key, not by dict type: a parser may legitimately return {}."""
    if isinstance(result, dict) and set(result) == {_VARIANTS_KEY}:
        return sorted(result[_VARIANTS_KEY].items())
    return [("", result)]


def idempotence_failures(corpus) -> list:
    """A stripper that is not idempotent shows different text depending on how a stream is chunked."""
    failures = []
    for mod_name in BEHAVIOUR_MODULES:
        for name, func in _guarded_functions(mod_name):
            if "strip" not in name or "parse" in name:
                continue
            witnessed = set()
            labels = None
            for text in corpus:
                results = _variants(_drive(func, text))
                if labels is None:
                    labels = {variant for variant, _ in results}
                for variant, once in results:
                    if variant in witnessed:
                        continue
                    if not isinstance(once, str) or once.startswith("<raised "):
                        continue
                    twice = dict(_variants(_drive(func, once))).get(variant)
                    if twice != once:
                        failures.append(
                            {
                                "module": mod_name,
                                "function": name,
                                "variant": variant,
                                "input": text,
                                "once": once,
                                "twice": twice,
                            }
                        )
                        witnessed.add(variant)
                if labels is not None and witnessed >= labels:
                    break
    return failures


_PATCH_TARGET_RE = re.compile(
    r"""(?:mock\.)?(?:patch|monkeypatch\.setattr)\(\s*"""
    r"""["']((?:core|routes|utils|state|hub|storage|picker)\.[\w.]+)["']"""
)


def patch_targets(tests_dir = None) -> dict:
    """String patch targets found in the test suite, grouped by module."""
    tests_dir = tests_dir or (BACKEND_ROOT / "tests")
    targets = {}
    for path in sorted(tests_dir.rglob("test_*.py")):
        for match in _PATCH_TARGET_RE.finditer(path.read_text(encoding = "utf-8", errors = "ignore")):
            dotted = match.group(1)
            # as_posix so Windows backslashes do not read as a changed inventory.
            targets.setdefault(dotted, []).append(path.relative_to(BACKEND_ROOT).as_posix())
    return targets


def unresolvable_patch_targets(targets = None) -> list:
    """Longest importable prefix of a dotted target is the module; the rest must resolve by getattr."""
    targets = targets if targets is not None else patch_targets()
    broken = []
    for dotted, users in sorted(targets.items()):
        parts = dotted.split(".")
        obj = None
        last_error = None
        for split in range(len(parts) - 1, 0, -1):
            try:
                obj = importlib.import_module(".".join(parts[:split]))
            except Exception as exc:  # noqa: BLE001 - an unimportable prefix is not the module
                last_error = exc
                continue
            rest = parts[split:]
            break
        else:
            # A ModuleNotFoundError naming a target prefix means gone; else a missing dependency.
            environment = not (
                isinstance(last_error, ModuleNotFoundError)
                and last_error.name in {".".join(parts[:i]) for i in range(1, len(parts))}
            )
            entry = {
                "target": dotted,
                "reason": (
                    f"no importable module prefix, unimportable here: {last_error!r}"
                    if environment
                    else "no importable module prefix"
                ),
                "tests": users,
            }
            if environment:
                entry["environment"] = True
            broken.append(entry)
            continue
        for attr in rest:
            # core.inference uses PEP 562 __getattr__, so missing deps raise ImportError.
            try:
                found = hasattr(obj, attr)
            except Exception as exc:  # noqa: BLE001
                environment = not isinstance(exc, AttributeError)
                entry = {
                    "target": dotted,
                    "reason": (
                        f"unresolvable in this environment: {type(exc).__name__}"
                        if environment
                        else f"lazy export {attr!r} is gone: {exc}"
                    ),
                    "tests": users,
                }
                if environment:
                    entry["environment"] = True
                broken.append(entry)
                break
            if not found:
                reason = f"missing attribute {attr!r}"
                environment = False
                if inspect.ismodule(obj) and hasattr(obj, "__path__"):
                    candidate = f"{obj.__name__}.{attr}"
                    try:
                        importlib.import_module(candidate)
                    except ModuleNotFoundError as exc:
                        if exc.name and exc.name != candidate:
                            reason = f"submodule {attr!r} needs {exc.name!r}, absent here"
                            environment = True
                    except ImportError as exc:
                        reason = f"submodule {attr!r} is unimportable here: {exc}"
                        environment = True
                    except Exception:
                        pass
                    else:
                        found = True
                if not found:
                    entry = {"target": dotted, "reason": reason, "tests": users}
                    if environment:
                        entry["environment"] = True
                    broken.append(entry)
                    break
            obj = getattr(obj, attr, None)
    return broken


TWIN_NAMES = (
    "_balanced_brace_end",
    "_balanced_bracket_end",
    "_inside_open_parameter",
    "_trim_param_value",
    "parse_tool_calls_from_text",
)


def twin_divergence(corpus) -> dict:
    healing = importlib.import_module("core.tool_healing")
    parser = importlib.import_module("core.inference.tool_call_parser")
    report = {}
    for name in TWIN_NAMES:
        h_func, p_func = getattr(healing, name, None), getattr(parser, name, None)
        if h_func is None or p_func is None:
            report[name] = {"status": "unified" if h_func is p_func else "missing"}
            continue
        if h_func is p_func:
            report[name] = {"status": "unified"}
            continue
        examples = []
        for text in corpus:
            h_out, p_out = _drive(h_func, text), _drive(p_func, text)
            if (h_out in (-1, None)) and (p_out in (-1, None)):
                continue
            if h_out != p_out:
                examples.append({"input": text, "healing": h_out, "parser": p_out})
        report[name] = {
            "status": "diverges" if examples else "equivalent",
            "count": len(examples),
            "examples": examples[:5],
        }
    return report


def _write(name, payload):
    BASELINE_DIR.mkdir(parents = True, exist_ok = True)
    path = BASELINE_DIR / name
    path.write_text(
        json.dumps(payload, indent = 2, sort_keys = True, ensure_ascii = False) + "\n",
        encoding = "utf-8",
    )
    return path


def _read(name):
    return json.loads((BASELINE_DIR / name).read_text(encoding = "utf-8"))


def _diff(
    label,
    old,
    new,
    *,
    additions_matter = True,
):
    """additions_matter = False reports only changes and removals; adding a symbol is not a regression."""
    if old == new:
        return []
    problems = []
    if isinstance(old, dict) and isinstance(new, dict):
        for key in sorted(set(old) | set(new)):
            if key not in old:
                if additions_matter:
                    problems.append(f"{label}.{key}: added")
            elif key not in new:
                problems.append(f"{label}.{key}: REMOVED")
            else:
                problems.extend(
                    _diff(f"{label}.{key}", old[key], new[key], additions_matter = additions_matter)
                )
    elif isinstance(old, list) and isinstance(new, list) and len(old) == len(new):
        for index, (a, b) in enumerate(zip(old, new)):
            problems.extend(_diff(f"{label}[{index}]", a, b, additions_matter = additions_matter))
    elif (
        isinstance(old, list)
        and isinstance(new, list)
        and all(isinstance(v, str) for v in old + new)
    ):
        # Counted, not set-compared, so dropping one of several identical entries shows.
        old_counts, new_counts = Counter(old), Counter(new)
        for name in sorted(set(old) | set(new)):
            delta = new_counts[name] - old_counts[name]
            if delta > 0:
                if additions_matter:
                    problems.append(f"{label}: added {name!r}")
            elif delta < 0:
                if new_counts[name]:
                    problems.append(f"{label}: {old_counts[name]} -> {new_counts[name]} x {name!r}")
                else:
                    problems.append(f"{label}: REMOVED {name!r}")
    else:
        problems.append(f"{label}: {old!r} -> {new!r}")
    return problems


def snapshot():
    corpus = build_corpus()
    _write("ast_inventory.json", ast_inventory())
    _write("runtime_inventory.json", runtime_inventory())
    _write("golden_outputs.json", golden_outputs(corpus))
    _write("patch_targets.json", patch_targets())
    failures = idempotence_failures(corpus)
    _write("idempotence_baseline.json", failures)
    print(f"baseline written to {BASELINE_DIR}")
    print(f"  corpus: {len(corpus)} inputs")
    print(f"  patch targets: {len(patch_targets())}")
    print(f"  pre-existing idempotence failures: {len(failures)}")
    for failure in failures:
        print(f"    {failure['module']}.{failure['function']}")


def _ast_problems() -> list:
    """AST diff, strict for the modules this branch owns and additive-tolerant elsewhere."""
    recorded = _read("ast_inventory.json")
    live = ast_inventory()
    problems = []
    for mod_name in sorted(set(recorded) | set(live)):
        problems += _diff(
            f"ast.{mod_name}",
            recorded.get(mod_name, {}),
            live.get(mod_name, {}),
            additions_matter = mod_name in BEHAVIOUR_MODULES,
        )
    return problems


def verify() -> int:
    corpus = build_corpus()
    problems = []
    problems += _ast_problems()
    problems += _diff("runtime", _read("runtime_inventory.json"), runtime_inventory())
    problems += _diff("golden", _read("golden_outputs.json"), golden_outputs(corpus))

    baseline_idempotence = {
        (f["module"], f["function"], f.get("variant", ""))
        for f in _read("idempotence_baseline.json")
    }
    for failure in idempotence_failures(corpus):
        key = (failure["module"], failure["function"], failure["variant"])
        if key not in baseline_idempotence:
            problems.append(
                f"idempotence.{failure['module']}.{failure['function']}"
                f"{'[' + failure['variant'] + ']' if failure['variant'] else ''}: "
                f"f(f(x)) != f(x) for {failure['input']!r}"
            )

    recorded = {target: sorted(tests) for target, tests in _read("patch_targets.json").items()}
    live = {target: sorted(tests) for target, tests in patch_targets().items()}
    problems += _diff("patch-targets", recorded, live, additions_matter = False)

    for broken in unresolvable_patch_targets(live):
        # An uninstalled optional backend is not a broken target.
        if not broken.get("environment"):
            problems.append(
                f"patch-target {broken['target']}: {broken['reason']} ({broken['tests'][0]})"
            )

    if problems:
        print(f"FAIL: {len(problems)} difference(s)")
        for problem in problems[:60]:
            print(f"  {problem}")
        if len(problems) > 60:
            print(f"  ... and {len(problems) - 60} more")
        return 1
    print("OK: inventory, golden outputs, idempotence and patch targets all match")
    return 0


def twins():
    report = twin_divergence(build_corpus())
    print(json.dumps(report, indent = 2, ensure_ascii = False))


def main() -> int:
    command = sys.argv[1] if len(sys.argv) > 1 else "verify"
    if command == "snapshot":
        snapshot()
        return 0
    if command == "verify":
        return verify()
    if command == "twins":
        twins()
        return 0
    print(f"unknown command {command!r}; expected snapshot, verify or twins")
    return 2


if __name__ == "__main__":
    raise SystemExit(main())