Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

3.1 KB
Raw Blame
"""Fixture diagnostics retain error classes without recording input values."""

# allow: no-sut-import -- exercises the isolated CI fixture diagnostics.

import importlib.util
import json
from pathlib import Path
from unittest.mock import Mock


ROOT = Path(__file__).resolve().parents[3]
SPEC = importlib.util.spec_from_file_location(
    "zap_api_diagnostics", ROOT / "scripts/ci/zap_api_diagnostics.py"
)
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)


def test_error_classification_omits_values_and_tracebacks(tmp_path):
    path = tmp_path / "diagnostics.json"
    diagnostics = MODULE.FixtureDiagnostics(path)
    underlying = Mock()
    logger = MODULE.DiagnosticLogger(underlying, diagnostics, "fixture")
    marker = "synthetic-private-value"

    class FixtureError(ValueError):
        def __str__(self):
            raise AssertionError("The collector must not render exceptions")

    try:
        raise FixtureError(marker)
    except FixtureError:
        logger.exception("Operation failed: {}", marker)

    underlying.exception.assert_not_called()
    underlying.opt.assert_called_once_with(exception=False)
    underlying.opt.return_value.error.assert_called_once_with(
        "CI fixture error; see classified diagnostics"
    )
    assert marker not in str(underlying.mock_calls)
    text = path.read_text(encoding="utf-8")
    assert marker not in text
    report = json.loads(text)
    assert report["errors"] == [
        {
            "source": "fixture",
            "function": "test_error_classification_omits_values_and_tracebacks",
            "exception": "FixtureError",
            "count": 1,
        }
    ]
    assert report["dropped"] == 0


def test_observation_without_active_exception_is_explicit(tmp_path):
    path = tmp_path / "diagnostics.json"
    diagnostics = MODULE.FixtureDiagnostics(path)
    logger = MODULE.DiagnosticLogger(Mock(), diagnostics, "fixture")
    logger.error("Fixed fixture message")
    report = json.loads(path.read_text(encoding="utf-8"))
    assert report["errors"][0]["exception"] == "NoActiveException"
    logger.info("Ordinary logging remains delegated")
    logger.logger.info.assert_called_once_with(
        "Ordinary logging remains delegated"
    )


def test_diagnostic_classes_and_counts_are_bounded(tmp_path):
    path = tmp_path / "diagnostics.json"
    diagnostics = MODULE.FixtureDiagnostics(path)
    for index in range(140):
        diagnostics.record("fixture", f"function_{index}", ValueError)
    report = json.loads(path.read_text(encoding="utf-8"))
    assert len(report["errors"]) == 128
    assert report["dropped"] == 12


def test_non_code_labels_are_replaced_before_storage(tmp_path):
    path = tmp_path / "diagnostics.json"
    diagnostics = MODULE.FixtureDiagnostics(path)
    diagnostics.record("not a code label", "not a code label", ValueError)
    report = json.loads(path.read_text(encoding="utf-8"))
    assert report["errors"] == [
        {
            "source": "unknown",
            "function": "unknown",
            "exception": "ValueError",
            "count": 1,
        }
    ]
    assert report["dropped"] == 0