Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

9.9 KB
Raw Blame
"""Unit tests for the decision provenance ladder, confidence formula, and the
anti-hallucination substring gate."""

from __future__ import annotations

from pathlib import Path

from repowise.core.analysis.decision_extractor import DecisionExtractor, ExtractedDecision
from repowise.core.analysis.decision_provenance import (
    BODY_FIELDS,
    MAX_SOURCE_RANK,
    completeness,
    compute_confidence,
    rank_for_source,
    verify_quote,
)

# ---------------------------------------------------------------------------
# Source ranking ladder
# ---------------------------------------------------------------------------


def test_source_rank_ladder_ordering():
    # session > adr > commit > changelog > inline_marker > comment > test_name > inferred
    order = [
        "session",
        "adr",
        "commit",
        "changelog",
        "inline_marker",
        "comment",
        "test_name",
        "inferred",
    ]
    ranks = [rank_for_source(s) for s in order]
    assert ranks == sorted(ranks, reverse=True)
    assert ranks == [8, 7, 6, 5, 4, 3, 2, 1]


def test_session_outranks_adr():
    """A user's own words beat a later write-up of them.

    The promotion branch in ``crud.decisions`` compares with ``>=``, so with
    ``session`` below ``adr`` a transcript could never take the headline and a
    mined document overwrote what the user actually said.
    """
    assert rank_for_source("session") > rank_for_source("adr")


def test_agent_ranks_with_session():
    """An agent's own proposal is first-hand reasoning, like a transcript."""
    from repowise.core.analysis.decisions.provenance import LISTABLE_SOURCES

    assert rank_for_source("agent") == rank_for_source("session") == 8
    assert "agent" in LISTABLE_SOURCES


def test_agent_proposal_is_not_self_authored():
    """Only a typed ``cli`` record is its own evidence; an agent must cite some."""
    from repowise.core.analysis.decisions.lifecycle import requirement

    assert not requirement({"source": "agent"}, accepter="reviewer").self_authored


def test_adr_and_pr_tie():
    """Both are write-ups rather than the decision being made; later one wins."""
    assert rank_for_source("adr") == rank_for_source("pr") == 7


def test_swapping_session_and_adr_left_the_scale_intact():
    """The ladder's value set is unchanged, so no other source's confidence moved.

    ``compute_confidence`` divides by ``MAX_SOURCE_RANK``. Had the swap been done
    by lifting ``session`` to a new top rank instead of trading places with
    ``adr``, every source's confidence would have silently rescaled and every
    stored value in every existing store would have been on a different scale
    from every new one.
    """
    assert MAX_SOURCE_RANK == 9
    assert compute_confidence(rank_for_source("commit")) == compute_confidence(6)


def test_git_archaeology_aliases_commit_rank():
    assert rank_for_source("git_archaeology") == rank_for_source("commit") == 6


def test_conventions_sits_with_commit_and_leaves_the_scale_intact():
    """Counted from the code itself, so it ranks where a commit does."""
    from repowise.core.analysis.decision_provenance import LISTABLE_SOURCES

    assert rank_for_source("conventions") == 6
    assert MAX_SOURCE_RANK == 9
    assert "conventions" in LISTABLE_SOURCES


def test_unknown_source_lowest_rank():
    assert rank_for_source("something_new") == 1
    assert rank_for_source(None) == 1


def test_cli_outranks_adr():
    # Human-authored manual entry is the most authoritative source.
    assert rank_for_source("cli") > rank_for_source("adr")
    assert rank_for_source("cli") == MAX_SOURCE_RANK


# ---------------------------------------------------------------------------
# Confidence formula
# ---------------------------------------------------------------------------


def test_confidence_rises_with_rank():
    low = compute_confidence(rank_for_source("inferred"))
    high = compute_confidence(rank_for_source("adr"))
    assert high > low


def test_confidence_rises_with_corroboration():
    single = compute_confidence(rank_for_source("commit"), corroboration_count=1)
    triple = compute_confidence(rank_for_source("commit"), corroboration_count=3)
    assert triple > single


def test_confidence_decays_for_weak_verification():
    exact = compute_confidence(8, 2, "exact")
    fuzzy = compute_confidence(8, 2, "fuzzy")
    unverified = compute_confidence(8, 2, "unverified")
    assert exact > fuzzy > unverified


def test_confidence_bounded():
    assert 0.0 <= compute_confidence(MAX_SOURCE_RANK, 99, "exact") <= 0.99


# ---------------------------------------------------------------------------
# verify_quote
# ---------------------------------------------------------------------------


def test_verify_quote_exact_ignores_whitespace_and_case():
    src = "We   adopted Redis\nfor the shared cache layer."
    assert verify_quote("adopted redis for the shared cache layer.", src) == "exact"


def test_verify_quote_fuzzy_on_paraphrase():
    src = "We adopted Redis because the in-process cache could not share state across workers."
    # Reworded but high token overlap.
    assert verify_quote("adopted Redis: in-process cache could not share state workers", src) == (
        "fuzzy"
    )


def test_verify_quote_unverified_on_hallucination():
    src = "We adopted Redis for the shared cache."
    assert verify_quote("We migrated the billing service to Kafka for event sourcing", src) == (
        "unverified"
    )


def test_verify_quote_unverified_without_source():
    assert verify_quote("anything", "") == "unverified"


# ---------------------------------------------------------------------------
# Substring gate
# ---------------------------------------------------------------------------


def _extractor(tmp_path: Path) -> DecisionExtractor:
    return DecisionExtractor(repo_path=tmp_path)


def test_gate_keeps_grounded_decision(tmp_path):
    ex = _extractor(tmp_path)
    d = ExtractedDecision(
        title="Adopt Redis",
        decision="adopt Redis for caching",
        rationale="the in-process cache could not share state across workers",
        source="commit",
        source_text=(
            "feat: adopt Redis for caching\n\nThe in-process cache could not share "
            "state across workers, so we adopt Redis for caching."
        ),
    )
    kept, rejected = ex._apply_substring_gate([d])
    assert rejected == 0
    assert len(kept) == 1
    assert kept[0].verification in ("exact", "fuzzy")
    assert kept[0].decision == "adopt Redis for caching"
    # Transient source text is cleared before persistence.
    assert kept[0].source_text == ""


def test_gate_rejects_fully_hallucinated_decision(tmp_path):
    ex = _extractor(tmp_path)
    d = ExtractedDecision(
        title="Switch to Kafka",
        decision="migrate the billing pipeline to Kafka event streaming",
        rationale="to support exactly-once delivery semantics at scale",
        source="git_archaeology",
        # The source says nothing about Kafka — every produced field is ungrounded.
        source_text="chore: bump dependencies\n\nRoutine dependency upgrade, no behaviour change.",
    )
    kept, rejected = ex._apply_substring_gate([d])
    assert rejected == 1
    assert kept == []


def test_gate_drops_only_the_hallucinated_field(tmp_path):
    ex = _extractor(tmp_path)
    d = ExtractedDecision(
        title="Use SQLite",
        decision="use SQLite for local persistence",
        rationale="because it is web-scale and horizontally sharded",  # not in source
        source="inline_marker",
        source_text="# DECISION: use SQLite for local persistence — zero-config, single file.",
    )
    kept, _rejected = ex._apply_substring_gate([d])
    assert len(kept) == 1
    # Decision survives (grounded); hallucinated rationale is dropped.
    assert kept[0].decision == "use SQLite for local persistence"
    assert kept[0].rationale == ""


def test_gate_keeps_unverifiable_when_no_source_text(tmp_path):
    ex = _extractor(tmp_path)
    d = ExtractedDecision(title="Manual entry", decision="some decision", source="cli")
    kept, rejected = ex._apply_substring_gate([d])
    assert rejected == 0
    assert len(kept) == 1
    assert kept[0].verification == "unverified"


# --- completeness into confidence ------------------------------------------


def test_completeness_counts_only_fields_that_say_something():
    assert completeness() == 0
    assert completeness(decision="x", rationale="  ", context=None) == 1
    assert (
        completeness(
            decision="d",
            rationale="r",
            context="c",
            consequences=["one"],
            alternatives=["two"],
        )
        == BODY_FIELDS
    )


def test_a_record_that_says_more_scores_higher_at_the_same_rank():
    thin = compute_confidence(8, 1, "exact", filled_fields=1)
    full = compute_confidence(8, 1, "exact", filled_fields=5)
    assert thin < full


def test_completeness_outranks_a_rung_of_source_rank():
    """The inversion this term exists to fix: a thin high-rank record used to
    outscore a complete lower-rank one."""
    thin_session = compute_confidence(8, 1, "exact", filled_fields=1)
    full_pr = compute_confidence(7, 1, "exact", filled_fields=5)
    assert full_pr > thin_session


def test_an_unknown_body_leaves_the_score_unadjusted():
    assert compute_confidence(8, 1, "exact") == compute_confidence(
        8, 1, "exact", filled_fields=BODY_FIELDS
    )


def test_a_record_that_says_nothing_keeps_the_evidence_floor():
    """The 0.4 floor means "there is evidence at all", which a blank record
    still has. The term withholds the rank credit above it, not the floor."""
    assert compute_confidence(9, 1, "exact", filled_fields=0) == 0.4


def test_completeness_never_breaks_the_bounds_or_the_comment_subtier():
    for filled in range(-1, BODY_FIELDS + 2):
        assert 0.0 <= compute_confidence(9, 99, "exact", filled_fields=filled) <= 0.99
        assert compute_confidence(2, 1, "exact", filled_fields=filled) < 0.5