Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

8.0 KB
Raw Blame
"""In-code rationale mining — the shared comment heuristics.

These heuristics feed the query-time MCP live-grep miner
(``mcp_server/_code_rationale.py``). The precision guardrails are the whole
risk, so they get the coverage. (The index-time ``code_comment`` harvest that
also consumed them was removed; see #751.)
"""

from __future__ import annotations

from repowise.core.analysis.decision_extractor import DecisionExtractor
from repowise.core.analysis.decisions.rationale_comments import (
    extract_comment_blocks,
    harvest_file_rationale,
    has_causal_marker,
    has_rationale_marker,
    is_license_or_boilerplate,
    looks_like_commented_out_code,
    marker_strength,
)

# A real rationale comment: prose, a causal marker, enclosing a function.
_RATIONALE_PY = '''\
"""Module docstring."""

import time


def call_api():
    # We retry on 429 here rather than in the client because the client is
    # shared across tenants and a global backoff would starve everyone.
    return time.time()
'''

_LICENSE_PY = """\
# Copyright 2026 Acme Inc. All rights reserved.
# Licensed under the Apache License because the board said so.
import os
"""


# ---------------------------------------------------------------------------
# Comment tokenizer
# ---------------------------------------------------------------------------


def test_extract_comment_blocks_coalesces_consecutive_lines():
    blocks = extract_comment_blocks(_RATIONALE_PY, "py")
    # The two-line rationale run coalesces into one block.
    runs = [b for b in blocks if "we retry on 429" in b.text.lower()]
    assert len(runs) == 1
    b = runs[0]
    assert b.start_line < b.end_line  # spans both comment lines
    assert "starve everyone" in b.text


def test_extract_comment_blocks_handles_c_style_and_returns_empty_for_unknown():
    js = "/* we batch writes because the disk fsync dominates latency */\nx = 1;\n"
    blocks = extract_comment_blocks(js, "js")
    assert any("batch writes" in b.text for b in blocks)
    # Unknown / non-code extension yields nothing.
    assert extract_comment_blocks("plain prose because yes", "txt") == []


def test_tree_sitter_query_comments_are_mined():
    scm = (
        ";;;;;;;;;;;;;;;;\n"
        ";; The receiver stays uncaptured because a qualified super call\n"
        "; binds to the parent class, not the field.\n"
        "(method_invocation name: (identifier) @call.target) @call.site\n"
    )
    blocks = extract_comment_blocks(scm, "scm")
    assert blocks[0].text.startswith("The receiver stays uncaptured")
    assert harvest_file_rationale(scm, "scm")


# ---------------------------------------------------------------------------
# Guardrails
# ---------------------------------------------------------------------------


def test_marker_detection_and_strength():
    assert has_rationale_marker("we do this because of X")
    assert not has_rationale_marker("just a plain label here")
    # Two distinct markers present → strength 2.
    assert marker_strength("because we must avoid the deadlock") >= 2


def test_causal_marker_is_stricter_than_rationale_marker():
    # An intent label has a rationale marker but no causal reason.
    assert has_rationale_marker("NOTE: always flush before close")
    assert not has_causal_marker("NOTE: always flush before close")
    # A stated reason is causal.
    assert has_causal_marker("we batch writes because fsync dominates")


def test_license_and_preamble_detected():
    assert is_license_or_boilerplate("Copyright 2026 Acme Inc. All rights reserved.")
    assert is_license_or_boilerplate("SPDX-License-Identifier: MIT")
    assert is_license_or_boilerplate("-*- coding: utf-8 -*-")
    assert not is_license_or_boilerplate("we cache this because it is hot")


def test_commented_out_code_detected_but_prose_kept():
    assert looks_like_commented_out_code(("x = compute(y)", "return x"))
    assert not looks_like_commented_out_code(
        ("we avoid the global lock because it serializes every tenant",)
    )


# ---------------------------------------------------------------------------
# Per-file harvest
# ---------------------------------------------------------------------------


def test_harvest_keeps_rationale_drops_license():
    kept = harvest_file_rationale(_RATIONALE_PY, "py")
    assert len(kept) == 1
    assert "starve everyone" in kept[0].text

    # The license block carries "because" but is boilerplate → dropped.
    assert harvest_file_rationale(_LICENSE_PY, "py") == []


def test_harvest_drops_thin_and_markerless_comments():
    src = "# hi\n# this is a plain comment with no causal marker at all here\nx = 1\n"
    assert harvest_file_rationale(src, "py") == []


def test_harvest_respects_per_file_cap():
    parts = []
    for i in range(6):
        parts.append(f"def f{i}():")
        parts.append(f"    # branch {i} exists because edge case {i} would otherwise crash")
        parts.append(f"    return {i}")
    src = "\n".join(parts) + "\n"
    kept = harvest_file_rationale(src, "py", max_per_file=3)
    assert len(kept) == 3


def test_harvest_non_code_extension_returns_empty():
    assert harvest_file_rationale("we did this because reasons", "md") == []


def test_harvest_excludes_docstrings_by_default_but_keeps_line_comments():
    src = (
        '"""We pick gRPC because REST added latency on the hot path."""\n'
        "# we cache the token because the lookup is expensive\n"
        "x = 1\n"
    )
    # Default: docstring rationale is skipped, the line comment is kept.
    kept = harvest_file_rationale(src, "py")
    assert len(kept) == 1
    assert "cache the token" in kept[0].text
    # Opt in to docstrings (the MCP recall mode) → both surface.
    both = harvest_file_rationale(src, "py", include_docstrings=True)
    assert len(both) == 2


def test_harvest_splits_and_drops_separator_rules():
    src = (
        "# ----------------------------------------------------------------\n"
        "# Section banner only\n"
        "# ----------------------------------------------------------------\n"
        "# we skip the cache here because the entry is already stale\n"
        "y = 2\n"
    )
    kept = harvest_file_rationale(src, "py")
    assert len(kept) == 1
    text = kept[0].text
    assert "skip the cache" in text
    assert "---" not in text  # divider rule never glued in
    assert "Section banner" not in text  # markerless run dropped


def test_require_causal_false_admits_intent_labels():
    src = "# NOTE: always flush the buffer before closing the handle here\nz = 3\n"
    assert harvest_file_rationale(src, "py") == []  # no causal reason
    loose = harvest_file_rationale(src, "py", require_causal=False)
    assert len(loose) == 1


# ---------------------------------------------------------------------------
# Extractor file walk (shared by inline markers)
# ---------------------------------------------------------------------------

_WHY_PY = (
    "def f():\n    # WHY: upstream API double-encodes, so we special-case here\n    return 1\n"
)


async def test_source_walk_skips_untracked_files_in_a_git_repo(tmp_path):
    """In a git checkout the extractor's file walk scopes to tracked files:
    untracked / excluded working dirs (local-stash, scratch dumps) must not
    produce records. Gitless repos fall back to the full walk."""
    import subprocess

    def _git(*args: str) -> None:
        subprocess.run(
            ["git", "-C", str(tmp_path), *args],
            check=True,
            capture_output=True,
        )

    _git("init")
    _git("config", "user.email", "[email protected]")
    _git("config", "user.name", "t")

    (tmp_path / "tracked.py").write_text(_WHY_PY, encoding="utf-8")
    _git("add", "tracked.py")
    _git("commit", "-m", "add tracked")

    # An untracked file carrying the same explicit WHY: marker.
    (tmp_path / "untracked.py").write_text(_WHY_PY, encoding="utf-8")

    ex = DecisionExtractor(repo_path=tmp_path)  # no provider → raw markers
    decisions = await ex.scan_inline_markers()

    files = {d.evidence_file for d in decisions}
    assert "tracked.py" in files
    assert "untracked.py" not in files