Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

11.3 KB
Raw Blame
"""Parse-cache equivalence: cached ingest must match a fresh parse exactly.

The cache snapshots ParsedFiles at parse time keyed by (path, content hash)
under a parser fingerprint. Every scenario asserts the cached run is
ParsedFile-equal AND graph-isomorphic to a from-scratch build — after file
edit, add, delete, rename, and an import-changing edit. Corrupt or
stale-fingerprint caches must degrade silently to a full parse.
"""

from __future__ import annotations

import pickle

import networkx as nx
import pytest

from repowise.cli.commands.update_cmd import _build_repo_graph
from repowise.core.cache_seal import seal, unseal
from repowise.core.ingestion import compute_content_hash
from repowise.core.ingestion.parse_cache import ParseCache, parser_fingerprint

_CACHE_DOMAIN = "parse_cache.pkl"



def _write_fixture(repo) -> None:
    (repo / "util.py").write_text(
        "def helper(x):\n    return x + 1\n",
        encoding="utf-8",
    )
    (repo / "main.py").write_text(
        "from util import helper\n\n\ndef main():\n    return helper(2)\n",
        encoding="utf-8",
    )
    (repo / "other.py").write_text(
        "VALUE = 3\n",
        encoding="utf-8",
    )


def _cache_path(repo) -> object:
    return repo / ".repowise" / "parse_cache.pkl"


def _build(repo):
    """Run the update path's full ingest; returns (parsed_files, graph)."""
    parsed, _src, gb, _structure, _count = _build_repo_graph(repo, [], collect_sources=False)
    return sorted(parsed, key=lambda p: p.file_info.path), gb.graph()


def _fresh_build(repo):
    """Ground truth: same build with the cache file removed."""
    cache_file = _cache_path(repo)
    if cache_file.exists():
        cache_file.unlink()
    return _build(repo)


def _assert_equivalent(repo) -> None:
    """Cached run == from-scratch run: ParsedFile-equal and graph-isomorphic."""
    cached_parsed, cached_graph = _build(repo)  # warm cache from prior run
    fresh_parsed, fresh_graph = _fresh_build(repo)

    assert [p.file_info.path for p in cached_parsed] == [p.file_info.path for p in fresh_parsed]
    for cp, fp in zip(cached_parsed, fresh_parsed, strict=True):
        assert cp == fp, cp.file_info.path

    assert nx.utils.graphs_equal(cached_graph, fresh_graph)


@pytest.fixture()
def repo(tmp_path):
    _write_fixture(tmp_path)
    (tmp_path / ".repowise").mkdir()
    return tmp_path


def test_warm_cache_equivalent_unchanged(repo):
    _build(repo)  # cold run populates the cache
    _assert_equivalent(repo)


def test_equivalent_after_edit(repo):
    _build(repo)
    (repo / "util.py").write_text(
        "def helper(x):\n    return x * 2\n\n\ndef extra(y):\n    return y\n",
        encoding="utf-8",
    )
    _assert_equivalent(repo)


def test_equivalent_after_add(repo):
    _build(repo)
    (repo / "added.py").write_text("def added_fn():\n    return 9\n", encoding="utf-8")
    _assert_equivalent(repo)


def test_equivalent_after_delete(repo):
    _build(repo)
    (repo / "other.py").unlink()
    _assert_equivalent(repo)


def test_equivalent_after_rename(repo):
    _build(repo)
    (repo / "other.py").rename(repo / "renamed.py")
    _assert_equivalent(repo)
    # Symbol IDs embed the path — the renamed file must carry the new path.
    parsed, _ = _build(repo)
    by_path = {p.file_info.path: p for p in parsed}
    assert "renamed.py" in by_path
    assert "other.py" not in by_path


def test_equivalent_after_import_changing_edit(repo):
    _build(repo)
    (repo / "main.py").write_text(
        "from other import VALUE\n\n\ndef main():\n    return VALUE\n",
        encoding="utf-8",
    )
    _assert_equivalent(repo)


def test_corrupt_cache_falls_back_to_full_parse(repo):
    _build(repo)
    _cache_path(repo).write_bytes(b"\x00not a pickle")
    cached_parsed, cached_graph = _build(repo)
    fresh_parsed, fresh_graph = _fresh_build(repo)
    for cp, fp in zip(cached_parsed, fresh_parsed, strict=True):
        assert cp == fp
    assert nx.utils.graphs_equal(cached_graph, fresh_graph)


def test_fingerprint_mismatch_invalidates(repo, tmp_path):
    _build(repo)
    cache_file = _cache_path(repo)
    payload = pickle.loads(unseal(cache_file.read_bytes(), domain=_CACHE_DOMAIN))  # repowise-security-ignore: pickle_loads
    assert payload["fingerprint"] == parser_fingerprint()
    payload["fingerprint"] = "stale-fingerprint"
    cache_file.write_bytes(seal(pickle.dumps(payload), domain=_CACHE_DOMAIN))

    cache = ParseCache(repo / ".repowise")
    cache.load()
    fi = _make_file_info(repo / "other.py", "other.py")
    source = (repo / "other.py").read_bytes()
    assert cache.get(fi, compute_content_hash(source)) is None
    assert cache.misses == 1


def test_dataclass_schema_change_invalidates_cache(repo, monkeypatch):
    """A ParsedFile (or nested dataclass) field change must self-invalidate.

    Regression: ``parse_cache.pkl`` pickles ParsedFile object graphs, so a new
    field on a cached dataclass made an old entry deserialize without the
    attribute, crashing the graph builder. The fingerprint now folds in every
    cached dataclass's field set, so the stale entry is treated as a miss.
    """
    import dataclasses

    from repowise.core.ingestion import models, parse_cache

    _build(repo)  # populate the cache under the current fingerprint
    cache_file = _cache_path(repo)
    assert (
        pickle.loads(unseal(cache_file.read_bytes(), domain=_CACHE_DOMAIN))["fingerprint"]  # repowise-security-ignore: pickle_loads
        == parser_fingerprint()
    )

    # Simulate a release that adds a field to ParsedFile.
    @dataclasses.dataclass
    class _Evolved(models.ParsedFile):
        new_field: int = 0

    _Evolved.__module__ = models.__name__  # masquerade as a models.py type
    monkeypatch.setattr(models, "ParsedFile", _Evolved)
    parse_cache.parser_fingerprint.cache_clear()
    try:
        assert (
            parser_fingerprint()
            != pickle.loads(unseal(cache_file.read_bytes(), domain=_CACHE_DOMAIN))[  # repowise-security-ignore: pickle_loads
                "fingerprint"
            ]
        )

        cache = ParseCache(repo / ".repowise")
        cache.load()
        fi = _make_file_info(repo / "other.py", "other.py")
        source = (repo / "other.py").read_bytes()
        assert cache.get(fi, compute_content_hash(source)) is None
    finally:
        parse_cache.parser_fingerprint.cache_clear()


def _make_file_info(abs_path, rel_path):
    from datetime import UTC, datetime

    from repowise.core.ingestion.models import FileInfo

    return FileInfo(
        path=rel_path,
        abs_path=str(abs_path),
        language="python",
        size_bytes=abs_path.stat().st_size,
        git_hash="",
        last_modified=datetime.now(UTC),
        is_test=False,
        is_config=False,
        is_api_contract=False,
        is_entry_point=False,
    )


def test_hit_rebinds_fresh_file_info_and_isolates_mutations(repo):
    from repowise.core.ingestion import ASTParser

    fi = _make_file_info(repo / "main.py", "main.py")
    source = (repo / "main.py").read_bytes()
    chash = compute_content_hash(source)
    parsed = ASTParser().parse_file(fi, source)

    cache = ParseCache(repo / ".repowise")
    cache.put(parsed, chash)

    fresh_fi = _make_file_info(repo / "main.py", "main.py")
    fresh_fi.git_hash = "abc123"
    hit1 = cache.get(fresh_fi, chash)
    hit2 = cache.get(fresh_fi, chash)

    # Fresh FileInfo rebound on the hit.
    assert hit1.file_info is fresh_fi
    assert hit1.file_info.git_hash == "abc123"
    # Distinct object graphs: graph-build mutations cannot leak between runs.
    assert hit1 is not hit2 and hit1 is not parsed
    assert hit1.imports and hit1.imports[0].resolved_file is None
    hit1.imports[0].resolved_file = "mutated.py"
    assert hit2.imports[0].resolved_file is None


def test_deleted_files_age_out_of_cache(repo):
    _build(repo)
    (repo / "other.py").unlink()
    _build(repo)  # rewrite keeps only entries touched this run
    payload = pickle.loads(unseal(_cache_path(repo).read_bytes(), domain=_CACHE_DOMAIN))  # repowise-security-ignore: pickle_loads
    cached_paths = {path for path, _hash in payload["files"]}
    assert "other.py" not in cached_paths
    assert "main.py" in cached_paths


def test_unsigned_legacy_pickle_is_ignored(repo, tmp_path):
    """Unsigned pickle (the #1390 attack shape) must not be loaded.

    Version / fingerprint checks used to run *after* pickle.load, so a
    crafted ``__reduce__`` payload already executed. We now refuse anything
    that is not HMAC-sealed with the machine-local key.
    """
    marker = tmp_path / "PWNED"

    class Evil:
        def __reduce__(self):
            return (marker.write_text, ("pwned",))

    # Legacy unsigned format — exactly what an attacker would ``git add -f``.
    _cache_path(repo).write_bytes(
        pickle.dumps(
            {
                "version": 1,
                "fingerprint": parser_fingerprint(),
                "files": {("main.py", "HASH"): Evil()},
            }
        )
    )

    cache = ParseCache(repo / ".repowise")
    cache.load()
    assert cache._entries == {}
    assert not marker.exists()


def test_forged_hmac_is_rejected(repo, monkeypatch, tmp_path):
    """A sealed envelope with a MAC under a different key is a miss."""
    from repowise.core import cache_seal

    monkeypatch.setenv("REPOWISE_CACHE_HMAC_KEY", "aa" * 32)
    cache_seal.reset_key_cache_for_tests()
    _build(repo)  # write under key A

    monkeypatch.setenv("REPOWISE_CACHE_HMAC_KEY", "bb" * 32)
    cache_seal.reset_key_cache_for_tests()
    cache = ParseCache(repo / ".repowise")
    cache.load()
    assert cache._entries == {}
    cache_seal.reset_key_cache_for_tests()


def test_cache_hit_stamps_content_hash(repo):
    """A hit must carry the raw-bytes hash even for entries pickled before
    ParsedFile had the field — the cache key already proves byte identity,
    and the cross-run page-reuse key is derived from this attribute."""
    from repowise.core.ingestion.parser import ASTParser

    fi = _make_file_info(repo / "main.py", "main.py")
    source = (repo / "main.py").read_bytes()
    parsed = ASTParser().parse_file(fi, source)
    parsed.content_hash = ""  # simulate an old pickle without the field

    cache = ParseCache(repo / ".repowise")
    cache.load()
    content_hash = compute_content_hash(source)
    cache.put(parsed, content_hash)

    hit = cache.get(fi, content_hash)
    assert hit is not None
    assert hit.content_hash == content_hash


def test_parse_without_grammar_is_not_cached(repo, monkeypatch):
    """A file parsed while its grammar was unavailable (a failed download, a
    missing package) must re-parse once the grammar loads, not be served
    symbol-less from the cache."""
    from repowise.core.ingestion import parser as parser_mod
    from repowise.core.ingestion.parser import ASTParser

    fi = _make_file_info(repo / "util.py", "util.py")
    source = (repo / "util.py").read_bytes()
    content_hash = compute_content_hash(source)

    monkeypatch.setattr(parser_mod, "_get_language", lambda _tag: None)
    degraded = ASTParser().parse_file(fi, source)
    assert degraded.grammar_missing and degraded.symbols == []

    cache = ParseCache(repo / ".repowise")
    cache.put(degraded, content_hash)
    cache.save()
    monkeypatch.undo()

    cache = ParseCache(repo / ".repowise")
    cache.load()
    assert cache.get(fi, content_hash) is None
    parsed = ASTParser().parse_file(fi, source)
    assert not parsed.grammar_missing and parsed.symbols