Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
fix(cache): version the parse cache by ParsedFile schema shape (#582)
Raghav Chamadiya6cde5da704fc
diff --git a/packages/core/src/repowise/core/ingestion/parse_cache.py b/packages/core/src/repowise/core/ingestion/parse_cache.pyindex 7d08566db..0423986d4 100644--- a/packages/core/src/repowise/core/ingestion/parse_cache.py+++ b/packages/core/src/repowise/core/ingestion/parse_cache.py@@ -25,6 +25,7 @@ so deleted files age out naturally.from __future__ import annotationsimport contextlib+import dataclassesimport hashlibimport osimport pickle@@ -35,6 +36,7 @@ from typing import Anyimport structlog+from . import modelsfrom .models import FileInfo, ParsedFilelog = structlog.get_logger(__name__)@@ -45,16 +47,48 @@ _CACHE_FILENAME = "parse_cache.pkl"__all__ = ["ParseCache", "parser_fingerprint"]+def _models_schema_fingerprint(h: Any) -> None:+ """Fold the field shape of every cached dataclass into *h*.++ The cache pickles ``ParsedFile`` object graphs, so the fields of+ ``ParsedFile`` *and* every dataclass it nests (``Symbol``, ``Import``,+ ``CallSite``, ``FileInfo``, …) are part of the on-disk contract. When the+ code adds, removes, or retypes a field, an entry pickled by the old build+ deserializes into an object that is missing the attribute, and the next+ consumer that reads it crashes (``AttributeError: 'ParsedFile' object has+ no attribute '<new field>'``).++ Hashing the field set of every dataclass declared in :mod:`.models` makes+ such a change invalidate the cache automatically, without anyone having to+ remember to bump ``PARSER_SCHEMA_VERSION`` for a pure data-shape change.+ Iterating in name order keeps the digest stable across runs.+ """+ for name in sorted(vars(models)):+ obj = getattr(models, name)+ if not (isinstance(obj, type) and dataclasses.is_dataclass(obj)):+ continue+ if getattr(obj, "__module__", None) != models.__name__:+ continue # skip re-exports; only fingerprint models.py's own types+ h.update(f"dc:{name}".encode())+ for f in dataclasses.fields(obj):+ # ``str(f.type)`` is the annotation text (string under+ # ``from __future__ import annotations``); it changes whenever a+ # field's declared type changes.+ h.update(f"|{f.name}:{f.type!s}".encode())++@lru_cache(maxsize=1)def parser_fingerprint() -> str:"""Fingerprint of everything that determines a ParsedFile besides bytes.- Hashes every ``.scm`` query source (extraction rules) plus- ``PARSER_SCHEMA_VERSION`` (Python-side extraction logic). Deliberately- *not* the package ``__version__``: an unrelated release must not churn a- user's whole parse cache. The schema version is bumped only when parser /- extractor behaviour actually changes, so a mismatch still invalidates the- cache when correctness demands it. See :mod:`repowise.core.upgrade.version`.+ Hashes every ``.scm`` query source (extraction rules), the field shape of+ every cached dataclass in :mod:`.models` (so a ``ParsedFile`` schema change+ self-invalidates), plus ``PARSER_SCHEMA_VERSION`` (Python-side extraction+ logic). Deliberately *not* the package ``__version__``: an unrelated release+ must not churn a user's whole parse cache. The schema version is bumped only+ when parser / extractor *behaviour* changes (same fields, different values),+ so a mismatch still invalidates the cache when correctness demands it. See+ :mod:`repowise.core.upgrade.version`."""h = hashlib.sha256()h.update(f"cache-version:{_CACHE_VERSION}".encode())@@ -64,6 +98,10 @@ def parser_fingerprint() -> str:h.update(f"parser-schema:{PARSER_SCHEMA_VERSION}".encode())except Exception:h.update(b"parser-schema:unknown")+ try:+ _models_schema_fingerprint(h)+ except Exception:+ h.update(b"models-schema:unreadable")queries_dir = Path(__file__).parent / "queries"try:for scm in sorted(queries_dir.glob("*.scm")):diff --git a/tests/unit/ingestion/test_parse_cache.py b/tests/unit/ingestion/test_parse_cache.pyindex c85c153b0..9a298afd6 100644--- a/tests/unit/ingestion/test_parse_cache.py+++ b/tests/unit/ingestion/test_parse_cache.py@@ -143,6 +143,42 @@ def test_fingerprint_mismatch_invalidates(repo, tmp_path):assert cache.misses == 1+def test_dataclass_schema_change_invalidates_cache(repo, monkeypatch):+ """A ParsedFile (or nested dataclass) field change must self-invalidate.++ Regression: ``parse_cache.pkl`` pickles ParsedFile object graphs, so a new+ field on a cached dataclass made an old entry deserialize without the+ attribute, crashing the graph builder. The fingerprint now folds in every+ cached dataclass's field set, so the stale entry is treated as a miss.+ """+ import dataclasses++ from repowise.core.ingestion import models, parse_cache++ _build(repo) # populate the cache under the current fingerprint+ cache_file = _cache_path(repo)+ assert pickle.loads(cache_file.read_bytes())["fingerprint"] == parser_fingerprint()++ # Simulate a release that adds a field to ParsedFile.+ @dataclasses.dataclass+ class _Evolved(models.ParsedFile):+ new_field: int = 0++ _Evolved.__module__ = models.__name__ # masquerade as a models.py type+ monkeypatch.setattr(models, "ParsedFile", _Evolved)+ parse_cache.parser_fingerprint.cache_clear()+ try:+ assert parser_fingerprint() != pickle.loads(cache_file.read_bytes())["fingerprint"]++ cache = ParseCache(repo / ".repowise")+ cache.load()+ fi = _make_file_info(repo / "other.py", "other.py")+ source = (repo / "other.py").read_bytes()+ assert cache.get(fi, compute_content_hash(source)) is None+ finally:+ parse_cache.parser_fingerprint.cache_clear()++def _make_file_info(abs_path, rel_path):from datetime import UTC, datetime