Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(doc-drift): check what documents claim about the tree (#2290)
Raghav Chamadiya17d0cf74ebdd
diff --git a/packages/cli/src/repowise/cli/ui/progress.py b/packages/cli/src/repowise/cli/ui/progress.pyindex 6474eeb6f..ddf408c0c 100644--- a/packages/cli/src/repowise/cli/ui/progress.py+++ b/packages/cli/src/repowise/cli/ui/progress.py@@ -103,6 +103,7 @@ _PHASE_LABELS: dict[str, str] = {"dead_code": "Detecting dead code…","health": "Scoring code health…","decisions": "Extracting decisions…",+ "doc_drift": "Checking documentation drift…","knowledge_graph.skeleton": "Building the knowledge graph…","knowledge_graph.enrich": " ↳ Naming layers and building the tour","generation": "Generating pages…",diff --git a/packages/core/alembic/versions/0065_doc_drift_findings.py b/packages/core/alembic/versions/0065_doc_drift_findings.pynew file mode 100644index 000000000..40e26f568--- /dev/null+++ b/packages/core/alembic/versions/0065_doc_drift_findings.py@@ -0,0 +1,86 @@+"""Add doc_drift_findings table.++Stores assertions a document makes that the repository no longer satisfies: a+path that no longer resolves, a link whose target moved, an in-page anchor+whose heading was renamed, a build target the manifest does not declare.++``file_path`` is the document that is wrong, not the target it names, because+the document is what a reader has to edit.++The unique constraint over+``(repository_id, file_path, kind, line_number, target)`` is what makes+re-running the pass converge instead of accumulating duplicates. ``target``+belongs in the key because one line can carry two references: a markdown link+with a fragment yields both a ``link`` row and an ``anchor`` row at the same+line, and a key without ``target`` would collapse them into one.++Local SQLite stores never run Alembic --- ``init_db``'s reconciler calls+``Base.metadata.create_all``, which creates a brand-new table with its full+current schema --- so this migration exists for managed Postgres. No+reconciler change is needed for a new table; that gap is only for columns+added to tables that already exist.++Revision ID: 0065+Revises: 0064+Create Date: 2026-09-16+"""++from __future__ import annotations++from collections.abc import Sequence++import sqlalchemy as sa+from alembic import op++# revision identifiers+revision: str = "0065"+down_revision: str | None = "0064"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None+++def upgrade() -> None:+ op.create_table(+ "doc_drift_findings",+ sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),+ sa.Column(+ "repository_id",+ sa.String(32),+ sa.ForeignKey("repositories.id", ondelete="CASCADE"),+ nullable=False,+ ),+ sa.Column("file_path", sa.String(1024), nullable=False),+ sa.Column("kind", sa.String(32), nullable=False),+ sa.Column("line_number", sa.Integer, nullable=False),+ sa.Column("target", sa.String(1024), nullable=False),+ sa.Column("confidence", sa.Float, nullable=False, server_default="0"),+ sa.Column("reason", sa.Text, nullable=False, server_default=""),+ sa.Column("origin", sa.String(64), nullable=False, server_default=""),+ sa.Column("evidence_json", sa.Text, nullable=False, server_default="[]"),+ sa.Column("raw", sa.Text, nullable=False, server_default=""),+ sa.Column("context", sa.Text, nullable=False, server_default=""),+ sa.Column(+ "detected_at",+ sa.DateTime(timezone=True),+ nullable=False,+ server_default=sa.func.now(),+ ),+ sa.UniqueConstraint(+ "repository_id",+ "file_path",+ "kind",+ "line_number",+ "target",+ name="uq_doc_drift_finding_site",+ ),+ )+ op.create_index(+ "ix_doc_drift_repo_path",+ "doc_drift_findings",+ ["repository_id", "file_path"],+ )+++def downgrade() -> None:+ op.drop_index("ix_doc_drift_repo_path", table_name="doc_drift_findings")+ op.drop_table("doc_drift_findings")diff --git a/packages/core/src/repowise/core/analysis/doc_drift/__init__.py b/packages/core/src/repowise/core/analysis/doc_drift/__init__.pynew file mode 100644index 000000000..db1ff4cb4--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/__init__.py@@ -0,0 +1,28 @@+"""Documentation drift detection.++Documents make assertions about the repository. This package checks the ones+that are checkable, and is explicit about the ones that are not.++Four reference classes ship: ``path``, ``link``, ``anchor`` and ``command``.+A fifth, ``symbol``, was measured and rejected --- see+:class:`~.models.DriftKind`.+"""++from .analyzer import DocDriftAnalyzer, summarize_confidence+from .models import (+ DocDriftFindingData,+ DocDriftReport,+ DocReference,+ DriftKind,+ DriftVerdict,+)++__all__ = [+ "DocDriftAnalyzer",+ "DocDriftFindingData",+ "DocDriftReport",+ "DocReference",+ "DriftKind",+ "DriftVerdict",+ "summarize_confidence",+]diff --git a/packages/core/src/repowise/core/analysis/doc_drift/analyzer.py b/packages/core/src/repowise/core/analysis/doc_drift/analyzer.pynew file mode 100644index 000000000..be01a863a--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/analyzer.py@@ -0,0 +1,221 @@+"""The documentation drift pass.++Reads markdown the pipeline already decoded, extracts the assertions each+document makes about the repository, resolves them against the real tree, and+reports the ones the tree no longer satisfies.++Shaped like :class:`~repowise.core.analysis.dead_code.analyzer.DeadCodeAnalyzer`:+a synchronous class taking its inputs at construction and exposing+``analyze(config, *, on_step)``. The pipeline wraps it in ``asyncio.to_thread``.+"""++from __future__ import annotations++from datetime import UTC, datetime+from typing import Any++import structlog++from .constants import (+ DEFAULT_MIN_CONFIDENCE,+ MAX_DOC_BYTES,+ bucket_confidences,+)+from .extractor import extract, is_checkable_document, is_guide_document+from .models import (+ DocDriftFindingData,+ DocDriftReport,+ DriftKind,+ DriftVerdict,+)+from .resolver import RepoIndex, resolve++logger = structlog.get_logger(__name__)++_MANIFEST_NAMES = frozenset({"Makefile", "makefile", "GNUmakefile", "package.json"})++_REASONS: dict[str, str] = {+ "path_no_candidate": "Document names {target}, which no longer exists.",+ "path_no_candidate_in_guide": (+ "Document names {target}, which does not exist. This is a guide, so it "+ "may be an illustration rather than drift."+ ),+ "anchor_no_heading": "Link points at #{fragment}, which {doc} no longer declares.",+ "command_no_target": "Document shows `{raw}`, but no such target is declared.",+}+++class DocDriftAnalyzer:+ """Find assertions in a repository's own markdown that the tree refutes."""++ def __init__(+ self,+ repo_id: str = "",+ source_map: dict[str, bytes] | None = None,+ tracked_paths: set[str] | frozenset[str] | None = None,+ ) -> None:+ """+ Args:+ repo_id: repository this report speaks for. Left empty by the+ pipeline and supplied by the persistence layer, as dead code+ does --- the id is a storage concern the analysis does not+ need.+ source_map: raw bytes the ingestion phase already read, keyed by+ repo-relative path. Markdown is present here: it is a+ registered language spec, and the parser returns an empty+ ``ParsedFile`` for it rather than ``None``, so its bytes reach+ the map like any other file.+ tracked_paths: every path the index knows about. Defaults to the+ ``source_map`` keys, which are already gitignore-filtered by+ the traverser (and also honour ``.repowiseIgnore`` and+ per-directory ignores, which a bare gitignore spec does not).+ """+ self._repo_id = repo_id+ self._source_map = source_map or {}+ self._tracked = frozenset(tracked_paths or self._source_map.keys())++ # -- input preparation ------------------------------------------------++ def _decode(self, rel: str) -> str | None:+ raw = self._source_map.get(rel)+ if raw is None:+ return None+ # Reproduces the 500KB ``max_file_size_kb`` ceiling that already keeps+ # oversized non-AST files out of the index. Redundant on the normal+ # path (the traverser applied it first) and deliberately kept, so a+ # caller handing this analyzer a source_map it built itself cannot+ # walk into a 40MB generated document.+ if len(raw) > MAX_DOC_BYTES:+ return None+ return raw.decode("utf-8", errors="replace")++ def _documents(self) -> dict[str, str]:+ out: dict[str, str] = {}+ for rel in self._source_map:+ if not is_checkable_document(rel):+ continue+ text = self._decode(rel)+ if text is not None:+ out[rel] = text+ return out++ def _manifests(self) -> dict[str, str]:+ out: dict[str, str] = {}+ for rel in self._source_map:+ if rel.rsplit("/", 1)[-1] not in _MANIFEST_NAMES:+ continue+ text = self._decode(rel)+ if text is not None:+ out[rel] = text+ return out++ # -- the pass ---------------------------------------------------------++ def analyze(+ self,+ config: dict | None = None,+ *,+ on_step: Any | None = None,+ ) -> DocDriftReport:+ """Run the drift pass.++ Config keys: ``min_confidence`` (float), and ``check_<kind>`` booleans+ for each :class:`~.models.DriftKind`, all defaulting to on.+ """+ cfg = config or {}+ enabled = {+ kind for kind in DriftKind if cfg.get(f"check_{kind.value}", True)+ }++ documents = self._documents()+ if on_step:+ on_step("collect")++ index = RepoIndex.build(+ tracked_paths=self._tracked,+ doc_text=documents,+ manifest_text=self._manifests(),+ )+ if on_step:+ on_step("index")++ findings: list[DocDriftFindingData] = []+ verdict_counts: dict[str, int] = {v.value: 0 for v in DriftVerdict}+ references_checked = 0++ for rel, text in documents.items():+ is_guide = is_guide_document(rel)+ for ref in extract(text, rel):+ if ref.kind not in enabled:+ continue+ references_checked += 1+ res = resolve(index, ref, is_guide=is_guide)+ verdict_counts[res.verdict.value] += 1+ if res.verdict is DriftVerdict.MISSING:+ findings.append(_to_finding(res))+ if on_step:+ on_step("resolve")++ min_conf = float(cfg.get("min_confidence", DEFAULT_MIN_CONFIDENCE))+ hidden = sum(1 for f in findings if f.confidence < min_conf)+ findings = [f for f in findings if f.confidence >= min_conf]+ findings.sort(key=lambda f: (-f.confidence, f.file_path, f.line_number))++ logger.debug(+ "doc_drift_complete",+ documents=len(documents),+ references=references_checked,+ findings=len(findings),+ renderer=index.renderers.summary(),+ )++ return DocDriftReport(+ repo_id=self._repo_id,+ analyzed_at=datetime.now(UTC),+ total_findings=len(findings),+ findings=findings,+ confidence_summary=summarize_confidence(findings),+ documents_scanned=len(documents),+ references_checked=references_checked,+ verdict_summary=verdict_counts,+ anchor_renderer=index.renderers.summary(),+ hidden_below_threshold=hidden,+ )+++def _to_finding(res: Any) -> DocDriftFindingData:+ ref = res.ref+ target, _, fragment = ref.target.partition("#")+ reason = _REASONS[res.origin].format(+ target=target or ref.target,+ fragment=fragment,+ doc=target or ref.doc_path,+ raw=ref.raw,+ )+ evidence = [+ f"{ref.doc_path}:{ref.line} states `{ref.raw}`",+ f"resolution: {res.detail}",+ ]+ if ref.section:+ evidence.append(f"under: {ref.section}")+ return DocDriftFindingData(+ kind=ref.kind,+ file_path=ref.doc_path,+ line_number=ref.line,+ target=ref.target,+ confidence=res.confidence,+ reason=reason,+ origin=res.origin,+ evidence=evidence,+ raw=ref.raw,+ context=ref.context[:300],+ )+++def summarize_confidence(findings: list[DocDriftFindingData]) -> dict:+ """Bucket *findings* into the house high/medium/low tiers.++ Same boundaries as dead code, so the word "high" means the same thing in+ both reports. Pinned by ``tests/unit/doc_drift/test_confidence_parity.py``.+ """+ return bucket_confidences(f.confidence for f in findings)diff --git a/packages/core/src/repowise/core/analysis/doc_drift/constants.py b/packages/core/src/repowise/core/analysis/doc_drift/constants.pynew file mode 100644index 000000000..9ac1e6bbf--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/constants.py@@ -0,0 +1,165 @@+"""Tunables and vocabularies for the documentation drift detector.++One module so the analyzer, the CRUD layer, the CLI and any later consumer+import the same numbers. Dead code learned this the hard way: its bucket+boundaries live in ``dead_code/risk_factors.py`` precisely because three+surfaces had already drifted on them+(``tests/unit/dead_code/test_confidence_parity.py``).+"""++from __future__ import annotations++import re+from typing import Literal, get_args++from repowise.core.ingestion.languages.registry import REGISTRY++# ---------------------------------------------------------------------------+# Confidence buckets+#+# Deliberately the same boundaries as dead code's ``SAFE_CONFIDENCE_THRESHOLD``+# / ``RISK_CAP_CONFIDENCE``, so "high" means the same thing to a reader looking+# at either report. Pinned by ``tests/unit/doc_drift/test_confidence_parity.py``+# rather than imported, because coupling this analysis to dead code's risk+# vocabulary would make a change to *its* deletion policy silently retune+# *this* detector.+# ---------------------------------------------------------------------------++HIGH_CONFIDENCE_THRESHOLD: float = 0.7+REVIEW_CONFIDENCE_THRESHOLD: float = 0.4+++def bucket_confidences(values) -> dict:+ """Bucket an iterable of confidences into the house high/medium/low tiers.++ One implementation, called by the analyzer when it builds a report and by+ the CRUD layer when it re-derives the summary on read. They were two+ copies of the same three comprehensions; a shared function is cheaper than+ the test that would have had to pin them together.+ """+ high = medium = low = 0+ for value in values:+ if value >= HIGH_CONFIDENCE_THRESHOLD:+ high += 1+ elif value >= REVIEW_CONFIDENCE_THRESHOLD:+ medium += 1+ else:+ low += 1+ return {"high": high, "medium": medium, "low": low}+++#: Default ``--min-confidence`` cutoff. Matches the lowest shipping origin's+#: tier so nothing is hidden by default; a caller that wants only the+#: near-certain findings raises it to ``HIGH_CONFIDENCE_THRESHOLD``.+DEFAULT_MIN_CONFIDENCE: float = REVIEW_CONFIDENCE_THRESHOLD+++# ---------------------------------------------------------------------------+# Resolution origins+#+# The ``ResolutionOrigin`` pattern from ``ingestion/models.py:407``: one named+# origin per way a verdict was reached, each carrying a fixed confidence. Phase+# 1 justified this over the coarser ``measured | inferred`` two-tier flag+# because the classes have measurably different precision (anchor 3/3, path+# 11/12) and because the one path false positive has a *nameable* shape --- an+# invented example inside a contributor guide --- which is a tier, not a rule.+#+# Confidences live in ``ORIGIN_CONFIDENCE`` below rather than in trailing+# comments, so the number a finding carries and the number documented here+# cannot disagree. A parity test asserts the two vocabularies stay in step.+# ---------------------------------------------------------------------------++DriftOrigin = Literal[+ "path_no_candidate",+ "path_no_candidate_in_guide",+ "anchor_no_heading",+ "command_no_target",+]++ORIGIN_CONFIDENCE: dict[str, float] = {+ # An anchored path whose basename appears nowhere in the tree. The bulk of+ # Phase 1's real findings (11/12), and the single most common drift shape:+ # a module became a package and the prose still names the ``.py``.+ "path_no_candidate": 0.90,+ # The same evidence, inside a document that teaches rather than describes.+ # Phase 1's only path false positive was ``languages/specs/mylang.py`` in a+ # guide on how to add a language: an invented path, correctly shaped, that+ # was never meant to exist. One tier down, not excluded --- a how-to guide+ # naming a real file that moved is still drift.+ "path_no_candidate_in_guide": 0.50,+ # A heading that was renamed with the inbound link left behind. 3/3 real on+ # this repo, and a class no linter catches: each document is internally+ # valid and the target file exists. Only the pair is wrong. Only ever+ # reached when the renderer is known to slug GitHub-style.+ "anchor_no_heading": 0.95,+ # A ``make``/``npm run`` target, written inside an inline code span, that+ # the manifest in the tree does not declare.+ "command_no_target": 0.85,+}++#: Every origin in the ``DriftOrigin`` vocabulary, for parity checks.+DRIFT_ORIGIN_VALUES: frozenset[str] = frozenset(get_args(DriftOrigin))+++# ---------------------------------------------------------------------------+# Document selection+# ---------------------------------------------------------------------------++#: Extensions this detector can read, taken from the markdown ``LanguageSpec``+#: rather than restated, so the set of files the traverser calls markdown and+#: the set this detector scans cannot diverge.+#:+#: ``.rst`` and ``.adoc`` are deliberately outside this: flask was in the Phase+#: 1 corpus and contributed nothing because its docs are reStructuredText.+#: Those trees are beyond this detector's reach, and pretending otherwise would+#: mean shipping an untested extractor.+MARKDOWN_SUFFIXES: frozenset[str] = REGISTRY.get("markdown").extensions++#: Path segments holding code this repository did not write. Test and fixture+#: trees are NOT listed here: they are recognised by+#: :func:`repowise.core.test_paths.is_test_related_path`, the same helper that+#: stamps ``FileInfo.is_test`` at ingestion, so this detector and the rest of+#: the engine cannot disagree about what a test is. Phase 2 found why that+#: matters: fastapi keeps deliberately-broken markdown at+#: ``scripts/tests/test_translation_fixer/.../data/*.md`` as input fixtures for+#: its own link fixer, and a hand-written directory list did not recognise the+#: shape.+VENDORED_DIR_PARTS: frozenset[str] = frozenset(+ {+ "node_modules",+ "vendor",+ "third_party",+ "site-packages",+ ".venv",+ "venv",+ "dist",+ "build",+ }+)++# A changelog is a historical record. Every entry describes the tree as it was+# at that release, so a reference that no longer resolves is the document doing+# its job; checking one is a category error.+#+# Matched on the STEM AS A PATTERN, never an exact filename list. Phase 1's+# first fixture was named ``CHANGELOG.md``, which silently failed on fastapi's+# ``release-notes.md`` --- and all 17 of that repo's path findings came from it.+HISTORICAL_STEM_RE = re.compile(+ r"^(changelog|changes|history|news|release[-_]?notes?|releases|whatsnew|"+ r"what[-_]s[-_]new|migration[-_]?guide|upgrading)$",+ re.I,+)++# Documents that teach rather than describe. A path in one of these is more+# likely to be an illustration the author invented, so an unresolved reference+# here is stamped with the lower ``path_no_candidate_in_guide`` tier.+GUIDE_STEM_RE = re.compile(+ r"(contributing|tutorial|guide|guides|how[-_]?to|howto|getting[-_]?started|"+ r"quickstart|walkthrough|cookbook|recipes|example|examples|template)",+ re.I,+)++#: Ceiling for a single document, mirroring the 500KB ``max_file_size_kb``+#: that already applies to non-AST files. A document larger than this is not+#: prose anyone maintains.+MAX_DOC_BYTES: int = 500 * 1024diff --git a/packages/core/src/repowise/core/analysis/doc_drift/extractor.py b/packages/core/src/repowise/core/analysis/doc_drift/extractor.pynew file mode 100644index 000000000..741eabd22--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/extractor.py@@ -0,0 +1,429 @@+"""Extraction of checkable references from markdown prose.++Why this is a third implementation rather than a reuse of the two that exist+---------------------------------------------------------------------------+``generation/page_generator/validation.py:373`` and+``generation/interlinking.py:32`` both regex backtick-quoted refs out of+markdown, and the second says in a comment that it is reusing the first. A+shared extractor was evaluated for this third caller and rejected, because the+three ask different questions:++* ``validation`` checks *generated* prose against the single ``ParsedFile`` the+ page was written about, and its regex matches identifiers only --- it cannot+ express a path, having no ``/`` in its character class. It is a symbol+ checker, and the symbol class is exactly what Phase 1 killed.+* ``interlinking`` resolves refs to ``page_id``s and strips fences with one+ ``re.DOTALL`` pass over the whole document, which destroys line positions.+* This detector files **line-level** findings against **user-authored** prose,+ checked against the **whole tree**. It needs line numbers (the persistence+ key includes one), markdown links, URL fragments and inline commands --- three+ of its four classes are things neither of the others extracts at all.++Folding them together would mean one function with three incompatible regexes+and a mode flag. The honest answer is a third extractor that shares the+vocabulary (:mod:`.models`) rather than the code. Recorded here so the next+reader does not re-litigate it.+"""++from __future__ import annotations++import re+from collections.abc import Iterator+from dataclasses import replace+from pathlib import Path++from repowise.core.ingestion.languages.registry import REGISTRY+from repowise.core.test_paths import is_test_related_path++from .constants import (+ GUIDE_STEM_RE,+ HISTORICAL_STEM_RE,+ MARKDOWN_SUFFIXES,+ VENDORED_DIR_PARTS,+)+from .models import DocReference, DriftKind++# ---------------------------------------------------------------------------+# Span handling+#+# Two kinds of span must never contribute a reference:+#+# 1. Fenced code blocks. Their paths and symbols are illustrative by+# construction. The decision extractor already tracks fences for the same+# reason (``decisions/extractor.py:475``).+# 2. Repowise's own generated blocks. ``editor_files/base.py:28`` wraps them in+# START / END HTML comment markers, so they are skippable exactly. The+# traverser deliberately does NOT drop generated markdown+# (``traverser.py:804`` exempts passthrough languages from the check), so+# this is ours to do.+#+# The START marker carries a trailing sentence and an em dash, so it is matched+# on its prefix rather than the full literal.+# ---------------------------------------------------------------------------++_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})")+# Hyphens belong in the tag: ``ALL-CONTRIBUTORS-LIST:START`` is the commonest+# generated block in the wild, and it is a wall of avatar image links.+_GENERATED_START_RE = re.compile(r"<!--\s*[A-Z][A-Z0-9_-]*:START\b")+_GENERATED_END_RE = re.compile(r"<!--\s*[A-Z][A-Z0-9_-]*:END\b")+++def prose_lines(text: str) -> Iterator[tuple[int, str]]:+ """Yield ``(1-indexed line number, line)`` for prose lines only.++ Line numbers stay absolute across skipped spans, because a finding's line+ has to point at the real file.+ """+ in_fence = False+ fence_marker = ""+ in_generated = False+ # Split on newlines only. ``str.splitlines()`` also breaks on the+ # vertical tab, form feed, NEL and U+2028/U+2029, which git and every+ # editor count as ordinary characters. A document carrying one of those+ # (they arrive routinely by copy-paste from the web) would shift every+ # later line number, and ``line_number`` is both the persistence key and+ # the only thing that makes a finding actionable.+ for i, raw_line in enumerate(text.split("\n"), start=1):+ line = raw_line.rstrip("\r")++ m = _FENCE_RE.match(line)+ if m:+ marker = m.group(1)[:3]+ if not in_fence:+ in_fence, fence_marker = True, marker+ elif marker == fence_marker:+ # Only the marker that opened the fence can close it. A ``~~~``+ # inside a ``` block is content, and treating it as a close+ # inverts the state for the rest of the document.+ in_fence, fence_marker = False, ""+ continue++ if in_fence:+ # Checked before the generated-marker tests, not after. A document+ # that *shows* a START marker in an example --- this project's own+ # docs do --- would otherwise turn the skip on with no matching END+ # outside the fence, silently discarding the rest of the file.+ continue++ if _GENERATED_START_RE.search(line):+ # A START and its END on one line opens and closes nothing.+ in_generated = not _GENERATED_END_RE.search(line)+ continue+ if _GENERATED_END_RE.search(line):+ in_generated = False+ continue+ if in_generated:+ continue+ yield i, line+++# ---------------------------------------------------------------------------+# Shared exclusions+# ---------------------------------------------------------------------------++_URL_RE = re.compile(r"^(https?|mailto|ftp|tel|data):", re.I)++# Tokens that are placeholders by convention. Flagging one means flagging the+# author's illustration.+_PLACEHOLDER_SEGMENTS = frozenset(+ {+ "path",+ "to",+ "your",+ "my",+ "some",+ "any",+ "example",+ "examples",+ "foo",+ "bar",+ "baz",+ "qux",+ "xxx",+ "yyy",+ "name",+ "project",+ "repo",+ "user",+ "username",+ "org",+ "owner",+ "dir",+ "folder",+ }+)++# Shell or templating metacharacters mean the token is a pattern, not a path.+_TEMPLATED_RE = re.compile(r"[{}$<>*?|\\\[\]()]|\.\.\.|\bTODO\b")++# Extensions that make a slashed token a path assertion rather than prose.+#+# Derived from the LanguageRegistry rather than hand-listed, following+# ``ingestion/git_indexer/_constants.py``. A hand-written list started 64+# extensions short of what the engine already recognised --- every markdown+# reference to a ``.lua``, ``.zig``, ``.jl`` or ``.kts`` file was dropped+# before resolution, so it never even reached the uncheckable count that makes+# this detector's coverage honest. It also disagreed with this module's own+# ``MARKDOWN_SUFFIXES``: ``.markdown`` was scannable as a document but+# unresolvable as a link target.+#+# The residue below is the genuinely unregistered remainder: data, asset and+# lockfile extensions that name no language.+_UNREGISTERED_EXTS = frozenset(+ {+ ".cfg", ".css", ".env", ".gif", ".ini", ".jpg", ".lock", ".png",+ ".ps1", ".rst", ".scss", ".svg", ".txt", ".xml",+ }+)+KNOWN_EXTS: frozenset[str] = frozenset(REGISTRY.all_extensions()) | _UNREGISTERED_EXTS+++def _is_placeholder(token: str) -> bool:+ segs = [s.lower() for s in token.strip("/").split("/") if s]+ if not segs:+ return True+ # Only the FIRST segment decides. Checking the first two swallowed real+ # paths wholesale --- ``app/user/models.py``, ``src/example/main.py`` ---+ # because ``user``, ``name``, ``org`` and ``example`` are ordinary+ # second-level directory names. The classic ``path/to/...`` shape is still+ # caught by its own first segment, and anything else that slips through+ # meets the anchoring rule in the resolver.+ return segs[0] in _PLACEHOLDER_SEGMENTS+++def _path_shaped(token: str) -> bool:+ """True when a token asserts a repository path rather than being prose."""+ if not token or _URL_RE.match(token) or _TEMPLATED_RE.search(token):+ return False+ if token.startswith(("#", "~")) or token.startswith("//"):+ return False+ if _is_placeholder(token):+ return False+ base = token.rstrip("/").split("/")[-1]+ ext = base[base.rfind(".") :].lower() if "." in base else ""+ if "/" in token:+ # A slashed token needs a known extension or an explicit directory+ # shape, so prose like "and/or" never qualifies.+ return ext in KNOWN_EXTS or token.endswith("/")+ # A bare filename only counts with a known extension, so "e.g" and "i.e"+ # never qualify. The resolver calls it uncheckable regardless; keeping it+ # out here saves the work.+ return ext in KNOWN_EXTS and len(base) > len(ext) + 1+++def _normalize(token: str) -> str:+ """Trim a reference to the path it names, without altering that path.++ ``lstrip("./")`` was wrong twice over: it strips *characters*, not a+ prefix, so ``.github/workflows/ci.yml`` became ``github/...`` (mis-keyed,+ then dropped as unanchored, and written to the database as something the+ document does not say), and ``../../docs/a.md`` lost the ``../`` segments+ the resolver needs to join the link against its own document.+ """+ t = token.strip().strip("`\"'")+ while t.startswith("./"):+ t = t[2:]+ t = t.lstrip("/")+ # Sentence punctuation that followed the reference in prose.+ #+ # The trailing slash goes with it, which keeps directory references out+ # of the detector. That is deliberate for this phase: Phase 1 measured+ # file references only, and admitting directories produced five findings+ # that were all illustration --- a document listing the conventional ADR+ # directory names (``docs/adr/``, ``docs/adrs/``, ``docs/decisions/``) is+ # naming a convention, not claiming this repository has one. Giving them+ # a measured class's confidence would be a claim the evidence does not+ # support.+ return t.rstrip(".,;:)").rstrip("/")+++# ---------------------------------------------------------------------------+# Per-class extractors+#+# One function per class, all yielding DocReference, so a class can be retuned+# or dropped without touching the others.+# ---------------------------------------------------------------------------++_INLINE_CODE_RE = re.compile(r"(?<!`)`([^`\n]{2,120})`(?!`)")+_MD_LINK_RE = re.compile(r"\[[^\]\n]{0,200}?\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)")++# Commands whose target is checkable against a manifest in the tree.+_COMMAND_RE = re.compile(+ r"\b(?:(make)\s+([A-Za-z0-9][\w./-]*)"+ r"|(npm|pnpm|yarn)\s+run\s+([A-Za-z0-9][\w:.-]*))"+)+++_HEADING_LINE_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")+++class _SectionTrail:+ """The heading trail above the line currently being read.++ Kept as a stack by heading level so an ancestor still counts: a reference+ under a plain ``#### Step 1`` inside ``## Adding a new language`` inherits+ the parent's instructional character, and a later ``## Architecture`` at+ the same level pops it again.+ """++ __slots__ = ("_stack",)++ def __init__(self) -> None:+ self._stack: list[tuple[int, str]] = []++ def feed(self, line: str) -> bool:+ """Consume *line*; return True when it was a heading."""+ m = _HEADING_LINE_RE.match(line)+ if not m:+ return False+ level = len(m.group(1))+ while self._stack and self._stack[-1][0] >= level:+ self._stack.pop()+ self._stack.append((level, m.group(2).strip()))+ return True++ def trail(self) -> str:+ return " > ".join(title for _, title in self._stack)+++def _from_links(line: str, lineno: int, doc: str) -> Iterator[DocReference]:+ """Markdown links: the LINK class, and the ANCHOR class from fragments."""+ for m in _MD_LINK_RE.finditer(line):+ raw = m.group(1)+ if _URL_RE.match(raw):+ continue+ target, _, frag = raw.partition("#")+ target = _normalize(target)+ if frag and not target:+ # Same-document anchor, checkable against this file's own headings.+ yield DocReference(+ DriftKind.ANCHOR, raw, f"{doc}#{frag}", doc, lineno, line.strip()+ )+ continue+ if not target or not _path_shaped(target):+ continue+ if frag:+ yield DocReference(+ DriftKind.ANCHOR, raw, f"{target}#{frag}", doc, lineno, line.strip()+ )+ yield DocReference(DriftKind.LINK, raw, target, doc, lineno, line.strip())+++def _from_inline_code(line: str, lineno: int, doc: str) -> Iterator[DocReference]:+ """Backtick-quoted paths: the PATH class.++ Only paths. A backtick-quoted identifier is NOT emitted as a symbol+ reference; see :class:`~.models.DriftKind` for why that class does not+ exist.+ """+ for m in _INLINE_CODE_RE.finditer(line):+ raw = m.group(1).strip()+ norm = _normalize(raw)+ if _path_shaped(norm):+ yield DocReference(DriftKind.PATH, raw, norm, doc, lineno, line.strip())+++def _from_commands(line: str, lineno: int, doc: str) -> Iterator[DocReference]:+ """Build-tool invocations: the COMMAND class.++ Read ONLY from inside inline code spans. Scanning bare prose matches+ English --- Phase 1's first run produced "make room", "make that", "make+ the" and "make no" as missing build targets, 43 of 55 findings and entirely+ noise. A command is a command when the author marked it as one.+ """+ for span in _INLINE_CODE_RE.finditer(line):+ for m in _COMMAND_RE.finditer(span.group(1)):+ target = f"make:{m.group(2)}" if m.group(1) else f"npm:{m.group(4)}"+ yield DocReference(+ DriftKind.COMMAND, m.group(0), target, doc, lineno, line.strip()+ )+++_EXTRACTORS = (_from_links, _from_inline_code, _from_commands)+++def extract(text: str, doc_path: str) -> list[DocReference]:+ """Every checkable assertion *text* makes, deduplicated, in document order."""+ seen: set[DocReference] = set()+ out: list[DocReference] = []+ trail = _SectionTrail()+ for lineno, line in prose_lines(text):+ # A heading updates the trail AND is still scanned: a section title+ # like "### 7.1 GitIndexer (`packages/core/ingestion/git_indexer.py`)"+ # asserts a path exactly as body prose does, and skipping headings+ # silently dropped one of this repository's real defects.+ trail.feed(line)+ section = trail.trail()+ for extractor in _EXTRACTORS:+ for ref in extractor(line, lineno, doc_path):+ ref = replace(ref, section=section)+ if ref in seen:+ continue+ seen.add(ref)+ out.append(ref)+ return out+++++# ---------------------------------------------------------------------------+# Document selection+# ---------------------------------------------------------------------------+++def is_checkable_document(rel_path: str) -> bool:+ """Whether *rel_path* is prose this repository wrote about itself.++ Excludes historical records by stem pattern (see+ :data:`~.constants.HISTORICAL_STEM_RE` for why a filename list is not+ enough), vendored trees, and anything the engine already classes as test+ or test support.++ Test documents are excluded through the shared+ :func:`~repowise.core.test_paths.is_test_related_path` rather than a local+ directory list, because a fixture is a fixture in more shapes than a list+ anticipates: fastapi stores deliberately-broken markdown under+ ``scripts/tests/.../data/`` to feed its own link fixer, and checking it+ reports the project's test inputs as its documentation.+ """+ p = Path(rel_path)+ if p.suffix.lower() not in MARKDOWN_SUFFIXES:+ return False+ if HISTORICAL_STEM_RE.match(p.stem):+ return False+ if set(p.parts) & VENDORED_DIR_PARTS:+ return False+ return not _in_test_tree(rel_path)+++def _in_test_tree(rel_path: str) -> bool:+ """Whether *rel_path* sits inside a test or fixture directory.++ Asks the shared classifier about the containing DIRECTORY rather than the+ file, because ``is_test_related_path`` also recognises the ``test_*``+ filename convention --- and that is a convention of *code*, not prose.+ Applied to the whole path it classes ``docs/layers/TEST_INTELLIGENCE.md``,+ a document describing the test-intelligence layer, as a test file and+ silently drops one of this repository's real defects.++ Probing with a neutral filename keeps the directory half of the+ classifier, which is the half that matters here: it still recognises+ ``tests/fixtures/...`` and fastapi's+ ``scripts/tests/.../data/`` without guessing at either.+ """+ parent = rel_path.rsplit("/", 1)[0] if "/" in rel_path else ""+ probe = f"{parent}/_.md" if parent else "_.md"+ return is_test_related_path(probe)+++def is_guide_document(rel_path: str) -> bool:+ """Whether *rel_path* teaches rather than describes.++ A path that resolves nowhere is likelier to be an invented illustration+ here, so the finding is stamped one confidence tier lower. This is a tier,+ not an exclusion: a tutorial naming a real file that moved is still drift.+ """+ # The STEM, not the whole path: searching the path meant a single+ # directory named ``examples/`` or ``templates/`` anywhere above a document+ # halved the confidence of every finding beneath it.+ return bool(GUIDE_STEM_RE.search(Path(rel_path).stem))diff --git a/packages/core/src/repowise/core/analysis/doc_drift/models.py b/packages/core/src/repowise/core/analysis/doc_drift/models.pynew file mode 100644index 000000000..db1d085e5--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/models.py@@ -0,0 +1,131 @@+"""Data model for documentation drift findings.++Shaped after dead code (``analysis/dead_code/models.py``) rather than health:+a drift finding is a claim about *existence* carried by a continuous+``confidence``, not an impact ranked by severity. Health's severity-first shape+expresses "how much does this hurt", which is the wrong axis for "is this+sentence still true".++As in dead code, the models carry no serializer. Each consumer (CLI, CRUD,+later surfaces) builds the dict it needs.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from datetime import datetime+from enum import StrEnum+++class DriftKind(StrEnum):+ """The reference classes this detector ships.++ ``SYMBOL`` is deliberately absent. Phase 1 measured it at a 55-69% flag+ rate across two repositories and killed it: backticks in technical prose+ mean "this is a literal token", not "this is a code symbol". The top+ flagged tokens were ``string``, ``boolean``, ``true``, ``OPENAI_API_KEY``,+ ``PATH`` and ``HEAD``. The extraction premise is wrong, so a better graph+ would not rescue it.+ """++ PATH = "path"+ LINK = "link"+ ANCHOR = "anchor"+ COMMAND = "command"+++class DriftVerdict(StrEnum):+ """The four answers a resolver may give about one reference.++ ``UNCHECKABLE`` is first-class and is the whole difference between a+ detector and a noise generator: 684 of 903 path references on this+ repository are uncheckable, and 100% of the probe's first-run false+ positives came from reporting them as drift. ``AMBIGUOUS`` is likewise+ never collapsed into ``MISSING`` --- a reference matching several+ candidates is not evidence of drift.+ """++ RESOLVED = "resolved"+ MISSING = "missing"+ AMBIGUOUS = "ambiguous"+ UNCHECKABLE = "uncheckable"+++@dataclass(frozen=True)+class DocReference:+ """One checkable assertion a document makes about the repository.++ Frozen and hashable so the extractor can deduplicate without a side table.+ Every extractor yields this same shape, which is what keeps the resolver+ class-agnostic and lets a fifth class be added without touching the other+ four.+ """++ kind: DriftKind+ raw: str+ """The reference exactly as written, for display."""+ target: str+ """Normalized lookup key. For ``ANCHOR`` this is ``"<doc path>#<fragment>"``."""+ doc_path: str+ """Repo-relative POSIX path of the document making the assertion."""+ line: int+ """1-indexed line within the document."""+ context: str+ """The source line, trimmed, so a reader can judge the finding without+ opening the file."""+ section: str = ""+ """The enclosing heading trail, joined by " > ". Used to tell prose that+ *describes* the repository from prose that *teaches* the reader to add to+ it, which is a confidence tier rather than an exclusion."""+++@dataclass+class DocDriftFindingData:+ """A document assertion the repository no longer satisfies."""++ kind: DriftKind+ file_path: str+ """The *document* that is wrong, not the target it names. This is the file+ a reader has to edit, so it is the one the finding is filed against."""+ line_number: int+ target: str+ """What the document claims exists."""+ confidence: float+ reason: str+ """One human sentence."""+ origin: str+ """Which named resolution strategy produced this, from+ :data:`~.constants.DriftOrigin`. Every origin carries a fixed confidence,+ so the origin distribution and the confidence histogram are two views of+ the same data."""+ evidence: list[str]+ """Lines a reader can check for themselves."""+ raw: str = ""+ context: str = ""+++@dataclass+class DocDriftReport:+ """The drift pass's output for one repository."""++ repo_id: str+ analyzed_at: datetime+ total_findings: int+ findings: list[DocDriftFindingData]+ confidence_summary: dict+ """``{"high": N, "medium": N, "low": N}`` over the post-cutoff findings."""+ documents_scanned: int+ references_checked: int+ verdict_summary: dict+ """``{"resolved": N, "missing": N, "ambiguous": N, "uncheckable": N}`` over+ every reference examined. Reported because the uncheckable count is the+ honest denominator: a detector that hides it is claiming coverage it does+ not have."""+ anchor_renderer: str = "github"+ """Which renderer's slug algorithm the anchor class used, or the renderer+ that caused it to stand down. See :mod:`~.renderer`."""+ hidden_below_threshold: int = 0+ authoritative_paths: frozenset[str] | None = field(default=None)+ """Document paths this report may speak for; ``None`` means all of them.+ Mirrors dead code's field of the same name so an incremental re-check can+ replace a subset of rows without deleting findings it did not recompute."""diff --git a/packages/core/src/repowise/core/analysis/doc_drift/renderer.py b/packages/core/src/repowise/core/analysis/doc_drift/renderer.pynew file mode 100644index 000000000..a2b2c3a9f--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/renderer.py@@ -0,0 +1,172 @@+"""Which renderer turns a given document's headings into anchors.++Anchor checking is the one class that does not generalize. A heading's anchor+is whatever the *renderer* decides it is, and renderers disagree. Phase 1 ran a+GitHub slug algorithm over fastapi and produced 93 anchor findings, every one+of them false: fastapi's docs are MkDocs, which slugs differently. On this+repository the same algorithm was 3/3 correct, because GitHub is what actually+renders those files.++So the class is gated. If a document is published through a renderer whose slug+algorithm this module does not implement, the anchor class stands down for that+document and reports ``uncheckable`` rather than guessing.++The gate is **per subtree, not per repository.** This repository is why: it+ships a Jekyll site at ``website/_config.yml`` that publishes the fifteen+documents under ``website/`` and nothing else. The ``docs/`` tree and+``.github/CONTRIBUTING.md`` beside it are read on GitHub. A repo-global gate+reads one marker file and stands the whole class down, losing every real+finding outside the site directory --- which is exactly what happened the first+time this ran. A marker governs the directory that contains it, and documents+outside every marked subtree are rendered by the forge.++Adding MkDocs or Docusaurus support later means implementing its slug function+here and dropping its tag from :data:`UNSUPPORTED_RENDERERS`. No other module+changes.+"""++from __future__ import annotations++import re+from collections.abc import Iterable++#: Renderer whose slug algorithm is implemented and measured.+GITHUB = "github"++#: Renderers this module can *detect* but cannot slug for. Detecting one is+#: what makes the anchor class stand down for the documents it governs; the+#: alternative is fastapi's 93 false findings.+UNSUPPORTED_RENDERERS: frozenset[str] = frozenset({"mkdocs", "docusaurus", "jekyll"})++#: Marker filename -> renderer tag. A marker governs the directory it sits in+#: and everything beneath it.+_MARKERS: dict[str, str] = {+ "mkdocs.yml": "mkdocs",+ "mkdocs.yaml": "mkdocs",+ "docusaurus.config.js": "docusaurus",+ "docusaurus.config.ts": "docusaurus",+ "docusaurus.config.mjs": "docusaurus",+ "_config.yml": "jekyll",+}+++class RendererMap:+ """Which renderer publishes each document.++ Built once per run from the tracked paths; queried per document.+ """++ __slots__ = ("_zones",)++ def __init__(self, zones: list[tuple[str, str]]) -> None:+ #: ``(prefix, renderer)`` ordered longest-prefix first, so a site+ #: nested inside another site wins for its own documents.+ self._zones = zones++ @classmethod+ def detect(cls, tracked_paths: Iterable[str]) -> RendererMap:+ """Find every renderer marker in the tree and the subtree it governs.++ A marker buried in ``node_modules`` or a test fixture says nothing+ about how this project publishes its own prose, so only markers+ outside those trees count. Import is local to avoid a cycle:+ :mod:`.constants` does not depend on this module.+ """+ from repowise.core.test_paths import is_test_related_path++ from .constants import VENDORED_DIR_PARTS++ zones: list[tuple[str, str]] = []+ for path in tracked_paths:+ prefix, _, name = path.rpartition("/")+ renderer = _MARKERS.get(name)+ if renderer is None:+ continue+ if prefix and set(prefix.split("/")) & VENDORED_DIR_PARTS:+ continue+ if is_test_related_path(path):+ continue+ zones.append((f"{prefix}/" if prefix else "", renderer))+ # Longest prefix first. The repo root ("") sorts last and so acts as+ # the fallback zone when a site is configured at the root.+ zones.sort(key=lambda z: len(z[0]), reverse=True)+ return cls(zones)++ def for_document(self, doc_path: str) -> str:+ """Name the renderer that publishes *doc_path*.++ Defaults to :data:`GITHUB`, which is the honest answer for a document+ no site generator picks up: it is read on the forge.+ """+ for prefix, renderer in self._zones:+ if not prefix or doc_path.startswith(prefix):+ return renderer+ return GITHUB++ def checkable(self, doc_path: str) -> bool:+ """Whether the anchor class may produce findings about *doc_path*."""+ return self.for_document(doc_path) == GITHUB++ def summary(self) -> str:+ """A short label for the report, naming what was detected.++ ``"github"`` when nothing else was found, otherwise the detected+ renderers and the subtrees they cover, so a reader can see why an+ anchor finding they expected is absent.+ """+ if not self._zones:+ return GITHUB+ return ",".join(f"{r}:{p or '/'}" for p, r in self._zones)+++# ---------------------------------------------------------------------------+# GitHub's slug algorithm+# ---------------------------------------------------------------------------++_MARKUP_RE = re.compile(r"[`*~\[\]()]")+_NON_WORD_RE = re.compile(r"[^\w\s-]")+_WHITESPACE_RE = re.compile(r"\s")++# A heading is slugged from its RENDERED text, so these have to be reduced to+# their visible part before anything else runs. Stripping only the punctuation+# keeps the invisible half: ``## [Contributing](CONTRIBUTING.md)`` slugged to+# ``contributingcontributingmd`` where GitHub produces ``contributing``, and+# ``## <code>foo</code>`` to ``codefoocode`` rather than ``foo``. Both shapes+# are ordinary in a README, and both produced findings at the highest+# confidence this detector issues.+_MD_LINK_TEXT_RE = re.compile(r"\[([^\]]*)\]\([^)]*\)")+_MD_IMAGE_RE = re.compile(r"!\[[^\]]*\]\([^)]*\)")+_HTML_TAG_RE = re.compile(r"<[^>]+>")++#: Splits a heading on inline code spans, so their contents land on the odd+#: indices and can be passed through untouched.+_CODE_SPAN_RE = re.compile(r"`([^`]*)`")+++def github_slug(heading: str) -> str:+ """GitHub's heading-anchor slug for *heading*.++ Each whitespace character becomes its own dash. Collapsing runs instead+ turns ``SQL + dbt`` into ``sql-dbt`` where GitHub produces ``sql--dbt``,+ because the ``+`` is stripped and the two spaces around it each survive as+ a dash. That one character of regex --- ``\\s`` rather than ``\\s+`` ---+ accounted for 70% of this class's findings in Phase 1's first run.+ """+ # Rendered text first, but ONLY outside inline code spans. Inside a span+ # the angle brackets are literal: ``### `repowise distill <command>` ``+ # renders the word "command" and GitHub slugs it in, so treating it as a+ # tag to strip invents a broken link out of a correct one. Four of this+ # repository's own CLI-reference links were reported that way.+ parts = _CODE_SPAN_RE.split(heading.strip())+ rendered: list[str] = []+ for i, part in enumerate(parts):+ if i % 2: # odd indices are the contents of a code span+ rendered.append(part)+ continue+ part = _MD_IMAGE_RE.sub("", part)+ part = _MD_LINK_TEXT_RE.sub(r"\1", part)+ rendered.append(_HTML_TAG_RE.sub("", part))+ s = "".join(rendered).lower()+ s = _MARKUP_RE.sub("", s)+ s = _NON_WORD_RE.sub("", s)+ return _WHITESPACE_RE.sub("-", s).strip("-")diff --git a/packages/core/src/repowise/core/analysis/doc_drift/resolver.py b/packages/core/src/repowise/core/analysis/doc_drift/resolver.pynew file mode 100644index 000000000..e2f28a543--- /dev/null+++ b/packages/core/src/repowise/core/analysis/doc_drift/resolver.py@@ -0,0 +1,324 @@+"""Resolution of extracted references against the real repository tree.++Four verdicts, not two. ``UNCHECKABLE`` and ``AMBIGUOUS`` are what separate a+detector from a noise generator, and each exists because Phase 1 measured the+cost of collapsing it into ``MISSING``.+"""++from __future__ import annotations++import json+import re+from dataclasses import dataclass, field++from repowise.core.ingestion.special_handlers import iter_make_targets++from .constants import ORIGIN_CONFIDENCE+from .models import DocReference, DriftKind, DriftVerdict+from .renderer import RendererMap, github_slug++_HEADING_RE = re.compile(r"^#{1,6}\s+(.+?)\s*$")++# Setext headings. ``Title`` over ``=====`` or ``-----`` is a heading, and+# GitHub gives it an anchor exactly as it does an ATX one; recognising only+# ``#`` reports every inbound link to a setext heading as broken.+_SETEXT_UNDERLINE_RE = re.compile(r"^(?:=+|-{2,})\s*$")++# Explicit anchors. A document that writes an <a id="..."> above a section is+# declaring a link target that no heading slug reproduces, and a checker that+# reads only headings reports every one of them as broken. This was the single+# largest false-positive family in Phase 1's first run.+#+# Any element, not just ``<a>``, and an unquoted value too: GitHub honours+# ``<div id="x">`` and ``<a href="..." id="x">`` as link targets, and requiring+# ``id`` to follow ``<a`` immediately missed both.+_HTML_ANCHOR_RE = re.compile(+ r"<[a-z][a-z0-9]*\s[^>]*?\b(?:id|name)\s*=\s*(?:\"([^\"]+)\"|'([^']+)'|([^\s>]+))",+ re.I,+)++_PARENT_SEGMENT_RE = re.compile(r"[^/]+/\.\./")+++@dataclass+class Resolution:+ """One reference's verdict, plus why."""++ ref: DocReference+ verdict: DriftVerdict+ detail: str = ""+ origin: str = ""+ """Set only on ``MISSING``; names the strategy that concluded it, which is+ what carries the confidence."""++ @property+ def confidence(self) -> float:+ return ORIGIN_CONFIDENCE.get(self.origin, 0.0)+++@dataclass+class RepoIndex:+ """Everything the resolver needs to know about the tree, built once.++ Built from bytes the pipeline already decoded rather than from disk: this+ joins the index pipeline and runs on every ``init`` and every ``update``,+ so a second pass over the tree would be a real regression.+ """++ files: frozenset[str] = frozenset()+ dirs: frozenset[str] = frozenset()+ top_level_dirs: frozenset[str] = frozenset()+ indexed_suffixes: frozenset[str] = frozenset()+ """Every file extension the index actually holds.++ The index is not the repository. The traverser keeps only files it can+ identify, so assets and data files --- ``.png``, ``.css``, ``.txt``,+ ``.lock``, ``.svg`` --- never reach it, and ``--skip-tests`` removes more.+ Resolving a reference to one of those against this index would find nothing+ and call the document wrong, when the truth is that the index cannot speak+ to it. A repository with no file of a given kind cannot be evidence about+ files of that kind.+ """+ by_basename: dict[str, list[str]] = field(default_factory=dict)+ make_targets: frozenset[str] = frozenset()+ npm_scripts: frozenset[str] = frozenset()+ renderers: RendererMap = field(default_factory=lambda: RendererMap([]))+ _doc_anchors: dict[str, frozenset[str]] = field(default_factory=dict, repr=False)+ _doc_text: dict[str, str] = field(default_factory=dict, repr=False)++ @classmethod+ def build(+ cls,+ tracked_paths: frozenset[str] | set[str],+ doc_text: dict[str, str],+ manifest_text: dict[str, str] | None = None,+ ) -> RepoIndex:+ """Compose the lookup tables.++ Args:+ tracked_paths: every repo-relative path the index knows about.+ doc_text: decoded markdown, keyed by path. Anchor targets are read+ from here, so a link into a document outside this map is+ uncheckable rather than missing.+ manifest_text: decoded ``Makefile`` / ``package.json`` contents,+ keyed by path. Absent means the command class stands down.+ """+ files = frozenset(tracked_paths)+ dirs: set[str] = set()+ by_basename: dict[str, list[str]] = {}+ for rel in files:+ by_basename.setdefault(rel.rsplit("/", 1)[-1], []).append(rel)+ parts = rel.split("/")+ for i in range(1, len(parts)):+ dirs.add("/".join(parts[:i]))++ make_targets, npm_scripts = _scan_manifests(manifest_text or {})+ return cls(+ files=files,+ dirs=frozenset(dirs),+ top_level_dirs=frozenset(f.split("/")[0] for f in files if "/" in f),+ indexed_suffixes=frozenset(_suffix(f) for f in files) - {""},+ by_basename=by_basename,+ make_targets=make_targets,+ npm_scripts=npm_scripts,+ renderers=RendererMap.detect(files),+ _doc_text=doc_text,+ )++ def doc_anchors(self, rel: str) -> frozenset[str]:+ """Every anchor *rel* declares: heading slugs plus explicit HTML ids.++ Read through :func:`~.extractor.prose_lines`, the same filter the+ extraction side uses. Scanning the raw text instead made the two halves+ disagree: a ``# Install dependencies`` comment inside a shell fence+ registered as a declared anchor, while a reference on the same line+ would have been skipped. That asymmetry only ever suppresses findings,+ but it suppresses them for a reason no reader could reconstruct.+ """+ cached = self._doc_anchors.get(rel)+ if cached is not None:+ return cached++ from .extractor import prose_lines++ found: set[str] = set()+ previous = ""+ for _lineno, line in prose_lines(self._doc_text.get(rel, "")):+ m = _HEADING_RE.match(line)+ if m:+ found.add(github_slug(m.group(1)))+ elif previous.strip() and _SETEXT_UNDERLINE_RE.match(line):+ found.add(github_slug(previous))+ for groups in _HTML_ANCHOR_RE.findall(line):+ value = next((g for g in groups if g), "")+ found.add(value.strip().lower())+ previous = line++ found.discard("")+ frozen = frozenset(found)+ self._doc_anchors[rel] = frozen+ return frozen+++def _suffix(path: str) -> str:+ """The lowercased extension of *path*, or ``""`` when it has none."""+ base = path.rsplit("/", 1)[-1]+ dot = base.rfind(".")+ return base[dot:].lower() if dot > 0 else ""+++def _scan_manifests(manifest_text: dict[str, str]) -> tuple[frozenset[str], frozenset[str]]:+ make_targets: set[str] = set()+ npm_scripts: set[str] = set()+ for rel, text in manifest_text.items():+ name = rel.rsplit("/", 1)[-1]+ if name in ("Makefile", "makefile", "GNUmakefile"):+ # The ingestion handler already owns this vocabulary, and its+ # character class admits ``/`` --- so a documented+ # ``make docs/build`` resolves instead of being reported as a+ # missing target at 0.85 confidence.+ make_targets.update(target for target, _ in iter_make_targets(text))+ elif name == "package.json":+ try:+ scripts = json.loads(text).get("scripts") or {}+ except (json.JSONDecodeError, AttributeError, TypeError):+ continue+ if isinstance(scripts, dict):+ npm_scripts.update(scripts)+ return frozenset(make_targets), frozenset(npm_scripts)+++# ---------------------------------------------------------------------------+# Per-class resolution+# ---------------------------------------------------------------------------+++def _join_relative(doc: str, target: str) -> str:+ """Resolve *target* as written relative to the document that names it."""+ doc_dir = doc.rsplit("/", 1)[0] if "/" in doc else ""+ if not doc_dir:+ return target+ joined = f"{doc_dir}/{target}"+ # Collapse "a/b/../c" without touching the filesystem.+ while True:+ collapsed = _PARENT_SEGMENT_RE.sub("", joined, count=1)+ if collapsed == joined:+ break+ joined = collapsed+ return joined.lstrip("./")+++def _exists(idx: RepoIndex, target: str) -> bool:+ """Whether *target* names a file or directory the index holds.++ A trailing slash is how a document says "directory", and it is kept through+ normalization so the reference reads back as written --- but the directory+ set is built from path prefixes, which carry no slash. Comparing the two+ forms directly reported every correctly-written directory reference in this+ repository as drift.+ """+ if target in idx.files or target in idx.dirs:+ return True+ trimmed = target.rstrip("/")+ return bool(trimmed) and (trimmed in idx.files or trimmed in idx.dirs)+++def _resolve_path(idx: RepoIndex, ref: DocReference, is_guide: bool) -> Resolution:+ target = ref.target+ if _exists(idx, target):+ return Resolution(ref, DriftVerdict.RESOLVED, "exact")++ relative = _join_relative(ref.doc_path, target)+ if relative != target and _exists(idx, relative):+ return Resolution(ref, DriftVerdict.RESOLVED, "relative-to-doc")++ # Anchoring. A reference is evidence about THIS repository only when it+ # carries a separator and its first segment names a directory the+ # repository actually has. Everything else is a file Repowise writes into+ # the user's tree (CLAUDE.md, config.yaml, state.json), a file belonging to+ # a framework the docs merely describe (routes/web.php, composer.json), or+ # an invented example (login.py). Phase 1's first run flagged all three+ # families; they are 100% false positives, and no amount of resolution+ # effort fixes them, because the target was never supposed to exist here.+ # This single rule took the path class from a 49% flag rate to 1.3%.+ if "/" not in target or target.split("/")[0] not in idx.top_level_dirs:+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "unanchored")++ # A basename match is not a resolution and is not drift either. Phase 1+ # found five of these; reporting them as missing would have been wrong.+ candidates = idx.by_basename.get(target.rsplit("/", 1)[-1], [])+ if len(candidates) == 1:+ return Resolution(ref, DriftVerdict.AMBIGUOUS, f"basename-only:{candidates[0]}")+ if len(candidates) > 1:+ return Resolution(ref, DriftVerdict.AMBIGUOUS, f"basename-multi:{len(candidates)}")++ # The index is not the repository, so its silence is only evidence about+ # kinds of file it actually carries. The traverser drops every extension it+ # cannot identify --- assets, lockfiles, plain text --- and ``--skip-tests``+ # drops more, so a reference to one of those resolves against a tree that+ # structurally could not contain it. Reporting that as drift blames the+ # document for the indexer's scope.+ suffix = _suffix(target)+ if suffix and suffix not in idx.indexed_suffixes:+ return Resolution(ref, DriftVerdict.UNCHECKABLE, f"unindexed-kind:{suffix}")++ origin = "path_no_candidate_in_guide" if is_guide else "path_no_candidate"+ return Resolution(ref, DriftVerdict.MISSING, "no-candidate", origin)+++def _resolve_anchor(idx: RepoIndex, ref: DocReference) -> Resolution:+ target, _, frag = ref.target.partition("#")++ # Joined relative to the referring document FIRST, because that is what a+ # markdown link means: every renderer resolves ``[g](guide.md#setup)`` in+ # ``docs/a.md`` as ``docs/guide.md``. Preferring a verbatim root-relative+ # hit read the wrong file whenever two documents shared a basename, and+ # then reported the correct link as a renamed heading at 0.95 --- the+ # highest confidence this detector issues.+ if target:+ relative = _join_relative(ref.doc_path, target)+ if relative in idx.files:+ target = relative++ # Gated on the renderer of the document that DECLARES the anchors, because+ # that is what decides the slug. Never guess an unimplemented renderer's+ # algorithm: fastapi produced 93 findings that way, all false.+ if not idx.renderers.checkable(target or ref.doc_path):+ renderer = idx.renderers.for_document(target or ref.doc_path)+ return Resolution(ref, DriftVerdict.UNCHECKABLE, f"renderer:{renderer}")++ if target not in idx.files:+ # The document itself is missing. The LINK row already says so; saying+ # it twice would double-count one defect.+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "host-doc-unresolved")+ if target not in idx._doc_text:+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "host-doc-unread")+ if github_slug(frag) in idx.doc_anchors(target):+ return Resolution(ref, DriftVerdict.RESOLVED, "heading")+ return Resolution(ref, DriftVerdict.MISSING, "no-heading", "anchor_no_heading")+++def _resolve_command(idx: RepoIndex, ref: DocReference) -> Resolution:+ runner, _, name = ref.target.partition(":")+ if runner == "make":+ if not idx.make_targets:+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "no-makefile")+ found = name in idx.make_targets+ else:+ if not idx.npm_scripts:+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "no-package-json")+ found = name in idx.npm_scripts+ if found:+ return Resolution(ref, DriftVerdict.RESOLVED, f"{runner}-target")+ return Resolution(ref, DriftVerdict.MISSING, f"{runner}-target", "command_no_target")+++def resolve(idx: RepoIndex, ref: DocReference, *, is_guide: bool = False) -> Resolution:+ """Answer one reference against the tree."""+ if ref.kind in (DriftKind.PATH, DriftKind.LINK):+ return _resolve_path(idx, ref, is_guide)+ if ref.kind is DriftKind.ANCHOR:+ return _resolve_anchor(idx, ref)+ if ref.kind is DriftKind.COMMAND:+ return _resolve_command(idx, ref)+ return Resolution(ref, DriftVerdict.UNCHECKABLE, "unknown-kind")diff --git a/packages/core/src/repowise/core/ingestion/special_handlers.py b/packages/core/src/repowise/core/ingestion/special_handlers.pyindex 328688bff..64d453424 100644--- a/packages/core/src/repowise/core/ingestion/special_handlers.py+++ b/packages/core/src/repowise/core/ingestion/special_handlers.py@@ -11,7 +11,7 @@ the tree-sitter parsers — so the rest of the pipeline treats them identically.from __future__ import annotationsimport re-from collections.abc import Callable+from collections.abc import Callable, Iteratorfrom functools import lru_cachefrom pathlib import Path@@ -218,11 +218,51 @@ def _parse_dockerfile(file_info: FileInfo, source: bytes) -> ParsedFile:# ---------------------------------------------------------------------------# Matches: target_name: [prerequisites...]-_TARGET_RE = re.compile(r"^([a-zA-Z0-9_][a-zA-Z0-9_\-./]*):[^=]")+#+# The colon is followed by a negative lookahead rather than a consumed+# non-``=`` character. Consuming one required something to follow the+# colon on the same line, so a target declared with no prerequisites ---+# ``build:`` on its own line, the commonest form there is --- matched+# nothing and never became a symbol. The lookahead still rejects the+# ``:=`` assignment operator, which is all the character was there for.+_TARGET_RE = re.compile(r"^([a-zA-Z0-9_][a-zA-Z0-9_\-./ \t]*?)\s*:(?!=)")++# One rule may declare several targets: ``test lint:`` is valid and common.+# The capture above is non-greedy and admits spaces, so the whole left-hand+# side arrives here to be split.+_TARGET_SEP_RE = re.compile(r"[ \t]+")_INCLUDE_RE = re.compile(r"^include\s+(.+)", re.IGNORECASE)_PHONY_RE = re.compile(r"^\.PHONY\s*:\s*(.+)")+def iter_make_targets(text: str) -> Iterator[tuple[str, int]]:+ """Yield ``(target name, 1-indexed line)`` for every target *text* declares.++ Public because the target vocabulary of a Makefile is asked for in more+ than one place: this handler turns them into graph symbols, and the+ documentation drift detector checks a documented ``make <target>`` against+ them. A second regex would drift --- the character class here deliberately+ admits ``.``, ``-`` and ``/``, so a directory-style target like+ ``docs/build`` is a target rather than an unrecognised string.++ One rule may declare several targets (``test lint:``); each is yielded+ separately. Dot-prefixed directives (``.PHONY``, ``.SUFFIXES``) are not+ targets and are skipped, as are comments and blank lines.+ """+ for lineno, line in enumerate(text.splitlines(), start=1):+ stripped = line.strip()+ if not stripped or stripped.startswith("#"):+ continue+ m = _TARGET_RE.match(line)+ if not m:+ continue+ for name in _TARGET_SEP_RE.split(m.group(1).strip()):+ # Dot-prefixed directives (``.PHONY``, ``.SUFFIXES``) are not+ # targets.+ if name and not name.startswith("."):+ yield name, lineno++def _parse_makefile(file_info: FileInfo, source: bytes) -> ParsedFile:text = source.decode("utf-8", errors="replace")lines = text.splitlines()@@ -236,30 +276,26 @@ def _parse_makefile(file_info: FileInfo, source: bytes) -> ParsedFile:if m:phony_targets.update(m.group(1).split())- # Second pass: extract targets- for lineno, line in enumerate(lines, start=1):- line_stripped = line.strip()- if not line_stripped or line_stripped.startswith("#"):- continue+ for target, lineno in iter_make_targets(text):+ symbols.append(+ Symbol(+ id=f"{file_info.path}::{target}",+ name=target,+ qualified_name=target,+ kind="function",+ signature=f"{target}:",+ start_line=lineno,+ end_line=lineno,+ docstring=None,+ visibility="public",+ language="makefile",+ )+ )- m = _TARGET_RE.match(line)- if m:- target = m.group(1)- if not target.startswith("."): # skip .PHONY, .SUFFIXES, etc.- symbols.append(- Symbol(- id=f"{file_info.path}::{target}",- name=target,- qualified_name=target,- kind="function",- signature=f"{target}:",- start_line=lineno,- end_line=lineno,- docstring=None,- visibility="public",- language="makefile",- )- )+ # Includes are scanned separately: a line is either a target or an+ # include, and ``iter_make_targets`` already skipped the ones it claimed.+ for line in lines:+ if _TARGET_RE.match(line):continuem = _INCLUDE_RE.match(line)diff --git a/packages/core/src/repowise/core/persistence/__init__.py b/packages/core/src/repowise/core/persistence/__init__.pyindex 6640f1a15..54a609f40 100644--- a/packages/core/src/repowise/core/persistence/__init__.py+++ b/packages/core/src/repowise/core/persistence/__init__.py@@ -129,6 +129,7 @@ from .models import (DecisionEvidence,DecisionNodeLink,DecisionRecord,+ DocDriftFinding,ExternalSystem,GenerationJob,GitCommit,@@ -168,6 +169,7 @@ __all__ = ["DecisionEvidence","DecisionNodeLink","DecisionRecord",+ "DocDriftFinding",# embedder"Embedder","ExternalSystem",diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.pyindex e7247a8c8..efd8ad568 100644--- a/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py+++ b/packages/core/src/repowise/core/persistence/crud/analysis/__init__.py@@ -8,7 +8,15 @@ imports are unaffected.from __future__ import annotations-from . import coverage, coverage_map, dead_code, health, performance, refactoring # noqa: F401+from . import ( # noqa: F401+ coverage,+ coverage_map,+ dead_code,+ doc_drift,+ health,+ performance,+ refactoring,+)from .coverage import (get_coverage_summary,load_coverage_for_repo,@@ -29,6 +37,11 @@ from .dead_code import (save_dead_code_findings,update_dead_code_status,)+from .doc_drift import (+ get_doc_drift_findings,+ replace_doc_drift_findings,+ summarize_confidence_rows,+)from .health import (FILE_TREND_SNAPSHOT_WINDOW,HEALTH_SNAPSHOT_RETENTION,@@ -106,6 +119,7 @@ __all__ = ["get_dead_code_findings","get_dead_code_summary","get_deduction_by_path",+ "get_doc_drift_findings","get_file_language_map","get_health_finding_by_public_id","get_health_findings",@@ -135,6 +149,7 @@ __all__ = ["prune_unscored_health_rows","refactoring_facet_counts","replace_dead_code_findings",+ "replace_doc_drift_findings","replace_governance_findings","save_coverage_files","save_dead_code_findings",@@ -144,6 +159,7 @@ __all__ = ["save_refactoring_suggestions","save_test_coverage","sort_metrics_worst_first",+ "summarize_confidence_rows","tests_covering","tests_covering_files","update_dead_code_status",diff --git a/packages/core/src/repowise/core/persistence/crud/analysis/doc_drift.py b/packages/core/src/repowise/core/persistence/crud/analysis/doc_drift.pynew file mode 100644index 000000000..b9ac91ea4--- /dev/null+++ b/packages/core/src/repowise/core/persistence/crud/analysis/doc_drift.py@@ -0,0 +1,148 @@+"""CRUD operations for documentation drift findings."""++from __future__ import annotations++import json+from typing import Any++from sqlalchemy import delete, select+from sqlalchemy.ext.asyncio import AsyncSession++from repowise.core.analysis.doc_drift.constants import bucket_confidences++from ...models import DocDriftFinding+from .._shared import _BATCH_SIZE++#: Width of the two ``String(1024)`` columns. ``target`` is lifted verbatim+#: from a document rather than minted by the traverser, so a pathological+#: reference (a long inline URL, a data: URI) can exceed it. SQLite ignores+#: the declared width and Postgres raises ``StringDataRightTruncation``, so+#: leaving it unclamped is a backend divergence that surfaces as total loss of+#: the write on the one backend the architecture docs recommend.+_PATH_COLUMN_WIDTH = 1024++#: Chunk for ``IN (...)`` deletes. Lower than ``_BATCH_SIZE`` deliberately:+#: the house figure for a parameterised ``IN`` clause is 400+#: (``security_scan.py`` and ``_shared._GATE_LOAD_CHUNK`` both use it) for+#: SQLite parameter-limit headroom.+_IN_CLAUSE_CHUNK = 400+++def _row_kwargs(finding: Any, repository_id: str) -> dict:+ """Normalize a ``DocDriftFindingData`` into ORM row kwargs."""+ kind = finding.kind+ return {+ "repository_id": repository_id,+ "file_path": finding.file_path[:_PATH_COLUMN_WIDTH],+ "kind": str(kind.value) if hasattr(kind, "value") else str(kind),+ "line_number": int(finding.line_number),+ "target": finding.target[:_PATH_COLUMN_WIDTH],+ "confidence": float(finding.confidence),+ "reason": finding.reason,+ "origin": finding.origin,+ "evidence_json": json.dumps(list(finding.evidence or [])),+ "raw": finding.raw,+ "context": finding.context,+ }+++async def replace_doc_drift_findings(+ session: AsyncSession,+ repository_id: str,+ findings: list[Any],+ *,+ scope: frozenset[str] | set[str] | None = None,+) -> int:+ """Replace this repository's drift findings, converging on re-run.++ Delete-then-insert, like dead code and security before it. ``scope``, when+ given, names the document paths this write may speak for: an incremental+ re-check that only re-read four documents must not delete findings for the+ other three hundred it never looked at. ``None`` means the run was+ repo-wide and every row is replaceable.++ Returns the number of rows inserted.++ Idempotency rests on two things. The unique constraint+ ``uq_doc_drift_finding_site`` is the durable one. The other is that the+ DELETE scope and the INSERT scope are identical --- every row inserted+ belongs to a path just deleted --- so no surviving row can collide. The+ Python-side dedup below closes the remaining gap, which is two findings in+ the *same batch* sharing a site; without it that batch would raise after+ the DELETE had already run. ``security_scan.replace_findings`` guards the+ same hazard the same way, and additionally uses a conflict-tolerant+ INSERT because it has a second writer (the history scan) that this table+ does not.+ """+ scoped = frozenset(scope) if scope is not None else None++ if scoped is None:+ await session.execute(+ delete(DocDriftFinding).where(DocDriftFinding.repository_id == repository_id)+ )+ else:+ # Chunked so a large scope stays under SQLite's parameter limit.+ paths = sorted(scoped)+ for i in range(0, len(paths), _IN_CLAUSE_CHUNK):+ chunk = paths[i : i + _IN_CLAUSE_CHUNK]+ await session.execute(+ delete(DocDriftFinding).where(+ DocDriftFinding.repository_id == repository_id,+ DocDriftFinding.file_path.in_(chunk),+ )+ )++ rows: list[dict] = []+ seen: set[tuple] = set()+ for finding in findings or []:+ if scoped is not None and finding.file_path not in scoped:+ # A finding outside the scope would be inserted and never deleted+ # by the next scoped run, so it would outlive its own evidence.+ continue+ kwargs = _row_kwargs(finding, repository_id)+ key = (+ kwargs["file_path"],+ kwargs["kind"],+ kwargs["line_number"],+ kwargs["target"],+ )+ if key in seen:+ continue+ seen.add(key)+ rows.append(kwargs)++ for i in range(0, len(rows), _BATCH_SIZE):+ session.add_all(DocDriftFinding(**r) for r in rows[i : i + _BATCH_SIZE])+ await session.flush()++ return len(rows)+++async def get_doc_drift_findings(+ session: AsyncSession,+ repository_id: str,+ *,+ file_path: str | None = None,+ min_confidence: float | None = None,+) -> list[DocDriftFinding]:+ """Read findings back, most confident first."""+ stmt = select(DocDriftFinding).where(DocDriftFinding.repository_id == repository_id)+ if file_path is not None:+ stmt = stmt.where(DocDriftFinding.file_path == file_path)+ if min_confidence is not None:+ stmt = stmt.where(DocDriftFinding.confidence >= min_confidence)+ stmt = stmt.order_by(+ DocDriftFinding.confidence.desc(),+ DocDriftFinding.file_path,+ DocDriftFinding.line_number,+ )+ return list((await session.execute(stmt)).scalars().all())+++def summarize_confidence_rows(rows: list[DocDriftFinding]) -> dict:+ """Re-derive the high/medium/low buckets on read.++ Calls the same function the analyzer does, so the summary a reader sees+ cannot disagree with the one the pass computed.+ """+ return bucket_confidences(r.confidence for r in rows)diff --git a/packages/core/src/repowise/core/persistence/models.py b/packages/core/src/repowise/core/persistence/models.pyindex 68ce0cef1..b289c5e5e 100644--- a/packages/core/src/repowise/core/persistence/models.py+++ b/packages/core/src/repowise/core/persistence/models.py@@ -1383,6 +1383,57 @@ class DeadCodeFinding(Base):__table_args__ = (Index("ix_dead_code_repo_path", "repository_id", "file_path"),)+class DocDriftFinding(Base):+ """An assertion a document makes that the repository no longer satisfies.++ ``file_path`` is the **document** that is wrong, not the target it names:+ that is the file a reader has to edit, so it is the one the finding is+ filed against.++ Persisted on the security-findings pattern rather than dead code's, because+ a drift finding is a line-level hit rather than a graph-derived aggregate.+ The ``(repository_id, file_path, kind, line_number, target)`` constraint+ makes re-runs idempotent. ``target`` is in the key because one line can+ carry two references --- a markdown link yields both a ``link`` row and an+ ``anchor`` row --- and dropping it would silently collapse them.+ """++ __tablename__ = "doc_drift_findings"+ __table_args__ = (+ UniqueConstraint(+ "repository_id",+ "file_path",+ "kind",+ "line_number",+ "target",+ name="uq_doc_drift_finding_site",+ ),+ Index("ix_doc_drift_repo_path", "repository_id", "file_path"),+ )++ id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)+ repository_id: Mapped[str] = mapped_column(+ String(32), ForeignKey("repositories.id", ondelete="CASCADE"), nullable=False+ )+ file_path: Mapped[str] = mapped_column(String(1024), nullable=False)+ kind: Mapped[str] = mapped_column(String(32), nullable=False)+ line_number: Mapped[int] = mapped_column(Integer, nullable=False)+ target: Mapped[str] = mapped_column(String(1024), nullable=False)+ confidence: Mapped[float] = mapped_column(Float, nullable=False, default=0.0)+ reason: Mapped[str] = mapped_column(Text, nullable=False, default="")+ #: Which named resolution strategy concluded this, from ``DriftOrigin``.+ #: Every origin carries a fixed confidence, so the origin distribution and+ #: the confidence histogram are two views of the same data.+ origin: Mapped[str] = mapped_column(String(64), nullable=False, default="")+ evidence_json: Mapped[str] = mapped_column(Text, nullable=False, default="[]")+ #: The reference exactly as written, and the line it was written on.+ raw: Mapped[str] = mapped_column(Text, nullable=False, default="")+ context: Mapped[str] = mapped_column(Text, nullable=False, default="")+ detected_at: Mapped[datetime] = mapped_column(+ DateTime(timezone=True), nullable=False, default=_now_utc+ )++class HealthFinding(Base):"""One biomarker hit produced by the code-health analyzer."""diff --git a/packages/core/src/repowise/core/pipeline/orchestrator.py b/packages/core/src/repowise/core/pipeline/orchestrator.pyindex 6b74155f6..9adbe288e 100644--- a/packages/core/src/repowise/core/pipeline/orchestrator.py+++ b/packages/core/src/repowise/core/pipeline/orchestrator.py@@ -37,6 +37,7 @@ from .phases._common import TEST_RUN_FILE_LIMIT, _phase_done, limit_to_top_pagerfrom .phases.analysis import (_run_dead_code_analysis,_run_decision_extraction,+ _run_doc_drift_analysis,_run_health_analysis,)from .phases.generation import run_generation@@ -115,6 +116,9 @@ class PipelineResult:health_report: Any | None = None"""``HealthReport`` or None — populated by ``_run_health_analysis``."""+ doc_drift_report: Any | None = None+ """``DocDriftReport`` or None — populated by ``_run_doc_drift_analysis``."""+# Traversal statstraversal_stats: Any | None = None"""``TraversalStats`` from the file traverser, or None."""@@ -480,6 +484,7 @@ async def run_pipeline(dead_code_report = Nonehealth_report = Nonedecision_report = None+ doc_drift_report = None# Reports actually fed to generation + KG — rehydrated on the skip path,# the freshly computed ones otherwise.gen_dead_code_report = None@@ -497,12 +502,17 @@ async def run_pipeline(skip_analysis = Falseif not skip_analysis:- # The three analyses share read-only inputs (graph, git_meta_map,+ # The four analyses share read-only inputs (graph, git_meta_map,# parsed_files; the lazy metric caches were warmed during ingestion)# and have no data dependency on each other, so run them concurrently:# decision extraction is I/O/LLM-bound and its wall clock hides# entirely behind the CPU-bound dead-code + health work.- dead_code_report, health_report, decision_report = await asyncio.gather(+ (+ dead_code_report,+ health_report,+ decision_report,+ doc_drift_report,+ ) = await asyncio.gather(_run_dead_code_analysis(graph_builder,git_meta_map,@@ -528,6 +538,11 @@ async def run_pipeline(source_map=source_map,progress=progress,),+ _run_doc_drift_analysis(+ source_map,+ file_infos=file_infos,+ progress=progress,+ ),)# Drop the in-memory-only ``BlameIndex`` now that the health biomarkers@@ -653,6 +668,7 @@ async def run_pipeline(dead_code_report=dead_code_report,health_report=health_report,decision_report=decision_report,+ doc_drift_report=doc_drift_report,git_metadata_list=git_metadata_list,progress=progress,)@@ -917,6 +933,7 @@ async def run_pipeline(dead_code_report=dead_code_report,decision_report=decision_report,health_report=health_report,+ doc_drift_report=doc_drift_report,execution_flow_report=execution_flow_report,knowledge_graph_result=knowledge_graph_result,generated_pages=generated_pages,diff --git a/packages/core/src/repowise/core/pipeline/persist.py b/packages/core/src/repowise/core/pipeline/persist.pyindex 8e31a1224..9eee08939 100644--- a/packages/core/src/repowise/core/pipeline/persist.py+++ b/packages/core/src/repowise/core/pipeline/persist.py@@ -1068,6 +1068,7 @@ async def prune_deleted_file_rows(from repowise.core.analysis.dead_code.analyzer import _is_synthetic_nodefrom repowise.core.persistence.models import (DeadCodeFinding,+ DocDriftFinding,GitMetadata,GraphEdge,GraphMetric,@@ -1161,6 +1162,12 @@ async def prune_deleted_file_rows(await _prune_table(WikiSymbol, WikiSymbol.file_path, "wiki_symbols")await _prune_table(SecurityFinding, SecurityFinding.file_path, "security_findings")await _prune_table(DeadCodeFinding, DeadCodeFinding.file_path, "dead_code_findings")+ # Keyed on the DOCUMENT, and the drift pass does not run on the+ # incremental path, so without this a deleted document's findings are+ # never recomputed and never removed: they outlive the file until the+ # next full index. ``_FileLiveness`` asks disk and ``git ls-files``+ # rather than the parse, so a ``.md`` path is judged correctly here.+ await _prune_table(DocDriftFinding, DocDriftFinding.file_path, "doc_drift_findings")await _prune_table(HealthFileMetric, HealthFileMetric.file_path, "health_file_metrics")await _prune_table(HealthFinding, HealthFinding.file_path, "health_findings")# git_metadata is keyed off the git indexer on a full run, but an@@ -1897,16 +1904,17 @@ async def save_full_health_report(async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:- """Persist analysis-phase outputs: dead code, health, decisions, governance.+ """Persist analysis-phase outputs: dead code, health, decisions, drift.- Dead-code and health writes are repo-wide DELETE-THEN-INSERT (so they- converge on re-run but don't support partial-within-phase resume);+ Dead-code, health and doc-drift writes are repo-wide DELETE-THEN-INSERT (so+ they converge on re-run but don't support partial-within-phase resume);decisions/governance are idempotent. Intended to run once the analysisphase has fully completed."""from repowise.core.persistence.crud import (bulk_upsert_decisions,recompute_decision_staleness,+ replace_doc_drift_findings,save_coverage_files,save_dead_code_findings,upsert_git_function_blame_bulk,@@ -1916,6 +1924,32 @@ async def persist_analysis(result: Any, session: Any, repo_id: str) -> None:if result.dead_code_report and result.dead_code_report.findings:await save_dead_code_findings(session, repo_id, result.dead_code_report.findings)+ # ---- Documentation drift findings ---------------------------------------+ # Written even when the finding list is empty, unlike dead code above: a+ # run that fixed the last drifted reference must clear the rows, and a+ # guard on ``.findings`` would leave the old ones standing as though the+ # documents were still wrong. ``authoritative_paths`` is None on a full+ # run, so the replace is repo-wide.+ drift = getattr(result, "doc_drift_report", None)+ if drift is not None:+ try:+ # Inside a savepoint because this is a DELETE followed by an+ # INSERT and the failure is swallowed. Without one, a failing+ # insert leaves the DELETE buffered in the caller's still-live+ # transaction, which then commits it: every drift row for the+ # repository wiped, reported only as a warning. On Postgres the+ # same failure poisons the transaction and takes health,+ # coverage and decisions down with it.+ async with session.begin_nested():+ await replace_doc_drift_findings(+ session,+ repo_id,+ drift.findings,+ scope=drift.authoritative_paths,+ )+ except Exception as exc:+ logger.warning("doc_drift_persist_skipped", error=str(exc))+# ---- Health findings + per-file metrics ---------------------------------if getattr(result, "health_report", None):hr = result.health_reportdiff --git a/packages/core/src/repowise/core/pipeline/phases/analysis.py b/packages/core/src/repowise/core/pipeline/phases/analysis.pyindex 8cb53d9eb..9efce9fc3 100644--- a/packages/core/src/repowise/core/pipeline/phases/analysis.py+++ b/packages/core/src/repowise/core/pipeline/phases/analysis.py@@ -107,6 +107,61 @@ async def _run_dead_code_analysis(return None+async def _run_doc_drift_analysis(+ source_map: dict[str, bytes] | None,+ *,+ file_infos: list[Any] | None = None,+ repo_id: str = "",+ progress: ProgressCallback | None,+) -> Any | None:+ """Check the repository's own markdown against the tree (no LLM).++ Reads bytes ingestion already decoded rather than re-walking the tree: this+ runs on every ``init`` and every ``update``, so a second pass over the+ repository would be a real regression.+ """+ try:+ from repowise.core.analysis.doc_drift import DocDriftAnalyzer++ # analyze() drives three stages: collect, index, resolve.+ if progress:+ progress.on_phase_start("doc_drift", 3)++ # Built here rather than at the call site: the gather has no+ # per-member exception wrapper, so an argument expression evaluated in+ # the caller's frame would raise outside this try and abort the whole+ # index run.+ tracked_paths = {fi.path for fi in file_infos} if file_infos else None++ analyzer = DocDriftAnalyzer(+ repo_id,+ source_map=source_map,+ tracked_paths=tracked_paths,+ )++ def _step(_stage: str) -> None:+ if progress:+ progress.on_item_done("doc_drift")++ report = await asyncio.to_thread(analyzer.analyze, None, on_step=_step)++ if progress and report.documents_scanned:+ progress.on_message(+ "info",+ f"→ {report.total_findings} drift findings across "+ f"{report.documents_scanned} documents "+ f"({report.references_checked:,} references checked)",+ )++ _phase_done(progress, "doc_drift")+ return report+ except Exception as exc:+ if progress:+ progress.on_message("warning", f"Documentation drift check skipped: {exc}")+ _phase_done(progress, "doc_drift")+ return None++def _build_pipeline_coverage(repo_path: Path,parsed_files: list[Any],diff --git a/packages/core/src/repowise/core/pipeline/resume/controller.py b/packages/core/src/repowise/core/pipeline/resume/controller.pyindex ea47f9a9b..f86b2f12f 100644--- a/packages/core/src/repowise/core/pipeline/resume/controller.py+++ b/packages/core/src/repowise/core/pipeline/resume/controller.py@@ -193,6 +193,7 @@ class ResumeController:health_report: Any | None,decision_report: Any | None,git_metadata_list: list[dict],+ doc_drift_report: Any | None = None,vector_store: Any | None = None,progress: Any | None = None,) -> None:@@ -211,6 +212,7 @@ class ResumeController:dead_code_report=dead_code_report,health_report=health_report,decision_report=decision_report,+ doc_drift_report=doc_drift_report,git_metadata_list=git_metadata_list,generated_pages=None,vector_store=vector_store,diff --git a/tests/unit/doc_drift/__init__.py b/tests/unit/doc_drift/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/unit/doc_drift/test_analyzer.py b/tests/unit/doc_drift/test_analyzer.pynew file mode 100644index 000000000..7f4fe1d73--- /dev/null+++ b/tests/unit/doc_drift/test_analyzer.py@@ -0,0 +1,213 @@+"""The drift pass end to end: report shape, cutoff, and the known defects."""++from __future__ import annotations++import pytest++from repowise.core.analysis.doc_drift import DocDriftAnalyzer, summarize_confidence+from repowise.core.analysis.doc_drift.constants import (+ DEFAULT_MIN_CONFIDENCE,+ DRIFT_ORIGIN_VALUES,+ HIGH_CONFIDENCE_THRESHOLD,+ ORIGIN_CONFIDENCE,+ REVIEW_CONFIDENCE_THRESHOLD,+)+from repowise.core.analysis.doc_drift.models import DriftKind, DriftVerdict+++def _analyze(files: dict[str, str], config: dict | None = None):+ source_map = {k: v.encode("utf-8") for k, v in files.items()}+ return DocDriftAnalyzer("repo", source_map=source_map).analyze(config)+++# ---------------------------------------------------------------------------+# The fourteen known defects, as fixtures+# ---------------------------------------------------------------------------+++def test_module_became_a_package():+ """The single most common drift shape Phase 1 found.++ Four of the eleven real path findings are this: a module became a package+ and the prose still names the ``.py``.+ """+ report = _analyze(+ {+ "docs/a.md": "See `packages/core/ingestion/git_indexer.py` for this.\n",+ "packages/core/ingestion/git_indexer/__init__.py": "",+ "packages/core/ingestion/git_indexer/walker.py": "",+ }+ )+ assert report.total_findings == 1+ assert report.findings[0].kind is DriftKind.PATH+ assert report.findings[0].line_number == 1+++def test_deleted_file_is_flagged():+ report = _analyze(+ {"docs/a.md": "See `src/persistence/vector.py`.\n", "src/persistence/store.py": ""}+ )+ assert report.total_findings == 1+++def test_renamed_heading_leaves_the_inbound_link_behind():+ """All three real anchor findings are this shape."""+ report = _analyze(+ {+ "docs/CONTRIBUTING.md": "See [quickstart](../README.md#quickstart-under-5-minutes).\n",+ "README.md": "## Start in minutes (no API key)\n",+ }+ )+ anchors = [f for f in report.findings if f.kind is DriftKind.ANCHOR]+ assert len(anchors) == 1+ assert anchors[0].origin == "anchor_no_heading"+++def test_a_correct_document_produces_nothing():+ report = _analyze(+ {+ "docs/a.md": "See `src/app.py` and [readme](../README.md#setup).\n",+ "README.md": "## Setup\n",+ "src/app.py": "",+ }+ )+ assert report.total_findings == 0+ assert report.verdict_summary[DriftVerdict.MISSING.value] == 0+++# ---------------------------------------------------------------------------+# Report shape+# ---------------------------------------------------------------------------+++def test_uncheckable_is_counted_not_hidden():+ """684 of 903 path references on this repo are uncheckable.++ Saying so is the difference between a detector and a noise generator.+ """+ report = _analyze({"docs/a.md": "Edit `CLAUDE.md` and `config.yaml`.\n", "src/x.py": ""})+ assert report.total_findings == 0+ assert report.verdict_summary[DriftVerdict.UNCHECKABLE.value] == 2+ assert report.references_checked == 2+++def test_verdict_summary_covers_every_reference():+ report = _analyze(+ {+ "docs/a.md": "`src/app.py` `CLAUDE.md` `src/gone.py`\n",+ "src/app.py": "",+ }+ )+ assert sum(report.verdict_summary.values()) == report.references_checked+++def test_historical_documents_are_not_scanned():+ report = _analyze({"CHANGELOG.md": "Removed `src/gone.py` in 1.0.\n", "src/a.py": ""})+ assert report.documents_scanned == 0+ assert report.total_findings == 0+++def test_min_confidence_cutoff_and_hidden_counter():+ files = {"docs/tutorial.md": "Create `src/specs/mylang.py`.\n", "src/specs/real.py": ""}+ shown = _analyze(files)+ assert shown.total_findings == 1+ assert shown.hidden_below_threshold == 0++ hidden = _analyze(files, {"min_confidence": 0.9})+ assert hidden.total_findings == 0+ assert hidden.hidden_below_threshold == 1+++def test_classes_can_be_disabled_independently():+ files = {+ "docs/a.md": "`src/gone.py` and [x](../README.md#nope)\n",+ "README.md": "## Yes\n",+ # A real file under src/, so the path reference is anchored and+ # therefore checkable. Without it the path is uncheckable and this+ # test would pass for the wrong reason.+ "src/real.py": "",+ }+ assert _analyze(files, {"check_path": False}).total_findings == 1+ assert _analyze(files, {"check_anchor": False}).total_findings == 1+ assert _analyze(files, {"check_path": False, "check_anchor": False}).total_findings == 0+++def test_findings_are_ordered_most_confident_first():+ report = _analyze(+ {+ "docs/guide.md": "`src/one.py`\n",+ "docs/architecture.md": "`src/two.py`\n",+ "src/real.py": "",+ }+ )+ confidences = [f.confidence for f in report.findings]+ assert confidences == sorted(confidences, reverse=True)+++def test_finding_carries_evidence_and_a_reason():+ report = _analyze({"docs/a.md": "See `src/gone.py`.\n", "src/real.py": ""})+ finding = report.findings[0]+ assert finding.reason+ assert any("docs/a.md:1" in e for e in finding.evidence)+ assert finding.raw == "src/gone.py"+++def test_oversized_documents_are_skipped():+ """Reproduces the 500KB ``max_file_size_kb`` ceiling."""+ big = "See `src/gone.py`.\n" + ("x" * (600 * 1024))+ report = _analyze({"docs/big.md": big, "src/real.py": ""})+ assert report.documents_scanned == 0+++def test_rerunning_changes_nothing():+ """Exit criterion: re-running changes no rows."""+ files = {"docs/a.md": "See `src/gone.py`.\n", "src/real.py": ""}+ first, second = _analyze(files), _analyze(files)+ key = lambda r: [ # noqa: E731+ (f.file_path, f.kind, f.line_number, f.target, f.confidence) for f in r.findings+ ]+ assert key(first) == key(second)+++def test_empty_repository_produces_an_empty_report():+ report = _analyze({})+ assert report.total_findings == 0+ assert report.documents_scanned == 0+ assert report.confidence_summary == {"high": 0, "medium": 0, "low": 0}+++# ---------------------------------------------------------------------------+# Confidence vocabulary+# ---------------------------------------------------------------------------+++def test_every_origin_has_a_confidence():+ """The Literal vocabulary and the confidence table are two views of one+ thing, so neither may carry a name the other lacks."""+ assert frozenset(ORIGIN_CONFIDENCE) == DRIFT_ORIGIN_VALUES++[email protected]("origin", sorted(DRIFT_ORIGIN_VALUES))+def test_confidences_are_probabilities(origin: str):+ assert 0.0 < ORIGIN_CONFIDENCE[origin] <= 1.0+++def test_default_cutoff_hides_nothing_that_ships():+ """Every shipping origin sits at or above the default cutoff, so a user+ who passes no flag sees the whole detector."""+ assert min(ORIGIN_CONFIDENCE.values()) >= DEFAULT_MIN_CONFIDENCE+++def test_bucket_boundaries_are_ordered():+ assert 0.0 < REVIEW_CONFIDENCE_THRESHOLD < HIGH_CONFIDENCE_THRESHOLD <= 1.0+++def test_summarize_confidence_buckets_are_exhaustive():+ class _F:+ def __init__(self, c):+ self.confidence = c++ findings = [_F(0.95), _F(0.9), _F(0.5), _F(0.2)]+ summary = summarize_confidence(findings)+ assert summary == {"high": 2, "medium": 1, "low": 1}+ assert sum(summary.values()) == len(findings)diff --git a/tests/unit/doc_drift/test_confidence_parity.py b/tests/unit/doc_drift/test_confidence_parity.pynew file mode 100644index 000000000..8d0a4edf4--- /dev/null+++ b/tests/unit/doc_drift/test_confidence_parity.py@@ -0,0 +1,62 @@+"""The drift tiers mean what the dead-code tiers mean.++``doc_drift/constants.py`` copies dead code's bucket boundaries rather than+importing them, so that a change to dead code's *deletion-safety* policy cannot+silently retune a *display* tier in an unrelated detector. That decoupling is+deliberate, but it is only honest if someone notices when the two drift: a+reader comparing the two reports is entitled to assume "high" means one thing.++This test is that notice. If dead code moves its thresholds and drift should+follow, change both and update this file in the same commit. If they should+diverge, delete this test and say why in the comment above the constants.+"""++from __future__ import annotations++from repowise.core.analysis.dead_code.risk_factors import (+ RISK_CAP_CONFIDENCE,+ SAFE_CONFIDENCE_THRESHOLD,+)+from repowise.core.analysis.doc_drift.constants import (+ HIGH_CONFIDENCE_THRESHOLD,+ REVIEW_CONFIDENCE_THRESHOLD,+ bucket_confidences,+)+++def test_high_boundary_matches_dead_code():+ assert HIGH_CONFIDENCE_THRESHOLD == SAFE_CONFIDENCE_THRESHOLD+++def test_review_boundary_matches_dead_code():+ assert REVIEW_CONFIDENCE_THRESHOLD == RISK_CAP_CONFIDENCE+++def test_buckets_are_exhaustive_and_disjoint():+ values = [1.0, 0.95, 0.7, 0.699, 0.5, 0.4, 0.399, 0.0]+ summary = bucket_confidences(values)+ assert sum(summary.values()) == len(values)+ assert set(summary) == {"high", "medium", "low"}+++def test_boundaries_are_inclusive_at_the_bottom_of_each_tier():+ """A value exactly on a boundary belongs to the higher tier."""+ assert bucket_confidences([HIGH_CONFIDENCE_THRESHOLD])["high"] == 1+ assert bucket_confidences([REVIEW_CONFIDENCE_THRESHOLD])["medium"] == 1+++def test_the_analyzer_and_the_read_path_bucket_identically():+ """The CRUD layer re-derives the summary on read; both call one function,+ and this pins that they keep agreeing."""+ from repowise.core.analysis.doc_drift import summarize_confidence+ from repowise.core.persistence.crud.analysis.doc_drift import (+ summarize_confidence_rows,+ )++ class _Row:+ def __init__(self, c: float) -> None:+ self.confidence = c++ values = [0.95, 0.9, 0.5, 0.2]+ rows = [_Row(v) for v in values]+ assert summarize_confidence(rows) == summarize_confidence_rows(rows)diff --git a/tests/unit/doc_drift/test_extractor.py b/tests/unit/doc_drift/test_extractor.pynew file mode 100644index 000000000..d27cdfe38--- /dev/null+++ b/tests/unit/doc_drift/test_extractor.py@@ -0,0 +1,266 @@+"""Extraction rules for the documentation drift detector.++One test per rule Phase 1 learned the hard way. Each docstring names the+measurement that justifies the rule, so a future reader retuning one can see+what it costs.+"""++from __future__ import annotations++import pytest++from repowise.core.analysis.doc_drift.extractor import (+ extract,+ is_checkable_document,+ is_guide_document,+ prose_lines,+)+from repowise.core.analysis.doc_drift.models import DriftKind+++def _kinds(text: str, doc: str = "docs/x.md") -> set[DriftKind]:+ return {r.kind for r in extract(text, doc)}+++def _targets(text: str, kind: DriftKind, doc: str = "docs/x.md") -> list[str]:+ return [r.target for r in extract(text, doc) if r.kind is kind]+++# ---------------------------------------------------------------------------+# Fenced and generated spans+# ---------------------------------------------------------------------------+++def test_fenced_code_contributes_nothing():+ """Paths inside a fence are illustrative by construction."""+ text = "Real: `src/a.py`\n```\nsee `src/invented.py`\n```\nAfter: `src/b.py`\n"+ targets = _targets(text, DriftKind.PATH)+ assert "src/invented.py" not in targets+ assert targets == ["src/a.py", "src/b.py"]+++def test_tilde_fences_are_fences_too():+ text = "~~~\n`src/invented.py`\n~~~\n"+ assert _targets(text, DriftKind.PATH) == []+++def test_generated_span_is_skipped():+ """Repowise's own output is not a drift target.++ The START marker carries a trailing sentence and an em dash+ (``editor_files/base.py:28``), so the skip must match on the prefix rather+ than the full literal.+ """+ text = (+ "Mine: `src/a.py`\n"+ "<!-- REPOWISE:START — Do not edit below this line. "+ "Auto-generated by Repowise. -->\n"+ "Generated: `src/generated.py`\n"+ "<!-- REPOWISE:END -->\n"+ "Mine again: `src/b.py`\n"+ )+ targets = _targets(text, DriftKind.PATH)+ assert targets == ["src/a.py", "src/b.py"]++[email protected]("tag", ["REPOWISE", "REPOWISE_AGENTS", "REPOWISE_DISTILL"])+def test_every_live_marker_tag_is_skipped(tag: str):+ """All three tags in use today, so a new generator does not leak findings."""+ text = f"<!-- {tag}:START -->\n`src/generated.py`\n<!-- {tag}:END -->\n"+ assert _targets(text, DriftKind.PATH) == []+++def test_line_numbers_survive_skipped_spans():+ """A finding's line has to point at the real file."""+ text = "intro\n```\nfenced\n```\n`src/a.py`\n"+ refs = extract(text, "docs/x.md")+ assert [r.line for r in refs] == [5]+++# ---------------------------------------------------------------------------+# Rule 3: commands only from inline code spans+# ---------------------------------------------------------------------------+++def test_commands_are_read_only_from_inline_code():+ """Scanning bare prose matched English.++ Phase 1's first run produced "make room", "make that", "make the" and+ "make no" as missing build targets: 43 of 55 findings, entirely noise.+ """+ assert _targets("We make room for this and make the call.", DriftKind.COMMAND) == []+ assert _targets("Run `make build` first.", DriftKind.COMMAND) == ["make:build"]+++def test_npm_run_commands_are_extracted():+ assert _targets("Run `npm run test`.", DriftKind.COMMAND) == ["npm:test"]+ assert _targets("Run `pnpm run type-check`.", DriftKind.COMMAND) == ["npm:type-check"]+++# ---------------------------------------------------------------------------+# The symbol class does not exist+# ---------------------------------------------------------------------------+++def test_backticked_identifiers_are_not_extracted():+ """Phase 1 killed the symbol class at a 55-69% flag rate.++ Backticks in technical prose mean "literal token", not "code symbol". The+ top flagged tokens were `string`, `boolean`, `true`, `OPENAI_API_KEY`.+ """+ text = "Set `OPENAI_API_KEY`, pass a `string`, and call `resolve_target`."+ assert extract(text, "docs/x.md") == []+++# ---------------------------------------------------------------------------+# Path shaping+# ---------------------------------------------------------------------------+++def test_bare_filename_is_still_extracted_but_prose_is_not():+ """Extraction is permissive; the resolver decides checkability."""+ assert _targets("See `config.yaml` for this.", DriftKind.PATH) == ["config.yaml"]+ assert _targets("That is, `e.g` and `i.e` here.", DriftKind.PATH) == []+++ "token",+ ["path/to/thing.py", "your/app/main.py", "examples/foo.py", "src/{name}.py"],+)+def test_placeholders_and_templates_are_not_paths(token: str):+ assert _targets(f"Create `{token}` now.", DriftKind.PATH) == []+++def test_urls_are_never_paths_or_links():+ text = "See [docs](https://example.com/a.py) and `https://x.dev/b.py`."+ assert extract(text, "docs/x.md") == []+++# ---------------------------------------------------------------------------+# Links and anchors+# ---------------------------------------------------------------------------+++def test_link_with_fragment_yields_both_a_link_and_an_anchor():+ """One line, two assertions: the file exists, and it declares the heading."""+ refs = extract("See [it](../README.md#setup).", "docs/x.md")+ assert {r.kind for r in refs} == {DriftKind.LINK, DriftKind.ANCHOR}+++def test_same_document_anchor_targets_its_own_file():+ refs = extract("See [below](#some-heading).", "docs/x.md")+ assert [(r.kind, r.target) for r in refs] == [+ (DriftKind.ANCHOR, "docs/x.md#some-heading")+ ]+++def test_heading_lines_still_assert_paths():+ """A section title asserts a path exactly as body prose does.++ Skipping heading lines silently dropped a real defect in this repository:+ ``ARCHITECTURE.md`` names ``git_indexer.py`` in a ``### 7.1`` heading.+ """+ text = "### 7.1 GitIndexer (`packages/core/ingestion/git_indexer.py`)\n"+ assert _targets(text, DriftKind.PATH) == ["packages/core/ingestion/git_indexer.py"]+++def test_section_trail_is_recorded_as_evidence():+ text = "## Adding a language\n\n### Step 1\n\nCreate `src/specs/x.py`.\n"+ refs = extract(text, "docs/x.md")+ assert refs[0].section == "Adding a language > Step 1"+++def test_sibling_heading_pops_the_trail():+ text = "## A\n\n### A1\n\n## B\n\nSee `src/x.py`.\n"+ refs = extract(text, "docs/x.md")+ assert refs[0].section == "B"+++# ---------------------------------------------------------------------------+# Rule 2: historical documents excluded by stem PATTERN+# ---------------------------------------------------------------------------+++ "path",+ [+ "CHANGELOG.md",+ "docs/changelog.md",+ "docs/release-notes.md",+ "docs/release_notes.md",+ "RELEASES.md",+ "docs/whatsnew.md",+ "docs/migration-guide.md",+ "UPGRADING.md",+ "docs/HISTORY.md",+ "docs/news.md",+ ],+)+def test_historical_documents_are_excluded(path: str):+ """A changelog describes the tree as it was at each release.++ Matched on the stem as a pattern, never an exact filename list: Phase 1's+ first fixture was ``CHANGELOG.md``, which silently failed on fastapi's+ ``release-notes.md`` --- and all 17 of that repo's path findings came from+ that one file.+ """+ assert not is_checkable_document(path)+++def test_ordinary_documents_are_checkable():+ assert is_checkable_document("docs/architecture/ARCHITECTURE.md")+ assert is_checkable_document("README.md")+++ "path",+ [+ "node_modules/pkg/README.md",+ "vendor/lib/README.md",+ "tests/fixtures/sample/README.md",+ "tests/unit/doc_drift/data/sample.md",+ # fastapi keeps deliberately-broken markdown here to feed its own+ # link fixer. Checking it reports a project's test inputs as its+ # documentation; this shape produced four false findings before the+ # shared classifier replaced a hand-written directory list.+ "scripts/tests/test_translation_fixer/test_markdown_links/data/en_doc.md",+ ],+)+def test_vendored_and_fixture_documents_are_excluded(path: str):+ assert not is_checkable_document(path)+++ "path",+ [+ "docs/layers/TEST_INTELLIGENCE.md",+ "docs/TESTING.md",+ "docs/test-strategy.md",+ ],+)+def test_documents_about_testing_are_not_test_files(path: str):+ """A document describing the test layer is prose, not a fixture.++ The shared classifier also recognises the ``test_*`` filename convention,+ which is a convention of code. Applied to the whole path it classes+ ``docs/layers/TEST_INTELLIGENCE.md`` as a test file and silently drops one+ of this repository's fourteen real defects, so the check is scoped to the+ containing directory.+ """+ assert is_checkable_document(path)+++def test_non_markdown_is_not_a_document():+ """`.rst` and `.adoc` are outside this detector's reach entirely."""+ assert not is_checkable_document("docs/index.rst")+ assert not is_checkable_document("docs/index.adoc")+++def test_guide_documents_are_recognised():+ assert is_guide_document(".github/CONTRIBUTING.md")+ assert is_guide_document("docs/tutorial.md")+ assert not is_guide_document("docs/architecture/ARCHITECTURE.md")+++def test_prose_lines_are_one_indexed():+ assert next(iter(prose_lines("first\nsecond\n"))) == (1, "first")diff --git a/tests/unit/doc_drift/test_renderer_gate.py b/tests/unit/doc_drift/test_renderer_gate.pynew file mode 100644index 000000000..3de7ea5fe--- /dev/null+++ b/tests/unit/doc_drift/test_renderer_gate.py@@ -0,0 +1,137 @@+"""The anchor class's renderer gate.++fastapi produced 93 anchor findings in Phase 1, every one false, purely from+assuming GitHub slugs over an MkDocs tree. The gate exists so that never ships.+"""++from __future__ import annotations++import pytest++from repowise.core.analysis.doc_drift.models import DocReference, DriftKind, DriftVerdict+from repowise.core.analysis.doc_drift.renderer import (+ GITHUB,+ RendererMap,+ github_slug,+)+from repowise.core.analysis.doc_drift.resolver import RepoIndex, resolve+++ ("marker", "expected"),+ [+ ("mkdocs.yml", "mkdocs"),+ ("mkdocs.yaml", "mkdocs"),+ ("docusaurus.config.js", "docusaurus"),+ ("docusaurus.config.ts", "docusaurus"),+ ("_config.yml", "jekyll"),+ ],+)+def test_root_marker_governs_the_whole_tree(marker: str, expected: str):+ rmap = RendererMap.detect({marker, "docs/a.md"})+ assert rmap.for_document("docs/a.md") == expected+ assert not rmap.checkable("docs/a.md")+++def test_no_marker_means_github():+ """The honest default: a repository with no site generator is read on its+ forge."""+ rmap = RendererMap.detect({"README.md", "docs/a.md"})+ assert rmap.for_document("docs/a.md") == GITHUB+ assert rmap.checkable("docs/a.md")+++def test_a_marker_governs_only_its_own_subtree():+ """This repository is why the gate is per-subtree.++ It ships a Jekyll site at ``website/_config.yml`` covering the documents+ under ``website/`` and nothing else. A repo-global gate stands the whole+ class down and loses every real anchor finding in ``docs/`` --- which is+ three of this repository's fourteen known defects.+ """+ rmap = RendererMap.detect({"website/_config.yml", "website/a.md", "docs/b.md"})+ assert rmap.for_document("website/a.md") == "jekyll"+ assert not rmap.checkable("website/a.md")+ assert rmap.for_document("docs/b.md") == GITHUB+ assert rmap.checkable("docs/b.md")+++def test_nested_site_wins_for_its_own_documents():+ rmap = RendererMap.detect(+ {"mkdocs.yml", "website/_config.yml", "website/a.md", "docs/b.md"}+ )+ assert rmap.for_document("website/a.md") == "jekyll"+ assert rmap.for_document("docs/b.md") == "mkdocs"+++def test_markers_in_vendored_or_fixture_trees_are_ignored():+ """A marker in node_modules says nothing about how this project publishes."""+ rmap = RendererMap.detect(+ {"node_modules/pkg/mkdocs.yml", "tests/fixtures/x/_config.yml", "docs/b.md"}+ )+ assert rmap.for_document("docs/b.md") == GITHUB+++def test_unsupported_renderer_makes_anchors_uncheckable_not_missing():+ """Never guess: report uncheckable rather than the wrong slug."""+ idx = RepoIndex.build(+ tracked_paths={"mkdocs.yml", "docs/a.md"},+ doc_text={"docs/a.md": "## Totally Different\n"},+ manifest_text={},+ )+ ref = DocReference(+ DriftKind.ANCHOR, "docs/a.md#gone", "docs/a.md#gone", "docs/a.md", 1, "c"+ )+ res = resolve(idx, ref)+ assert res.verdict is DriftVerdict.UNCHECKABLE+ assert res.detail == "renderer:mkdocs"+++def test_gate_follows_the_document_that_declares_the_anchors():+ """The slug is decided by how the TARGET renders, not the linking file."""+ idx = RepoIndex.build(+ tracked_paths={"website/_config.yml", "website/a.md", "docs/b.md"},+ doc_text={"website/a.md": "## Heading\n", "docs/b.md": "## Heading\n"},+ manifest_text={},+ )+ into_site = DocReference(+ DriftKind.ANCHOR, "website/a.md#gone", "website/a.md#gone", "docs/b.md", 1, "c"+ )+ into_docs = DocReference(+ DriftKind.ANCHOR, "docs/b.md#gone", "docs/b.md#gone", "docs/b.md", 1, "c"+ )+ assert resolve(idx, into_site).verdict is DriftVerdict.UNCHECKABLE+ assert resolve(idx, into_docs).verdict is DriftVerdict.MISSING+++def test_summary_names_the_detected_zones():+ assert RendererMap.detect({"README.md"}).summary() == GITHUB+ assert "jekyll:website/" in RendererMap.detect({"website/_config.yml"}).summary()+++# ---------------------------------------------------------------------------+# The slug algorithm itself+# ---------------------------------------------------------------------------+++ ("heading", "slug"),+ [+ ("Simple Heading", "simple-heading"),+ ("SQL + dbt", "sql--dbt"),+ ("Opt-in code generation", "opt-in-code-generation"),+ ("Hook efficacy: `repowise hook stats`", "hook-efficacy-repowise-hook-stats"),+ ("Quickstart (under 5 minutes, no API key)", "quickstart-under-5-minutes-no-api-key"),+ ("**Bold** heading", "bold-heading"),+ ("Trailing punctuation!", "trailing-punctuation"),+ ],+)+def test_github_slug(heading: str, slug: str):+ assert github_slug(heading) == slug+++def test_whitespace_runs_are_not_collapsed():+ """One character of regex --- ``\\s`` rather than ``\\s+`` --- was 70% of+ this class's findings in Phase 1's first run."""+ assert github_slug("a b") == "a--b"+ assert github_slug("a b") == "a---b"diff --git a/tests/unit/doc_drift/test_resolver.py b/tests/unit/doc_drift/test_resolver.pynew file mode 100644index 000000000..99c06fd9c--- /dev/null+++ b/tests/unit/doc_drift/test_resolver.py@@ -0,0 +1,248 @@+"""Resolution rules: four verdicts, and the anchoring rule that earns them."""++from __future__ import annotations++import pytest++from repowise.core.analysis.doc_drift.extractor import extract+from repowise.core.analysis.doc_drift.models import DocReference, DriftKind, DriftVerdict+from repowise.core.analysis.doc_drift.resolver import RepoIndex, resolve++TREE = frozenset(+ {+ "README.md",+ "Makefile",+ "package.json",+ "docs/guide.md",+ "docs/architecture/ARCHITECTURE.md",+ "src/app.py",+ "src/util/helpers.py",+ "src/util/dup.py",+ "other/dup.py",+ }+)+++def _index(**kw) -> RepoIndex:+ return RepoIndex.build(+ tracked_paths=kw.get("tree", TREE),+ doc_text=kw.get("docs", {}),+ manifest_text=kw.get("manifests", {}),+ )+++def _ref(target: str, kind=DriftKind.PATH, doc="docs/architecture/ARCHITECTURE.md"):+ return DocReference(kind, target, target, doc, 1, "ctx")+++# ---------------------------------------------------------------------------+# Rule 1: a reference is checkable only when anchored+# ---------------------------------------------------------------------------+++def test_exact_path_resolves():+ assert resolve(_index(), _ref("src/app.py")).verdict is DriftVerdict.RESOLVED+++def test_directory_resolves():+ assert resolve(_index(), _ref("src/util")).verdict is DriftVerdict.RESOLVED+++def test_anchored_path_with_no_candidate_is_missing():+ res = resolve(_index(), _ref("src/util/gone.py"))+ assert res.verdict is DriftVerdict.MISSING+ assert res.origin == "path_no_candidate"+++ "target",+ ["CLAUDE.md", "config.yaml", "state.json", "login.py", "opencode.json"],+)+def test_bare_filenames_are_never_checkable(target: str):+ """Rule 1. Bare filenames are never checkable.++ These are files Repowise writes into the user's tree, or invented+ examples. Phase 1's first run flagged 639 paths at a 49% rate, nearly all+ of this family. Requiring a separator plus a real first segment took the+ class to 1.3%.+ """+ assert resolve(_index(), _ref(target)).verdict is DriftVerdict.UNCHECKABLE+++ "target", ["routes/web.php", "vendor/thing/ext_localconf.php", "app/models/user.rb"]+)+def test_foreign_first_segment_is_uncheckable(target: str):+ """A framework the docs merely describe is not this repository."""+ res = resolve(_index(), _ref(target))+ assert res.verdict is DriftVerdict.UNCHECKABLE+ assert res.detail == "unanchored"+++# ---------------------------------------------------------------------------+# Rule 6: ambiguous is never reported as missing+# ---------------------------------------------------------------------------+++def test_single_basename_match_is_ambiguous_not_missing():+ """Phase 1 found five of these. Reporting them as drift would be wrong."""+ res = resolve(_index(), _ref("src/helpers.py"))+ assert res.verdict is DriftVerdict.AMBIGUOUS+ assert res.confidence == 0.0+++def test_multiple_basename_matches_are_ambiguous():+ res = resolve(_index(), _ref("src/nested/dup.py"))+ assert res.verdict is DriftVerdict.AMBIGUOUS+ assert "basename-multi" in res.detail+++def test_relative_link_resolves_against_its_own_document():+ ref = DocReference(DriftKind.LINK, "../guide.md", "../guide.md",+ "docs/architecture/ARCHITECTURE.md", 1, "ctx")+ res = resolve(_index(), ref)+ assert res.verdict is DriftVerdict.RESOLVED+ assert res.detail == "relative-to-doc"+++# ---------------------------------------------------------------------------+# Anchors: rules 4 and 5+# ---------------------------------------------------------------------------+++def _anchor_index(readme: str, tree=None):+ return RepoIndex.build(+ tracked_paths=tree or TREE,+ doc_text={"README.md": readme},+ manifest_text={},+ )+++def _anchor_ref(fragment: str):+ return DocReference(+ DriftKind.ANCHOR, f"README.md#{fragment}", f"README.md#{fragment}",+ "docs/guide.md", 1, "ctx",+ )+++def test_heading_anchor_resolves():+ idx = _anchor_index("## Start in minutes\n")+ assert resolve(idx, _anchor_ref("start-in-minutes")).verdict is DriftVerdict.RESOLVED+++def test_renamed_heading_is_missing():+ """The class no linter catches: each document is internally valid and the+ target file exists. Only the pair is wrong."""+ idx = _anchor_index("## Start in minutes (no API key)\n")+ res = resolve(idx, _anchor_ref("quickstart-under-5-minutes-no-api-key"))+ assert res.verdict is DriftVerdict.MISSING+ assert res.origin == "anchor_no_heading"++[email protected]("tag", ['<a id="code-health"></a>', "<a name='code-health'>"])+def test_explicit_html_anchors_are_link_targets(tag: str):+ """Rule 4. A declared target no heading slug reproduces.++ A heading-only checker calls every one of them broken; this was the single+ largest false-positive family in Phase 1's first run.+ """+ idx = _anchor_index(f"{tag}\n\n## Something Else\n")+ assert resolve(idx, _anchor_ref("code-health")).verdict is DriftVerdict.RESOLVED+++def test_each_whitespace_character_becomes_its_own_dash():+ """Rule 5. ``SQL + dbt`` slugs to ``sql--dbt``, not ``sql-dbt``.++ Collapsing whitespace runs accounted for 70% of this class's findings in+ Phase 1's first run.+ """+ idx = _anchor_index("## SQL + dbt\n")+ assert resolve(idx, _anchor_ref("sql--dbt")).verdict is DriftVerdict.RESOLVED+ assert resolve(idx, _anchor_ref("sql-dbt")).verdict is DriftVerdict.MISSING+++def test_anchor_into_a_missing_document_is_uncheckable_not_missing():+ """The LINK row already reports the missing file; saying it twice would+ double-count one defect."""+ idx = _anchor_index("## Heading\n")+ ref = DocReference(DriftKind.ANCHOR, "docs/gone.md#x", "docs/gone.md#x",+ "docs/guide.md", 1, "ctx")+ assert resolve(idx, ref).verdict is DriftVerdict.UNCHECKABLE+++# ---------------------------------------------------------------------------+# Commands+# ---------------------------------------------------------------------------+++def test_make_target_resolves_and_missing_target_is_flagged():+ idx = _index(manifests={"Makefile": "build:\n\techo hi\ntest:\n\techo t\n"})+ ok = DocReference(DriftKind.COMMAND, "make build", "make:build", "docs/guide.md", 1, "c")+ bad = DocReference(DriftKind.COMMAND, "make gone", "make:gone", "docs/guide.md", 1, "c")+ assert resolve(idx, ok).verdict is DriftVerdict.RESOLVED+ assert resolve(idx, bad).verdict is DriftVerdict.MISSING+++def test_make_variable_assignment_is_not_a_target():+ """``CC := gcc`` is an assignment, not a target."""+ idx = _index(manifests={"Makefile": "CC := gcc\nbuild:\n\techo hi\n"})+ ref = DocReference(DriftKind.COMMAND, "make CC", "make:CC", "docs/guide.md", 1, "c")+ assert resolve(idx, ref).verdict is DriftVerdict.MISSING+++def test_commands_are_uncheckable_without_a_manifest():+ idx = _index(manifests={})+ ref = DocReference(DriftKind.COMMAND, "make build", "make:build", "docs/guide.md", 1, "c")+ assert resolve(idx, ref).verdict is DriftVerdict.UNCHECKABLE+++def test_npm_script_resolves():+ idx = _index(manifests={"package.json": '{"scripts": {"build": "tsc"}}'})+ ok = DocReference(DriftKind.COMMAND, "npm run build", "npm:build", "docs/guide.md", 1, "c")+ assert resolve(idx, ok).verdict is DriftVerdict.RESOLVED+++def test_malformed_package_json_does_not_crash():+ idx = _index(manifests={"package.json": "{not json"})+ ref = DocReference(DriftKind.COMMAND, "npm run build", "npm:build", "docs/guide.md", 1, "c")+ assert resolve(idx, ref).verdict is DriftVerdict.UNCHECKABLE+++# ---------------------------------------------------------------------------+# The guide confidence tier+# ---------------------------------------------------------------------------+++def test_guide_documents_drop_one_confidence_tier():+ """A tier, not an exclusion: a guide naming a real file that moved is+ still drift."""+ plain = resolve(_index(), _ref("src/util/gone.py"), is_guide=False)+ guide = resolve(_index(), _ref("src/util/gone.py"), is_guide=True)+ assert plain.verdict is guide.verdict is DriftVerdict.MISSING+ assert guide.confidence < plain.confidence+++def test_only_missing_carries_confidence():+ for target in ("src/app.py", "CLAUDE.md", "src/helpers.py"):+ assert resolve(_index(), _ref(target)).confidence == 0.0+++def test_end_to_end_on_a_small_document():+ """Extraction and resolution agree on a document mixing all four classes."""+ text = (+ "# Doc\n\n"+ "Real path `src/app.py`, gone path `src/util/gone.py`.\n"+ "Run `make build`. See [readme](README.md#heading).\n"+ )+ idx = RepoIndex.build(+ tracked_paths=TREE,+ doc_text={"README.md": "## Heading\n"},+ manifest_text={"Makefile": "build:\n\techo\n"},+ )+ verdicts = {+ (r.kind, resolve(idx, r).verdict) for r in extract(text, "docs/guide.md")+ }+ assert (DriftKind.PATH, DriftVerdict.RESOLVED) in verdicts+ assert (DriftKind.PATH, DriftVerdict.MISSING) in verdicts+ assert (DriftKind.COMMAND, DriftVerdict.RESOLVED) in verdicts+ assert (DriftKind.ANCHOR, DriftVerdict.RESOLVED) in verdictsdiff --git a/tests/unit/doc_drift/test_review_regressions.py b/tests/unit/doc_drift/test_review_regressions.pynew file mode 100644index 000000000..6426741a1--- /dev/null+++ b/tests/unit/doc_drift/test_review_regressions.py@@ -0,0 +1,228 @@+"""One test per defect found reviewing this phase.++Each of these shipped briefly and each was a *false positive at high+confidence* or a silently dropped document, which are the two ways this+detector can lose a reader's trust. They are collected here, rather than spread+through the other files, so the cost of the design is visible in one place.+"""++from __future__ import annotations++import pytest++from repowise.core.analysis.doc_drift.extractor import _normalize, extract, prose_lines+from repowise.core.analysis.doc_drift.models import DriftKind, DriftVerdict+from repowise.core.analysis.doc_drift.renderer import github_slug+from repowise.core.analysis.doc_drift.resolver import RepoIndex, resolve++TREE = frozenset({"README.md", "docs/a.md", "docs/guide.md", "guide.md", "src/app.py"})+++def _index(tree=TREE, docs=None, manifests=None) -> RepoIndex:+ return RepoIndex.build(tree, docs or {}, manifests or {})+++def _one(text: str, kind: DriftKind, doc: str = "docs/a.md"):+ return next(r for r in extract(text, doc) if r.kind is kind)+++# ---------------------------------------------------------------------------+# The index is not the repository+# ---------------------------------------------------------------------------+++ "target",+ ["docs/images/logo.png", "docs/style.css", "docs/notes.txt", "docs/yarn.lock"],+)+def test_references_to_file_kinds_the_index_cannot_hold_are_uncheckable(target: str):+ """The traverser keeps only files it can identify, so assets, stylesheets,+ plain text and lockfiles never reach the index. Resolving against a tree+ that structurally cannot contain them and calling the document wrong blames+ the author for the indexer's scope --- the single largest false-positive+ family in this design.+ """+ ref = _one(f"See [x]({target}).", DriftKind.LINK)+ res = resolve(_index(), ref)+ assert res.verdict is DriftVerdict.UNCHECKABLE+ assert res.detail.startswith("unindexed-kind:")+++def test_a_kind_the_index_does_hold_is_still_checkable():+ """The rule must not swallow the classes it was built for."""+ ref = _one("See `src/gone.py`.", DriftKind.PATH)+ assert resolve(_index(), ref).verdict is DriftVerdict.MISSING+++# ---------------------------------------------------------------------------+# Normalization must not rewrite the path it is normalizing+# ---------------------------------------------------------------------------+++def test_dot_directories_survive_normalization():+ """``lstrip("./")`` strips characters, not a prefix, so ``.github/...``+ became ``github/...``: mis-keyed, dropped as unanchored, and written to the+ database as something the document does not say."""+ assert _normalize(".github/workflows/ci.yml") == ".github/workflows/ci.yml"+++def test_parent_segments_survive_normalization():+ """``../`` is what lets a link be resolved against its own document."""+ assert _normalize("../../docs/a.md") == "../../docs/a.md"+++def test_leading_current_directory_is_still_removed():+ assert _normalize("./docs/a.md") == "docs/a.md"+++# ---------------------------------------------------------------------------+# Span handling+# ---------------------------------------------------------------------------+++def test_a_marker_shown_inside_a_fence_does_not_swallow_the_document():+ """This project's own docs show the generated-content markers in examples.+ Testing for START before honouring the fence turned the skip on with no+ matching END outside the fence, discarding the rest of the file."""+ text = (+ "# Title\n```markdown\n<!-- REPOWISE:START - example -->\n```\n"+ "See `src/gone.py` here.\n"+ )+ assert [(r.line, r.target) for r in extract(text, "docs/a.md")] == [+ (5, "src/gone.py")+ ]+++def test_a_start_and_end_on_one_line_open_nothing():+ text = "<!-- X:START --><!-- X:END -->\nSee `src/gone.py`.\n"+ assert [r.target for r in extract(text, "docs/a.md")] == ["src/gone.py"]+++def test_a_tilde_run_does_not_close_a_backtick_fence():+ """Only the marker that opened a fence may close it; otherwise the state+ inverts for the remainder of the document."""+ text = "```\n~~~\n`src/invented.py`\n```\nSee `src/real.py`.\n"+ assert [r.target for r in extract(text, "docs/a.md")] == ["src/real.py"]+++def test_line_numbers_count_newlines_only():+ """``splitlines()`` also breaks on U+2028 and friends, which arrive by+ copy-paste; git does not count them, and ``line_number`` is the key."""+ refs = extract("Intro more\nSee `src/gone.py`\n", "docs/a.md")+ assert [r.line for r in refs] == [2]+++def test_contributor_list_markers_are_recognised():+ """``ALL-CONTRIBUTORS-LIST`` is the commonest generated block in the wild,+ and it is full of avatar image links."""+ text = (+ "<!-- ALL-CONTRIBUTORS-LIST:START - Do not edit -->\n"+ "`src/generated.py`\n"+ "<!-- ALL-CONTRIBUTORS-LIST:END -->\n"+ )+ assert extract(text, "docs/a.md") == []+++# ---------------------------------------------------------------------------+# Slugging reads rendered text+# ---------------------------------------------------------------------------+++ ("heading", "slug"),+ [+ ("[Contributing](CONTRIBUTING.md)", "contributing"),+ ("<code>foo</code>", "foo"),+ (" Title", "title"),+ # Inside a code span the angle brackets are literal, and GitHub slugs+ # the word in. Stripping it as a tag invented four broken links out of+ # four correct ones in this repository's CLI reference.+ ("`repowise distill <command>`", "repowise-distill-command"),+ ("`repowise workspace add <path>`", "repowise-workspace-add-path"),+ ],+)+def test_slug_uses_rendered_text(heading: str, slug: str):+ assert github_slug(heading) == slug+++def test_setext_headings_declare_anchors():+ """``Title`` over ``=====`` is a heading and GitHub gives it an anchor."""+ idx = _index(docs={"README.md": "Installation\n============\n"})+ ref = _one("See [i](../README.md#installation).", DriftKind.ANCHOR)+ assert resolve(idx, ref).verdict is DriftVerdict.RESOLVED+++def test_anchors_declared_on_any_element_count():+ idx = _index(+ docs={"README.md": '<div id="code-health"></div>\n<a href="x" id="other">z</a>\n'}+ )+ for frag in ("code-health", "other"):+ ref = _one(f"See [x](../README.md#{frag}).", DriftKind.ANCHOR)+ assert resolve(idx, ref).verdict is DriftVerdict.RESOLVED+++def test_headings_inside_a_fence_do_not_declare_anchors():+ """The anchor index reads through the same prose filter the extractor uses,+ so the two halves cannot disagree about what is code."""+ idx = _index(docs={"README.md": "```sh\n# Install dependencies\n```\n"})+ ref = _one("See [x](../README.md#install-dependencies).", DriftKind.ANCHOR)+ assert resolve(idx, ref).verdict is DriftVerdict.MISSING+++def test_anchor_host_resolves_relative_to_the_linking_document():+ """A markdown link means the file beside its own document. Preferring a+ verbatim root-relative hit read the wrong file whenever two documents+ shared a basename, then reported the correct link as drift at 0.95."""+ idx = _index(docs={"guide.md": "# Intro\n", "docs/guide.md": "# Setup\n"})+ ref = _one("See [g](guide.md#setup).", DriftKind.ANCHOR)+ assert resolve(idx, ref).verdict is DriftVerdict.RESOLVED+++# ---------------------------------------------------------------------------+# Commands+# ---------------------------------------------------------------------------+++def test_bare_and_slashed_and_multi_targets_all_resolve():+ """Three shapes the shared Makefile parser has to get right: a target with+ no prerequisites (the commonest form of all, and previously unmatched), a+ directory-style target, and one rule declaring two targets."""+ makefile = "docs/build:\n\techo\nbuild:\n\techo\ntest lint:\n\techo\n"+ idx = _index(tree=TREE | {"Makefile"}, manifests={"Makefile": makefile})+ for name in ("docs/build", "build", "test", "lint"):+ ref = _one(f"Run `make {name}`.", DriftKind.COMMAND)+ assert resolve(idx, ref).verdict is DriftVerdict.RESOLVED, name+++def test_assignment_is_still_not_a_target():+ idx = _index(tree=TREE | {"Makefile"}, manifests={"Makefile": "CC := gcc\nb:\n\techo\n"})+ ref = _one("Run `make CC`.", DriftKind.COMMAND)+ assert resolve(idx, ref).verdict is DriftVerdict.MISSING+++# ---------------------------------------------------------------------------+# Tuning that turned out to matter+# ---------------------------------------------------------------------------+++def test_real_second_level_directories_are_not_placeholders():+ """Checking the first two segments dropped ``app/user/models.py`` and+ ``src/example/main.py`` wholesale: ``user`` and ``example`` are ordinary+ second-level directory names."""+ assert [r.target for r in extract("See `app/user/models.py`.", "docs/a.md")] == [+ "app/user/models.py"+ ]+++def test_placeholder_first_segment_is_still_rejected():+ assert extract("See `path/to/thing.py`.", "docs/a.md") == []+++def test_directory_references_are_out_of_scope_for_this_phase():+ """Phase 1 measured file references only. Admitting directories produced+ five findings on this repository that were all illustration."""+ assert extract("See `docs/adr/` for records.", "docs/a.md") == []+++def test_prose_lines_is_one_indexed_and_skips_nothing_it_should_keep():+ assert list(prose_lines("a\nb\n"))[:2] == [(1, "a"), (2, "b")]diff --git a/tests/unit/persistence/test_doc_drift_crud.py b/tests/unit/persistence/test_doc_drift_crud.pynew file mode 100644index 000000000..de700b270--- /dev/null+++ b/tests/unit/persistence/test_doc_drift_crud.py@@ -0,0 +1,186 @@+"""Persistence for documentation drift findings.++The exit criterion for this phase is that re-running the pass changes no rows,+so idempotency is the thing these tests exist to pin.+"""++from __future__ import annotations++from sqlalchemy import select++from repowise.core.analysis.doc_drift.models import DocDriftFindingData, DriftKind+from repowise.core.persistence.crud import (+ get_doc_drift_findings,+ replace_doc_drift_findings,+)+from repowise.core.persistence.crud.analysis.doc_drift import summarize_confidence_rows+from repowise.core.persistence.models import DocDriftFinding+from tests.unit.persistence.helpers import insert_repo+++def _finding(+ file_path: str = "docs/a.md",+ line: int = 10,+ target: str = "src/gone.py",+ kind: DriftKind = DriftKind.PATH,+ confidence: float = 0.9,+) -> DocDriftFindingData:+ return DocDriftFindingData(+ kind=kind,+ file_path=file_path,+ line_number=line,+ target=target,+ confidence=confidence,+ reason="Document names src/gone.py, which no longer exists.",+ origin="path_no_candidate",+ evidence=[f"{file_path}:{line} states `{target}`"],+ raw=target,+ context=f"See `{target}`.",+ )+++async def _rows(session, repo_id: str) -> list[DocDriftFinding]:+ stmt = select(DocDriftFinding).where(DocDriftFinding.repository_id == repo_id)+ return list((await session.execute(stmt)).scalars().all())+++async def test_findings_round_trip(async_session):+ repo = await insert_repo(async_session)+ await replace_doc_drift_findings(async_session, repo.id, [_finding()])+ await async_session.commit()++ rows = await get_doc_drift_findings(async_session, repo.id)+ assert len(rows) == 1+ assert rows[0].file_path == "docs/a.md"+ assert rows[0].kind == "path"+ assert rows[0].line_number == 10+ assert rows[0].origin == "path_no_candidate"+ assert rows[0].evidence_json.startswith("[")+++async def test_rerunning_the_same_pass_changes_no_rows(async_session):+ """The phase exit criterion."""+ repo = await insert_repo(async_session)+ findings = [_finding(), _finding(line=20, target="src/other.py")]++ await replace_doc_drift_findings(async_session, repo.id, findings)+ await async_session.commit()+ first = {(r.file_path, r.kind, r.line_number, r.target) for r in await _rows(async_session, repo.id)}++ await replace_doc_drift_findings(async_session, repo.id, findings)+ await async_session.commit()+ rows = await _rows(async_session, repo.id)++ assert len(rows) == 2+ assert {(r.file_path, r.kind, r.line_number, r.target) for r in rows} == first+++async def test_a_fixed_document_loses_its_rows(async_session):+ """A run that fixed the last drifted reference must clear the rows."""+ repo = await insert_repo(async_session)+ await replace_doc_drift_findings(async_session, repo.id, [_finding()])+ await async_session.commit()++ await replace_doc_drift_findings(async_session, repo.id, [])+ await async_session.commit()+ assert await _rows(async_session, repo.id) == []+++async def test_two_references_on_one_line_are_separate_rows(async_session):+ """A markdown link with a fragment yields both a link row and an anchor+ row at the same line. A key without ``target`` would collapse them."""+ repo = await insert_repo(async_session)+ await replace_doc_drift_findings(+ async_session,+ repo.id,+ [+ _finding(kind=DriftKind.LINK, target="docs/gone.md"),+ _finding(kind=DriftKind.ANCHOR, target="docs/gone.md#frag"),+ ],+ )+ await async_session.commit()+ assert len(await _rows(async_session, repo.id)) == 2+++async def test_duplicate_findings_are_deduplicated_before_insert(async_session):+ """Two identical findings must not abort the batch after the DELETE has+ already run --- the hazard ``security_scan.replace_findings`` guards."""+ repo = await insert_repo(async_session)+ inserted = await replace_doc_drift_findings(+ async_session, repo.id, [_finding(), _finding()]+ )+ await async_session.commit()+ assert inserted == 1+ assert len(await _rows(async_session, repo.id)) == 1+++async def test_scoped_replace_leaves_other_documents_alone(async_session):+ """An incremental re-check that read four documents must not delete+ findings for the three hundred it never looked at."""+ repo = await insert_repo(async_session)+ await replace_doc_drift_findings(+ async_session,+ repo.id,+ [_finding(file_path="docs/a.md"), _finding(file_path="docs/b.md")],+ )+ await async_session.commit()++ await replace_doc_drift_findings(+ async_session,+ repo.id,+ [_finding(file_path="docs/a.md", line=99)],+ scope={"docs/a.md"},+ )+ await async_session.commit()++ rows = await _rows(async_session, repo.id)+ by_doc = {(r.file_path, r.line_number) for r in rows}+ assert ("docs/b.md", 10) in by_doc+ assert ("docs/a.md", 99) in by_doc+ assert ("docs/a.md", 10) not in by_doc+++async def test_scoped_replace_drops_findings_outside_its_scope(async_session):+ """A row outside the scope would never be deleted by the next scoped run,+ so it would outlive its own evidence."""+ repo = await insert_repo(async_session)+ inserted = await replace_doc_drift_findings(+ async_session,+ repo.id,+ [_finding(file_path="docs/a.md"), _finding(file_path="docs/elsewhere.md")],+ scope={"docs/a.md"},+ )+ await async_session.commit()+ assert inserted == 1+ assert {r.file_path for r in await _rows(async_session, repo.id)} == {"docs/a.md"}+++async def test_min_confidence_filter_on_read(async_session):+ repo = await insert_repo(async_session)+ await replace_doc_drift_findings(+ async_session,+ repo.id,+ [_finding(confidence=0.95), _finding(line=11, confidence=0.5)],+ )+ await async_session.commit()++ assert len(await get_doc_drift_findings(async_session, repo.id, min_confidence=0.9)) == 1+ assert len(await get_doc_drift_findings(async_session, repo.id)) == 2+++async def test_read_side_buckets_match_the_analyzer(async_session):+ """The CRUD layer re-derives the buckets on read; a drift between the two+ is what ``test_confidence_parity`` in dead code exists to prevent."""+ from repowise.core.analysis.doc_drift import summarize_confidence++ repo = await insert_repo(async_session)+ findings = [+ _finding(line=1, confidence=0.95),+ _finding(line=2, confidence=0.9),+ _finding(line=3, confidence=0.5),+ ]+ await replace_doc_drift_findings(async_session, repo.id, findings)+ await async_session.commit()++ rows = await get_doc_drift_findings(async_session, repo.id)+ assert summarize_confidence_rows(rows) == summarize_confidence(findings)diff --git a/tests/unit/persistence/test_models.py b/tests/unit/persistence/test_models.pyindex fbcd4fe54..d0f6b5385 100644--- a/tests/unit/persistence/test_models.py+++ b/tests/unit/persistence/test_models.py@@ -305,6 +305,7 @@ def test_base_includes_all_models():"git_function_blame","fix_events","dead_code_findings",+ "doc_drift_findings","decision_records","decision_evidence","decision_edges",diff --git a/tests/unit/pipeline/test_doc_drift_phase.py b/tests/unit/pipeline/test_doc_drift_phase.pynew file mode 100644index 000000000..058b1d105--- /dev/null+++ b/tests/unit/pipeline/test_doc_drift_phase.py@@ -0,0 +1,76 @@+"""The drift pass as a member of the analysis gather.++Pins the wiring rather than the detection rules, which+``tests/unit/doc_drift/`` owns: that the pass runs as part of a real pipeline,+that its report reaches ``PipelineResult``, and that a failure inside it cannot+take the other three analyses down with it.+"""++from __future__ import annotations++from repowise.core.pipeline import run_pipeline+from repowise.core.pipeline.phases.analysis import _run_doc_drift_analysis+++def _write_repo(root):+ """A repo whose README tells one truth and one lie."""+ root.mkdir()+ (root / "pkg").mkdir()+ (root / "pkg" / "main.py").write_text("def hello():\n return 1\n", encoding="utf-8")+ (root / "README.md").write_text(+ "# Demo\n\n"+ "The entry point is `pkg/main.py`.\n"+ "Helpers live in `pkg/helpers.py`.\n",+ encoding="utf-8",+ )+ return root+++async def test_pipeline_produces_a_drift_report(tmp_path):+ result = await run_pipeline(_write_repo(tmp_path / "repo"), test_run=True)++ report = result.doc_drift_report+ assert report is not None+ assert report.documents_scanned == 1+ assert [f.target for f in report.findings] == ["pkg/helpers.py"]+ assert report.findings[0].file_path == "README.md"+++async def test_markdown_bytes_reach_the_pass_through_source_map(tmp_path):+ """The pass reads bytes ingestion already decoded rather than re-walking.++ Markdown is a registered language spec whose parser returns an empty+ ``ParsedFile`` rather than ``None``, so its bytes land in ``source_map``+ like any other file. If that ever changes, this fails rather than the+ detector silently scanning nothing.+ """+ result = await run_pipeline(_write_repo(tmp_path / "repo"), test_run=True)+ assert "README.md" in result.source_map+ assert result.source_map["README.md"].startswith(b"# Demo")+++async def test_a_correct_document_yields_no_findings(tmp_path):+ root = tmp_path / "repo"+ root.mkdir()+ (root / "pkg").mkdir()+ (root / "pkg" / "main.py").write_text("def hello():\n return 1\n", encoding="utf-8")+ (root / "README.md").write_text("Entry point: `pkg/main.py`.\n", encoding="utf-8")++ result = await run_pipeline(root, test_run=True)+ assert result.doc_drift_report.total_findings == 0+++async def test_a_failure_inside_the_pass_is_swallowed():+ """The gather has no per-member exception wrapper, so each member must be+ total: a raising analysis would otherwise kill dead code, health and+ decisions alongside it.+ """++ class _Exploding(dict):+ def __iter__(self):+ raise RuntimeError("boom")++ # Non-empty, so the analyzer does not swap in a plain ``{}`` for a falsy+ # source map before it ever touches this one.+ exploding = _Exploding({"README.md": b"# hi\n"})+ assert await _run_doc_drift_analysis(exploding, progress=None) is None