Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(workspace): read contracts from the parsed index, and walk each repo once (#1581)

Raghav Chamadiya2cf2b86436bf
diff --git a/packages/core/src/repowise/core/workspace/contracts.py b/packages/core/src/repowise/core/workspace/contracts.py
index e9830d81a..e088b7741 100644
--- a/packages/core/src/repowise/core/workspace/contracts.py
+++ b/packages/core/src/repowise/core/workspace/contracts.py
@@ -17,7 +17,7 @@ from collections import defaultdict
from dataclasses import asdict, dataclass, field
from datetime import UTC, datetime
from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
from repowise.core.workspace.config import (
WORKSPACE_DATA_DIR,
@@ -26,6 +26,9 @@ from repowise.core.workspace.config import (
)
from repowise.core.workspace.contract_schema import ContractSchema
+if TYPE_CHECKING:
+ from repowise.core.workspace.extractors.service_boundary import ServiceBoundary
+
_log = logging.getLogger("repowise.workspace.contracts")
# ---------------------------------------------------------------------------
@@ -674,17 +677,22 @@ async def run_contract_extraction(
ws_config: WorkspaceConfig,
workspace_root: Path,
changed_repos: list[str],
+ boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,
) -> ContractStore:
"""Full contract extraction pipeline.
Called from :func:`run_cross_repo_hooks` during ``repowise update --workspace``.
- 1. For each repo: scan files with each extractor (via ``to_thread``)
- 2. Detect service boundaries per repo
- 3. Assign service to each contract
- 4. Run matching engine
- 5. Merge manual links from ``WorkspaceConfig``
- 6. Save ``contracts.json``
+ 1. For each repo: walk once, then run every extractor over that file list
+ 2. Assign service to each contract from the repo's boundaries
+ 3. Run matching engine
+ 4. Merge manual links from ``WorkspaceConfig``
+ 5. Save ``contracts.json``
+
+ *boundaries_by_repo* is the workspace-wide boundary map computed once by
+ :func:`run_cross_repo_hooks` and shared with the system-graph build, which
+ needs the same answer. When None (a direct call, or a test), boundaries are
+ detected here instead.
"""
from .extractors import (
DataExtractor,
@@ -695,7 +703,12 @@ async def run_contract_extraction(
assign_service,
detect_service_boundaries,
)
- from .extractors.base import make_exclude_predicate
+ from .extractors.base import iter_source_files, make_exclude_predicate
+ from .extractors.from_index import (
+ EXTRACTION_LAYER_KEY,
+ LAYER_REGEX,
+ load_repo_index,
+ )
contract_config = ws_config.contracts
exclude = make_exclude_predicate(tuple(contract_config.exclude_globs))
@@ -716,8 +729,10 @@ async def run_contract_extraction(
async def _extract_one_repo(alias: str, repo_path: Path) -> list[Contract]:
contracts: list[Contract] = []
- # Service boundary detection
- boundaries = await asyncio.to_thread(detect_service_boundaries, repo_path)
+ if boundaries_by_repo is not None:
+ boundaries = boundaries_by_repo.get(alias, [])
+ else:
+ boundaries = await asyncio.to_thread(detect_service_boundaries, repo_path)
# Run enabled extractors
extractors = []
@@ -731,11 +746,45 @@ async def run_contract_extraction(
extractors.append(TopicExtractor())
if contract_config.detect_data:
extractors.append(DataExtractor())
+ if not extractors:
+ return contracts
+
+ # One walk per repo, shared by every extractor. Each used to walk and
+ # re-read the tree itself, so a file claimed by N extractors was read N
+ # times; the union of their extensions is walked once here instead.
+ wanted: frozenset[str] = frozenset()
+ for extractor in extractors:
+ wanted |= extractor.source_extensions()
+ # The walk records each file's content hash as it reads it, which is
+ # what lets the index path tell a matching parse from a stale one
+ # without a second read.
+ content_hashes: dict[str, str] = {}
+ files = await asyncio.to_thread(
+ lambda: list(iter_source_files(repo_path, wanted, exclude, content_hashes))
+ )
+
+ # Symbols ingestion already parsed for this repo, when its parse cache
+ # is usable. None means the regex dialects are the only path, which is
+ # also the answer for languages with no AST tier. Only the HTTP
+ # extractor consults it, so nothing is loaded when it is disabled.
+ index = None
+ if contract_config.detect_http:
+ index = await asyncio.to_thread(load_repo_index, repo_path)
for extractor in extractors:
- found = await asyncio.to_thread(extractor.extract, repo_path, alias, exclude)
+ kwargs = (
+ {"index": index, "content_hashes": content_hashes}
+ if isinstance(extractor, HttpExtractor)
+ else {}
+ )
+ found = await asyncio.to_thread(
+ lambda e=extractor, kw=kwargs: e.extract(
+ repo_path, alias, exclude, files, **kw
+ )
+ )
for c in found:
c.service = assign_service(c.file_path, boundaries)
+ c.meta.setdefault(EXTRACTION_LAYER_KEY, LAYER_REGEX)
contracts.extend(found)
return contracts
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index 078a9f714..3fb48c7e9 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -12,7 +12,7 @@ descend into sibling/vendored repos rooted under the workspace.
from __future__ import annotations
import os
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from fnmatch import fnmatch
from pathlib import Path
@@ -39,6 +39,15 @@ _TEST_FILE_PATTERNS = (
)
+# Repowise's own contract-extractor sources. Their docstrings and comments spell
+# out the syntax each dialect matches (``@app.get("/path")``, ``topic::orders``,
+# ``AuthServiceStub``), so scanning them yields contracts for endpoints and
+# queues that do not exist — on this repo, 14 of them. A regex over raw text
+# cannot tell an example in a comment from a live route; excluding the tree that
+# is nothing *but* examples is the cheap half of that fix.
+_SELF_EXCLUDE_GLOBS = ("*/repowise/core/workspace/extractors/*",)
+
+
def is_test_path(rel_path: str) -> bool:
"""True when *rel_path* (POSIX) lives in a test tree or is a test file."""
parts = rel_path.split("/")
@@ -55,16 +64,18 @@ def make_exclude_predicate(
) -> Callable[[str], bool]:
"""Build a ``rel_path -> bool`` skip predicate for contract extraction.
- Skips the default test/spec trees (unless *exclude_tests* is False) plus any
+ Skips the default test/spec trees (unless *exclude_tests* is False), the
+ extractors' own source tree (see :data:`_SELF_EXCLUDE_GLOBS`), plus any
user-supplied ``extra_globs`` (matched against the full POSIX path and the
bare filename).
"""
+ globs = _SELF_EXCLUDE_GLOBS + tuple(extra_globs)
def skip(rel_path: str) -> bool:
if exclude_tests and is_test_path(rel_path):
return True
name = rel_path.rsplit("/", 1)[-1]
- return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in extra_globs)
+ return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in globs)
return skip
@@ -87,10 +98,34 @@ class ScanContext:
mounts: Mapping[str, str] = field(default_factory=dict)
+# One scanned source file: ``(rel_path, suffix, content)``.
+SourceFile = tuple[str, str, str]
+
+
+def select_files(
+ repo_path: Path,
+ extensions: frozenset[str],
+ exclude: Callable[[str], bool] | None,
+ files: Sequence[SourceFile] | None,
+ content_hashes: dict[str, str] | None = None,
+) -> list[SourceFile]:
+ """Filter an already-walked *files* list, or walk *repo_path* when it is None.
+
+ Five extractors each used to walk the repo independently, so a repo was
+ traversed and every matching file re-read once per extractor. The
+ orchestrator now walks once and passes the result down; *files* being None
+ keeps the standalone call (and every direct test) working unchanged.
+ """
+ if files is None:
+ return list(iter_source_files(repo_path, extensions, exclude, content_hashes))
+ return [f for f in files if f[1] in extensions]
+
+
def iter_source_files(
repo_root: Path,
extensions: frozenset[str],
exclude: Callable[[str], bool] | None = None,
+ content_hashes: dict[str, str] | None = None,
) -> Iterator[tuple[str, str, str]]:
"""Yield ``(rel_path, suffix, content)`` for each scannable source file.
@@ -109,6 +144,14 @@ def iter_source_files(
Only files whose lower-cased suffix is in *extensions* are yielded.
Oversized, binary, generated, and ignored files are already excluded by the
traverser. Unreadable files are silently skipped.
+
+ When *content_hashes* is given it is filled with ``rel_path -> sha256`` of
+ each file's raw bytes — the same key the ingestion parse cache is stored
+ under, so a caller can tell a matching parse from a stale one. Files are
+ read as bytes and decoded here rather than via ``read_text`` so that hash
+ describes the bytes on disk; the newline translation ``read_text`` would
+ have applied is reproduced verbatim, leaving every dialect's input
+ unchanged.
"""
from repowise.core.ingestion.traverser import FileTraverser
@@ -124,7 +167,16 @@ def iter_source_files(
if exclude is not None and exclude(info.path):
continue
try:
- content = Path(info.abs_path).read_text(encoding="utf-8", errors="replace")
+ raw = Path(info.abs_path).read_bytes()
except OSError:
continue
+ content = raw.decode("utf-8", errors="replace")
+ # Universal newlines, as ``read_text`` applied before this read moved
+ # to bytes. Dialect regexes are written against ``\n``.
+ if "\r" in content:
+ content = content.replace("\r\n", "\n").replace("\r", "\n")
+ if content_hashes is not None:
+ from repowise.core.ingestion.models import compute_content_hash
+
+ content_hashes[info.path] = compute_content_hash(raw)
yield info.path, suffix, content
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/__init__.py b/packages/core/src/repowise/core/workspace/extractors/data/__init__.py
index 09cafa98c..6f058c500 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/__init__.py
@@ -13,7 +13,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from ..base import ScanContext, iter_source_files
+from ..base import ScanContext, select_files
from .ddl import DdlDialect
from .dialect import DataDialect
from .names import normalize_table_name
@@ -27,11 +27,13 @@ from .orm_models import (
from .sql_strings import SqlStringsDialect
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from pathlib import Path
from repowise.core.workspace.contracts import Contract
+ from ..base import SourceFile
+
# Table-ownership recognisers (DDL + one per ORM family).
PROVIDER_DIALECTS: tuple[DataDialect, ...] = (
DdlDialect(),
@@ -59,18 +61,25 @@ class DataExtractor:
provider_dialects: tuple[DataDialect, ...] = PROVIDER_DIALECTS
consumer_dialects: tuple[DataDialect, ...] = CONSUMER_DIALECTS
+ @classmethod
+ def source_extensions(cls) -> frozenset[str]:
+ """Every extension this extractor's dialects claim."""
+ return _union_extensions(cls.provider_dialects) | _union_extensions(
+ cls.consumer_dialects
+ )
+
def extract(
self,
repo_path: Path,
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
+ files: Sequence[SourceFile] | None = None,
) -> list[Contract]:
"""Scan all source files in *repo_path* and return Contract instances."""
- all_exts = _union_extensions(self.provider_dialects) | _union_extensions(
- self.consumer_dialects
- )
contracts: list[Contract] = []
- for rel_path, suffix, content in iter_source_files(repo_path, all_exts, exclude):
+ for rel_path, suffix, content in select_files(
+ repo_path, self.source_extensions(), exclude, files
+ ):
ctx = ScanContext(repo_alias, rel_path, suffix, content)
for dialect in self.provider_dialects:
if suffix in dialect.extensions:
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
index 01d5ef223..695ace2b1 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
@@ -84,6 +84,33 @@ _SQL_KEYWORDS = frozenset(
_MIN_LITERAL_LEN = 12
+# A SQL literal *opens* with SQL. Prose that happens to satisfy the verb shape
+# does not: a docstring reading "The incremental update path re-parses ... would
+# SELECT the whole table ... vanished from a file" clears every gate above and
+# yielded the tables ``path`` and ``would``. Anchoring to the start rejects
+# prose as a class instead of blocklisting the English words it leaks.
+#
+# The opener set is deliberately wider than the statement verbs. A query is
+# routinely preceded by a named-query comment (``-- name: get_user``, the
+# aiosql/yesql/sqlc/HugSQL convention), a ``CREATE TABLE x AS SELECT ... FROM
+# y`` whose read of ``y`` is a real consumer, an ``EXPLAIN``, or a T-SQL
+# ``DECLARE``/``SET``/``BEGIN`` preamble. Requiring a bare statement verb threw
+# all of those away; the verb gate below still decides whether it is SQL.
+_SQL_COMMENT_PREFIX_RE = re.compile(r"^(?:\s|--[^\n]*\n|/\*.*?\*/)+", re.DOTALL)
+_SQL_OPENER_RE = re.compile(
+ r"^[(\s]*(?:"
+ r"WITH|SELECT|INSERT|UPDATE|DELETE|MERGE|REPLACE|UPSERT"
+ r"|CREATE|EXPLAIN|TRUNCATE|CALL|EXEC(?:UTE)?|DECLARE|SET|BEGIN|COPY"
+ r")\b",
+ re.IGNORECASE,
+)
+
+
+def _opens_with_sql(literal: str) -> bool:
+ """True when *literal* begins with SQL, ignoring any leading comment."""
+ stripped = _SQL_COMMENT_PREFIX_RE.sub("", literal, count=1)
+ return bool(_SQL_OPENER_RE.match(stripped))
+
def _verb_for_statement(stmt: object) -> str:
kind = type(stmt).__name__.lower()
@@ -121,6 +148,7 @@ class SqlStringsDialect:
s = next(g for g in m.groups() if g is not None)
if (
len(s) >= _MIN_LITERAL_LEN
+ and _opens_with_sql(s)
and _SQL_VERB_RE.search(s)
and (_UPPER_VERB_RE.search(s) or _SQL_PUNCT_RE.search(s))
):
@@ -156,10 +184,19 @@ class SqlStringsDialect:
if stmt is None:
return None
verb = _verb_for_statement(stmt)
+ # A CTE alias is a name bound by this statement, not a table it
+ # reads. sqlglot models a reference to one as exp.Table all the
+ # same, so ``WITH ranked AS (...) ... FROM ranked`` reported a
+ # table named ``ranked``. Only unqualified references shadow.
+ cte_names = {
+ cte.alias.lower() for cte in stmt.find_all(exp.CTE) if cte.alias
+ }
for table in stmt.find_all(exp.Table):
name = table.name
if not name:
continue
+ if not table.db and name.lower() in cte_names:
+ continue
raw = f"{table.db}.{name}" if table.db else name
found.append((raw, verb))
return found if found else None
diff --git a/packages/core/src/repowise/core/workspace/extractors/from_index.py b/packages/core/src/repowise/core/workspace/extractors/from_index.py
new file mode 100644
index 000000000..0d54b2f21
--- /dev/null
+++ b/packages/core/src/repowise/core/workspace/extractors/from_index.py
@@ -0,0 +1,262 @@
+"""Index-backed contract extraction: read the symbols ingestion already parsed.
+
+Workspace mode re-derived route tables from raw text with regexes, running
+*after* the ingestion pipeline had parsed the very files it then re-read from
+disk (``update.py`` indexes every stale repo, and only then calls
+``run_cross_repo_hooks``). This module reads that parse instead.
+
+The source is the per-repo parse cache (``<repo>/.repowise/parse_cache.pkl``),
+which stores a :class:`~repowise.core.ingestion.models.ParsedFile` per file —
+symbols, their kinds, their line ranges, and ``Symbol.decorators`` holding the
+verbatim decorator text (``@router.post("")``). Reading a decorator node cannot
+pick up a route-shaped string from a comment or a docstring, and cannot lose an
+empty path, so two of the defect classes that motivated this module are gone by
+construction rather than by a better regex.
+
+**What the index does not carry.** A router's mount prefix comes from a call
+expression (``APIRouter(prefix="/x")``, ``include_router(r, prefix="/y")``), not
+from a decorator, and the cache stores extraction results rather than the tree.
+Prefix resolution therefore still reads the file text via :mod:`.http.mounts`.
+The route's *identity* is what moves onto the parse; its prefix is stitched on
+exactly as before.
+
+**Availability is not guaranteed.** :class:`ParseCache` gates every entry on
+``parser_fingerprint()``, so a repo indexed by a different repowise version
+loads as empty — measured on this workspace, two of three repos. Callers must
+treat :func:`load_repo_index` returning ``None`` as "use the regex path", which
+is also the honest answer for languages with no AST tier at all.
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import TYPE_CHECKING
+
+from .http.dialect import build_provider_contract
+from .http.mounts import compose_prefix, router_prefixes
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from repowise.core.ingestion.models import ParsedFile
+ from repowise.core.workspace.contracts import Contract
+
+ from .base import ScanContext
+
+_log = logging.getLogger(__name__)
+
+# Recorded on every contract as ``meta[EXTRACTION_LAYER_KEY]`` so the coverage
+# metric can report which layer produced it. Deliberately not ``meta["source"]``:
+# eleven gRPC dialects already use that key for *dialect* provenance
+# ("py_servicer", "go_client", "proto", ...), and reusing it would have left
+# every gRPC contract classified as neither index- nor regex-sourced.
+EXTRACTION_LAYER_KEY = "extraction_layer"
+LAYER_INDEX = "index"
+LAYER_REGEX = "regex"
+
+# ---------------------------------------------------------------------------
+# Decorator parsing
+# ---------------------------------------------------------------------------
+
+# A decorator's head and its first string argument, from the verbatim text the
+# parser stores. ``@router.post("")`` -> ("router.post", ""). The argument is
+# ``*`` not ``+``: an empty path is the idiomatic collection root on a prefixed
+# router and must survive to be stitched onto that prefix.
+_DECORATOR_RE = re.compile(r"""^@([\w.]+)\s*\(\s*(?:(?P<q>['"])(?P<arg>[^'"]*)(?P=q))?""")
+
+# HTTP verbs usable as the trailing segment of a decorator head
+# (``@router.get`` / ``@app.post``).
+_VERB_METHODS = frozenset({"get", "post", "put", "patch", "delete", "head", "options"})
+
+# Flask/Django-style ``@app.route("/x", methods=["POST"])``. Ingestion resolves
+# Flask (framework edges + dynamic hints) while the workspace regex layer was
+# FastAPI-only, so this is the framework the index path inherits for free.
+_METHODS_KW_RE = re.compile(r"""methods\s*=\s*\[([^\]]*)\]""")
+_METHOD_LITERAL_RE = re.compile(r"""['"]([A-Za-z]+)['"]""")
+
+_ROUTE_HEADS = frozenset({"route"})
+
+# The languages anything here reads decorators for. Widening this means
+# widening :func:`extract_http_providers` to match.
+INDEX_SUFFIXES = frozenset({".py"})
+
+
+def _decorator_head_and_arg(decorator: str) -> tuple[str, str | None]:
+ """Split verbatim decorator text into its dotted head and first string arg.
+
+ ``@router.post("/x")`` -> ``("router.post", "/x")``;
+ ``@router.post("")`` -> ``("router.post", "")``;
+ ``@dataclass`` -> ``("dataclass", None)``.
+ """
+ m = _DECORATOR_RE.match(decorator.strip())
+ if m is None:
+ return decorator.strip().lstrip("@"), None
+ return m.group(1), m.group("arg")
+
+
+def _routes_in_decorator(decorator: str) -> list[tuple[str, str, str]]:
+ """Return ``(router_var, METHOD, raw_path)`` for a route decorator.
+
+ A ``@app.route(...)`` carrying ``methods=[...]`` yields one entry per verb
+ (Flask's way of declaring several on one handler); a verb-named decorator
+ yields exactly one. Anything else yields nothing.
+ """
+ head, arg = _decorator_head_and_arg(decorator)
+ if arg is None or "." not in head:
+ return []
+ var, _, tail = head.rpartition(".")
+ tail = tail.lower()
+
+ if tail in _VERB_METHODS:
+ return [(var, tail.upper(), arg)]
+
+ if tail in _ROUTE_HEADS:
+ mk = _METHODS_KW_RE.search(decorator)
+ verbs = [v.upper() for v in _METHOD_LITERAL_RE.findall(mk.group(1))] if mk else []
+ # Flask defaults to GET when ``methods`` is absent.
+ return [(var, verb, arg) for verb in (verbs or ["GET"])]
+
+ return []
+
+
+# ---------------------------------------------------------------------------
+# Reading the parse cache
+# ---------------------------------------------------------------------------
+
+
+def load_repo_index(
+ repo_root: Path,
+ suffixes: frozenset[str] = INDEX_SUFFIXES,
+) -> dict[str, ParsedFile] | None:
+ """Return ``rel_path -> ParsedFile`` from the repo's parse cache.
+
+ Only entries whose path ends in one of *suffixes* are deserialized. The
+ cache holds every file the repo indexed (27 MB of pickled ``ParsedFile``
+ graphs on this repo) and only the languages read here are ever consulted,
+ so unpickling the rest would be pure memory cost — multiplied by the repos
+ running concurrently.
+
+ ``None`` when the cache is absent or unusable — a repo indexed by another
+ repowise version fails the ``parser_fingerprint`` gate and loads empty,
+ which is indistinguishable from "never indexed" and gets the same answer:
+ fall back to the regex path.
+ """
+ import pickle
+
+ from repowise.core.ingestion.parse_cache import ParseCache
+
+ cache_dir = repo_root / ".repowise"
+ if not (cache_dir / "parse_cache.pkl").is_file():
+ return None
+
+ cache = ParseCache(cache_dir)
+ try:
+ cache.load()
+ except Exception:
+ _log.debug("Parse cache unreadable for %s", repo_root, exc_info=True)
+ return None
+
+ # ParseCache exposes lookup by (FileInfo, hash) but not enumeration, and
+ # this needs the whole map. Reaching for the private dict is the coupling
+ # that buys that; log loudly rather than go dark if it ever moves.
+ entries = getattr(cache, "_entries", None)
+ if entries is None:
+ _log.warning(
+ "ParseCache no longer exposes _entries; index-backed contract "
+ "extraction is disabled and everything falls back to regex"
+ )
+ return None
+ if not entries:
+ _log.info(
+ "Parse cache for %s loaded no entries (version mismatch or empty); "
+ "contract extraction falls back to the regex path",
+ repo_root,
+ )
+ return None
+
+ out: dict[str, ParsedFile] = {}
+ for (rel_path, content_hash), blob in entries.items():
+ if not rel_path.endswith(tuple(suffixes)):
+ continue
+ try:
+ parsed = pickle.loads(blob)
+ except Exception: # a single corrupt entry must not lose the repo
+ _log.debug("Skipping unpicklable parse-cache entry %s", rel_path)
+ continue
+ # The cache key carries the hash the entry was parsed from; keep it on
+ # the object so :func:`parsed_for` can reject a stale entry.
+ parsed.content_hash = content_hash
+ out[rel_path] = parsed
+ return out or None
+
+
+def parsed_for(
+ index: dict[str, ParsedFile] | None,
+ rel_path: str,
+ content_hash: str | None,
+) -> ParsedFile | None:
+ """The parse for *rel_path*, but only if it describes the bytes on disk.
+
+ Parse-cache entries are keyed by content hash. An entry whose hash does not
+ match the file just walked describes an older version of it, and trusting
+ that would report routes which no longer exist (or miss ones that now do).
+ Without a hash to check against there is nothing to trust, so the caller
+ gets ``None`` and stays on the regex path.
+ """
+ if index is None or content_hash is None:
+ return None
+ parsed = index.get(rel_path)
+ if parsed is None:
+ return None
+ if parsed.content_hash != content_hash:
+ _log.debug("Parse-cache entry for %s is stale; using the regex path", rel_path)
+ return None
+ return parsed
+
+
+# ---------------------------------------------------------------------------
+# Provider extraction
+# ---------------------------------------------------------------------------
+
+def extract_http_providers(
+ ctx: ScanContext,
+ parsed: ParsedFile,
+) -> list[Contract]:
+ """HTTP provider contracts from *parsed*'s decorated symbols.
+
+ Routes come from ``Symbol.decorators``; the mount prefix still comes from
+ the file text (see the module docstring), so ``ctx.content`` is required.
+ """
+ prefixes = router_prefixes(ctx.content, "APIRouter|FastAPI|Flask|Blueprint")
+ known = set(prefixes) | {"app", "router", "bp", "blueprint"}
+
+ out: list[Contract] = []
+ seen: set[tuple[str, str]] = set()
+ for symbol in parsed.symbols:
+ # ``Symbol.decorators`` can repeat an entry (the parser appends per
+ # matching capture), so dedupe per symbol before building contracts.
+ for decorator in dict.fromkeys(symbol.decorators):
+ for var, method, raw_path in _routes_in_decorator(decorator):
+ if var not in known:
+ continue
+ path = compose_prefix(prefixes.get(var, ""), raw_path)
+ path = compose_prefix(ctx.mounts.get(var, ""), path)
+ key = (method, path)
+ if key in seen:
+ continue
+ seen.add(key)
+ framework = "flask" if _is_route_head(decorator) else "fastapi"
+ contract = build_provider_contract(
+ ctx, method=method, path_raw=path, framework=framework
+ )
+ if contract is not None:
+ contract.meta[EXTRACTION_LAYER_KEY] = LAYER_INDEX
+ contract.meta["handler"] = symbol.name
+ out.append(contract)
+ return out
+
+
+def _is_route_head(decorator: str) -> bool:
+ head, _ = _decorator_head_and_arg(decorator)
+ return head.rpartition(".")[2].lower() in _ROUTE_HEADS
diff --git a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
index 96b37ba73..a83a42d1a 100644
--- a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
@@ -11,7 +11,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from ..base import ScanContext, iter_source_files
+from ..base import ScanContext, select_files
from .csharp import CSharpGrpcDialect
from .dialect import GrpcDialect
from .go import GoGrpcDialect
@@ -21,11 +21,13 @@ from .python import PythonGrpcDialect
from .typescript import TypeScriptGrpcDialect
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from pathlib import Path
from repowise.core.workspace.contracts import Contract
+ from ..base import SourceFile
+
# One dialect per language/IDL; extension sets are disjoint, so exactly one runs
# per file.
DIALECTS: tuple[GrpcDialect, ...] = (
@@ -50,16 +52,22 @@ class GrpcExtractor:
dialects: tuple[GrpcDialect, ...] = DIALECTS
+ @classmethod
+ def source_extensions(cls) -> frozenset[str]:
+ """Every extension this extractor's dialects claim."""
+ return _union_extensions(cls.dialects)
+
def extract(
self,
repo_path: Path,
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
+ files: Sequence[SourceFile] | None = None,
) -> list[Contract]:
- all_exts = _union_extensions(self.dialects)
-
contracts: list[Contract] = []
- for rel_path, suffix, content in iter_source_files(repo_path, all_exts, exclude):
+ for rel_path, suffix, content in select_files(
+ repo_path, self.source_extensions(), exclude, files
+ ):
ctx = ScanContext(repo_alias, rel_path, suffix, content)
for dialect in self.dialects:
if suffix in dialect.extensions:
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
index 491a3925c..8725c6253 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
@@ -12,7 +12,8 @@ from __future__ import annotations
from typing import TYPE_CHECKING
-from ..base import ScanContext, iter_source_files
+from ..base import ScanContext, select_files
+from ..langs import PYTHON
from .aspnet import AspNetDialect
from .csharp_http import CSharpHttpDialect
from .dialect import HttpDialect
@@ -29,11 +30,13 @@ from .rust_clients import RustClientsDialect
from .spring import SpringDialect
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from pathlib import Path
from repowise.core.workspace.contracts import Contract
+ from ..base import SourceFile
+
# Route-declaration recognisers (one framework each).
PROVIDER_DIALECTS: tuple[HttpDialect, ...] = (
ExpressDialect(),
@@ -54,6 +57,12 @@ CONSUMER_DIALECTS: tuple[HttpDialect, ...] = (
)
+# Provider dialects the index path fully replaces for a file it has parsed.
+# Only the Python route regex is superseded today; every other framework still
+# runs its dialect, because nothing reads their decorators yet.
+_INDEX_BACKED_DIALECTS = frozenset({"fastapi"})
+
+
def _union_extensions(dialects: tuple[HttpDialect, ...]) -> frozenset[str]:
out: set[str] = set()
for d in dialects:
@@ -67,31 +76,70 @@ class HttpExtractor:
provider_dialects: tuple[HttpDialect, ...] = PROVIDER_DIALECTS
consumer_dialects: tuple[HttpDialect, ...] = CONSUMER_DIALECTS
+ @classmethod
+ def source_extensions(cls) -> frozenset[str]:
+ """Every extension this extractor's dialects claim."""
+ return _union_extensions(cls.provider_dialects) | _union_extensions(
+ cls.consumer_dialects
+ )
+
def extract(
self,
repo_path: Path,
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
+ files: Sequence[SourceFile] | None = None,
+ index: dict[str, object] | None = None,
+ content_hashes: dict[str, str] | None = None,
) -> list[Contract]:
"""Scan all source files in *repo_path* and return Contract instances.
Files are read once into memory so a first pass can collect repo-wide
router mounts (``include_router(prefix=...)`` / ``app.use('/x', router)``)
before the extraction pass stitches them onto each provider route.
+
+ *files* is an already-walked ``(rel_path, suffix, content)`` list from
+ the orchestrator's single traversal; None walks the repo directly.
+ *index* maps ``rel_path -> ParsedFile`` from the repo's parse cache. A
+ file present there gets its Python routes from the parsed decorators
+ rather than from the FastAPI text regex — a decorator node cannot carry
+ a route from a comment, and cannot lose an empty path. *content_hashes*
+ is the walk's ``rel_path -> sha256`` map, which is what proves a cached
+ parse describes the file as it is now; without it the index is unused.
"""
- all_exts = _union_extensions(self.provider_dialects) | _union_extensions(
- self.consumer_dialects
+ own_hashes: dict[str, str] | None = None
+ if files is None and index is not None and content_hashes is None:
+ own_hashes = content_hashes = {}
+ scanned = select_files(
+ repo_path, self.source_extensions(), exclude, files, own_hashes
)
+ mounts = self._collect_mounts(scanned)
- files = list(iter_source_files(repo_path, all_exts, exclude))
- mounts = self._collect_mounts(files)
+ from ..from_index import extract_http_providers, parsed_for
contracts: list[Contract] = []
- for rel_path, suffix, content in files:
+ for rel_path, suffix, content in scanned:
ctx = ScanContext(repo_alias, rel_path, suffix, content, mounts)
+ # Only a parse that matches the bytes just walked supersedes the
+ # regex; a stale or missing entry leaves the dialect in charge.
+ parsed = (
+ parsed_for(index, rel_path, (content_hashes or {}).get(rel_path))
+ if suffix in PYTHON
+ else None
+ )
+ # Run the index pass first: it only supersedes the text dialect for
+ # a file it actually produced routes for. A parse that yielded no
+ # decorated symbols (a grammar the parser stumbled on, a route
+ # shape the queries do not capture) must not silently delete the
+ # routes the regex can still see in the file's text.
+ from_parse = extract_http_providers(ctx, parsed) if parsed else []
+ contracts.extend(from_parse)
for dialect in self.provider_dialects:
- if suffix in dialect.extensions:
- contracts.extend(dialect.extract(ctx))
+ if suffix not in dialect.extensions:
+ continue
+ if from_parse and dialect.name in _INDEX_BACKED_DIALECTS:
+ continue # superseded by the index pass above
+ contracts.extend(dialect.extract(ctx))
for dialect in self.consumer_dialects:
if suffix in dialect.extensions:
contracts.extend(dialect.extract(ctx))
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
index d4acb6255..3b37f6ad5 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
@@ -23,8 +23,12 @@ if TYPE_CHECKING:
# router prefix can be resolved; only known routers (those bound to APIRouter /
# FastAPI in-file, plus the conventional ``app`` / ``router`` names) are kept, so
# an unrelated ``@cache.get(...)`` decorator is not mistaken for a route.
+# The path is ``*`` not ``+``: ``@router.post("")`` on a prefixed router is the
+# idiomatic way to serve the collection root, and requiring one character made
+# those routes invisible entirely. The empty path contributes nothing on its own,
+# so ``build_provider_contract`` still drops it when no prefix stitches on.
_FASTAPI_RE = re.compile(
- rf"""@(\w+)\.({METHODS})\s*\(\s*['"]([^'"]+)['"]""",
+ rf"""@(\w+)\.({METHODS})\s*\(\s*['"]([^'"]*)['"]""",
re.IGNORECASE,
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py b/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py
index 48c7469e6..56b28e4c2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py
@@ -12,7 +12,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import TYPE_CHECKING
-from .base import iter_source_files
+from .base import select_files
from .http.paths import (
extract_path_from_url,
is_unusable_consumer_path,
@@ -22,10 +22,12 @@ from .http.paths import (
from .langs import CSHARP, PYTHON
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from repowise.core.workspace.contracts import Contract
+ from .base import SourceFile
+
_log = logging.getLogger("repowise.workspace.extractors.socket")
_EXTENSIONS = CSHARP | PYTHON
@@ -141,18 +143,26 @@ def _normalize_socket_identity(raw: str, prefix: str = "") -> str | None:
class SocketExtractor:
"""Extract socket/websocket contracts from source files."""
+ @classmethod
+ def source_extensions(cls) -> frozenset[str]:
+ """Every extension this extractor scans."""
+ return _EXTENSIONS
+
def extract(
self,
repo_path: Path,
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
+ files: Sequence[SourceFile] | None = None,
) -> list[Contract]:
from repowise.core.workspace.contracts import Contract
contracts: list[Contract] = []
seen: set[tuple[str, str, str]] = set()
- for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS, exclude):
+ for rel_path, _suffix, content in select_files(
+ repo_path, _EXTENSIONS, exclude, files
+ ):
for pdef in _CLIENT_PATTERNS + _PROVIDER_PATTERNS:
if pdef.context_regex is not None and not pdef.context_regex.search(content):
continue
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
index 7466036cd..a5882dd98 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
@@ -14,13 +14,15 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY
-from .base import iter_source_files
+from .base import select_files
if TYPE_CHECKING:
- from collections.abc import Callable
+ from collections.abc import Callable, Sequence
from repowise.core.workspace.contracts import Contract
+ from .base import SourceFile
+
_log = logging.getLogger("repowise.workspace.extractors.topic")
# ---------------------------------------------------------------------------
@@ -222,11 +224,17 @@ _ALL_PATTERNS = _KAFKA_PATTERNS + _RABBITMQ_PATTERNS + _NATS_PATTERNS
class TopicExtractor:
"""Extract message topic/queue contracts from source files."""
+ @classmethod
+ def source_extensions(cls) -> frozenset[str]:
+ """Every extension this extractor scans."""
+ return _EXTENSIONS
+
def extract(
self,
repo_path: Path,
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
+ files: Sequence[SourceFile] | None = None,
) -> list[Contract]:
from repowise.core.workspace.contracts import Contract
@@ -236,7 +244,9 @@ class TopicExtractor:
# File discovery is gitignore- and nested-repo-aware (see
# ``iter_source_files``) so a workspace repo whose root contains nested
# repos or large ignored trees is not scanned end-to-end.
- for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS, exclude):
+ for rel_path, _suffix, content in select_files(
+ repo_path, _EXTENSIONS, exclude, files
+ ):
for pdef in _ALL_PATTERNS:
for match in pdef.regex.finditer(content):
topic_name = match.group(pdef.topic_group).strip()
diff --git a/packages/core/src/repowise/core/workspace/system_graph.py b/packages/core/src/repowise/core/workspace/system_graph.py
index 4b339a6ea..2b5eeadf0 100644
--- a/packages/core/src/repowise/core/workspace/system_graph.py
+++ b/packages/core/src/repowise/core/workspace/system_graph.py
@@ -471,15 +471,21 @@ async def run_system_graph_build(
workspace_root: Path,
store: ContractStore,
overlay: CrossRepoOverlay,
+ boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,
) -> SystemGraph:
"""Build and persist the system graph from the latest contracts + overlay.
Called from ``run_cross_repo_hooks`` after contract extraction and
cross-repo analysis. Boundary detection (a filesystem walk) runs off-thread.
+
+ *boundaries_by_repo* is the map ``run_cross_repo_hooks`` already computed
+ for contract extraction. Both consumers need the same answer, so passing it
+ in halves the boundary walks per update; None re-detects for direct callers.
"""
- boundaries_by_repo = await asyncio.to_thread(
- _detect_boundaries_by_repo, ws_config, workspace_root
- )
+ if boundaries_by_repo is None:
+ boundaries_by_repo = await asyncio.to_thread(
+ _detect_boundaries_by_repo, ws_config, workspace_root
+ )
diagnostics = build_diagnostics(store.contracts, store.contract_links)
graph = build_system_graph(
diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py
index b7394bd8f..a3376f563 100644
--- a/packages/core/src/repowise/core/workspace/update.py
+++ b/packages/core/src/repowise/core/workspace/update.py
@@ -16,7 +16,10 @@ from contextlib import suppress
from dataclasses import dataclass
from datetime import UTC, datetime
from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from repowise.core.workspace.extractors.service_boundary import ServiceBoundary
# Per-repo update lock — the shared single-flight guard (one implementation
# for the CLI update command and this workspace updater, which used to carry
@@ -904,7 +907,22 @@ async def run_cross_repo_hooks(
from .conformance import run_conformance_check
from .contracts import ContractStore, load_contract_store, run_contract_extraction
from .cross_repo import CrossRepoOverlay, run_cross_repo_analysis
- from .system_graph import SystemGraph, run_system_graph_build
+ from .system_graph import (
+ SystemGraph,
+ _detect_boundaries_by_repo,
+ run_system_graph_build,
+ )
+
+ # Service boundaries, detected once for the whole workspace. Contract
+ # extraction and the system-graph build both need them and each used to walk
+ # every repo for its own copy.
+ boundaries_by_repo: dict[str, list[ServiceBoundary]] = {}
+ try:
+ boundaries_by_repo = await asyncio.to_thread(
+ _detect_boundaries_by_repo, ws_config, workspace_root
+ )
+ except Exception:
+ _log.warning("Service boundary detection failed", exc_info=True)
overlay = CrossRepoOverlay()
try:
@@ -920,7 +938,9 @@ async def run_cross_repo_hooks(
# Contract extraction (overwrites contracts.json).
store = ContractStore()
try:
- store = await run_contract_extraction(ws_config, workspace_root, changed_repos)
+ store = await run_contract_extraction(
+ ws_config, workspace_root, changed_repos, boundaries_by_repo or None
+ )
except Exception:
_log.warning("Contract extraction failed", exc_info=True)
@@ -928,7 +948,9 @@ async def run_cross_repo_hooks(
# view reads. Built last so it folds in the contracts and overlay above.
system_graph: SystemGraph | None = None
try:
- system_graph = await run_system_graph_build(ws_config, workspace_root, store, overlay)
+ system_graph = await run_system_graph_build(
+ ws_config, workspace_root, store, overlay, boundaries_by_repo or None
+ )
except Exception:
_log.warning("System graph build failed", exc_info=True)
diff --git a/tests/unit/workspace/test_dialect_regressions.py b/tests/unit/workspace/test_dialect_regressions.py
new file mode 100644
index 000000000..f71461276
--- /dev/null
+++ b/tests/unit/workspace/test_dialect_regressions.py
@@ -0,0 +1,249 @@
+"""Per-dialect regression locks for contracts that were fabricated or lost.
+
+The dialects under ``workspace/extractors/`` had no direct tests: coverage was
+indirect, through ``test_contracts.py`` and ``test_extractor_traversal.py``
+exercising the top-level extractors. Four defects reached the live workspace
+artifact through that gap, and each one gets a test here that fails against the
+pre-fix code:
+
+1. ``@router.post("")`` on a prefixed router was invisible — the decorator
+ regex required at least one path character. It cost 2 of the 3 real SSE
+ endpoints in the hosted backend.
+2. The extractors' own sources were scanned, so the example syntax in their
+ docstrings became 14 live contracts (``topic::orders``, ``http::GET::/path``,
+ ``grpc::AuthService/*``).
+3. ``WITH ranked AS (...)`` reported a table named ``ranked`` — sqlglot models a
+ CTE reference as a table.
+4. A docstring reading "...update path..." / "...update would..." satisfied the
+ SQL verb gate and became the tables ``path`` and ``would``.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.workspace.extractors.base import (
+ ScanContext,
+ make_exclude_predicate,
+)
+from repowise.core.workspace.extractors.data import DataExtractor
+from repowise.core.workspace.extractors.data.sql_strings import SqlStringsDialect
+from repowise.core.workspace.extractors.http.fastapi import FastApiDialect
+
+
+def _ctx(content: str, rel_path: str = "app/routers/chat.py") -> ScanContext:
+ return ScanContext(
+ repo_alias="backend", rel_path=rel_path, suffix=".py", content=content
+ )
+
+
+def _ids(contracts) -> set[str]:
+ return {c.contract_id for c in contracts}
+
+
+class TestFastApiEmptyPath:
+ """Regression 1: ``@router.post("")`` mounted at a router prefix."""
+
+ # Mirrors backend/app/routers/workspace_chat.py:34,162 and chat.py:49,352.
+ SOURCE = '''
+router = APIRouter(prefix="/snapshots/{snapshot_id}/chat", tags=["chat"])
+
+
+async def chat_message(request: Request) -> None:
+ ...
+
+
[email protected]("/conversations")
+async def list_conversations() -> None:
+ ...
+'''
+
+ def test_empty_path_route_is_extracted_at_the_router_prefix(self) -> None:
+ ids = _ids(FastApiDialect().extract(_ctx(self.SOURCE)))
+ assert "http::POST::/snapshots/{param}/chat" in ids
+
+ def test_sibling_routes_still_extracted(self) -> None:
+ ids = _ids(FastApiDialect().extract(_ctx(self.SOURCE)))
+ assert "http::GET::/snapshots/{param}/chat/conversations" in ids
+
+ def test_empty_path_without_a_prefix_contributes_nothing(self) -> None:
+ # No prefix to stitch onto, so there is no path to record. Dropped by
+ # build_provider_contract rather than silently becoming bare "/".
+ src = '''
+app = FastAPI()
+
+
+async def nothing() -> None:
+ ...
+'''
+ assert FastApiDialect().extract(_ctx(src)) == []
+
+ def test_non_router_decorator_is_not_a_route(self) -> None:
+ # The empty-path relaxation must not turn every @x.get() into a route.
+ src = '''
+router = APIRouter(prefix="/api")
+
+
[email protected]("/not-a-route")
+def cached() -> None:
+ ...
+'''
+ assert FastApiDialect().extract(_ctx(src)) == []
+
+
+class TestExtractorSelfExclusion:
+ """Regression 2: the extractors' own example syntax became live contracts."""
+
+ SELF_PATHS = (
+ "packages/core/src/repowise/core/workspace/extractors/http/fastapi.py",
+ "packages/core/src/repowise/core/workspace/extractors/http/mounts.py",
+ "packages/core/src/repowise/core/workspace/extractors/topic_extractor.py",
+ "packages/core/src/repowise/core/workspace/extractors/grpc/python.py",
+ "packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py",
+ )
+
+ def test_extractor_sources_are_skipped(self) -> None:
+ skip = make_exclude_predicate()
+ for path in self.SELF_PATHS:
+ assert skip(path), path
+
+ def test_the_rest_of_the_workspace_package_is_still_scanned(self) -> None:
+ # The exclusion is the extractors tree only, not workspace/ at large.
+ skip = make_exclude_predicate()
+ assert not skip("packages/core/src/repowise/core/workspace/contracts.py")
+
+ def test_ordinary_source_is_unaffected(self) -> None:
+ skip = make_exclude_predicate()
+ assert not skip("app/routers/chat.py")
+ assert not skip("src/lib/api/client.ts")
+
+ def test_user_supplied_globs_still_apply(self) -> None:
+ skip = make_exclude_predicate(("vendor/*",))
+ assert skip("vendor/thing.py")
+ assert not skip("app/thing.py")
+
+
+class TestSqlCteAlias:
+ """Regression 3: ``WITH ranked AS (...)`` emitted a table named ``ranked``."""
+
+ # Reduced from packages/core/src/repowise/core/persistence/crud/git.py:258.
+ LITERAL = """
+WITH ranked AS (
+ SELECT id, PERCENT_RANK() OVER (ORDER BY commit_count_90d) AS prank
+ FROM git_metadata
+ WHERE repository_id = :repo_id
+)
+UPDATE git_metadata
+SET churn_percentile = (SELECT prank FROM ranked WHERE ranked.id = git_metadata.id)
+"""
+
+ def test_cte_alias_is_not_a_table(self) -> None:
+ tables = {t for t, _verb in SqlStringsDialect()._tables_in(self.LITERAL)}
+ assert "ranked" not in tables
+
+ def test_the_real_table_is_still_found(self) -> None:
+ tables = {t for t, _verb in SqlStringsDialect()._tables_in(self.LITERAL)}
+ assert "git_metadata" in tables
+
+ def test_a_real_table_sharing_a_cte_name_elsewhere_is_unaffected(self) -> None:
+ # Shadowing is per-statement: no CTE here, so ``ranked`` is a table.
+ tables = {
+ t for t, _verb in SqlStringsDialect()._tables_in("SELECT id FROM ranked")
+ }
+ assert "ranked" in tables
+
+
+class TestSqlProseRejection:
+ """Regression 4: a docstring became the tables ``path`` and ``would``."""
+
+ # Reduced from the batch_upsert_symbols docstring in
+ # packages/core/src/repowise/core/persistence/crud/external_systems.py.
+ DOCSTRING_SOURCE = '''
+def batch_upsert_symbols(file_paths, symbols):
+ """Make ``wiki_symbols`` for *file_paths* match a fresh parse (*symbols*).
+
+ The incremental update path re-parses changed files but historically never
+ persisted their symbols. Calling the repo-wide helper per update would
+ SELECT the whole table. This prunes rows for symbols that vanished from a
+ still-existing file.
+ """
+'''
+
+ def test_docstring_prose_yields_no_tables(self) -> None:
+ ctx = _ctx(self.DOCSTRING_SOURCE, rel_path="crud/external_systems.py")
+ assert SqlStringsDialect().extract(ctx) == []
+
+ def test_the_leaked_words_are_gone(self) -> None:
+ ctx = _ctx(self.DOCSTRING_SOURCE, rel_path="crud/external_systems.py")
+ ids = _ids(SqlStringsDialect().extract(ctx))
+ assert "data::path" not in ids
+ assert "data::would" not in ids
+
+ def test_real_sql_in_the_same_file_is_still_extracted(self) -> None:
+ src = self.DOCSTRING_SOURCE + '''
+ rows = conn.execute("SELECT id, name FROM wiki_symbols WHERE repo_id = :r")
+'''
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "crud/external_systems.py")))
+ assert "data::wiki_symbols" in ids
+
+ def test_named_query_comment_still_opens_the_gate(self) -> None:
+ # aiosql / yesql / sqlc / HugSQL all prefix a query with a name comment.
+ src = '''
+q = """
+-- name: get_user
+SELECT id FROM accounts WHERE id = :id
+"""
+'''
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))
+ assert "data::accounts" in ids
+
+ def test_block_comment_prefix_still_opens_the_gate(self) -> None:
+ src = 'q = "/* cached */ SELECT id FROM accounts WHERE id = 1"\n'
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))
+ assert "data::accounts" in ids
+
+ def test_create_table_as_select_records_the_table_it_reads(self) -> None:
+ src = 'q = "CREATE TABLE snapshot AS SELECT id FROM accounts WHERE x = 1"\n'
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))
+ assert "data::accounts" in ids
+
+ def test_explain_still_opens_the_gate(self) -> None:
+ src = 'q = "EXPLAIN ANALYZE SELECT id FROM accounts WHERE id = 1"\n'
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))
+ assert "data::accounts" in ids
+
+ def test_prose_beginning_with_a_dash_is_still_rejected(self) -> None:
+ # The comment strip must not become a way back in for prose.
+ src = '''
+def f():
+ """- The incremental update path would SELECT rows from a file."""
+'''
+ assert SqlStringsDialect().extract(_ctx(src, "f.py")) == []
+
+ def test_indented_sql_literal_still_opens_the_gate(self) -> None:
+ # Multi-line SQL literals usually start with a newline and indentation.
+ src = '''
+q = """
+ SELECT id FROM projects WHERE owner = :owner
+"""
+'''
+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))
+ assert "data::projects" in ids
+
+
+class TestDataExtractorEndToEnd:
+ """The two SQL fixes hold through the DataExtractor, not just the dialect."""
+
+ def test_prose_and_cte_do_not_reach_contracts(self, tmp_path: Path) -> None:
+ (tmp_path / "git.py").write_text(
+ f'SQL = """{TestSqlCteAlias.LITERAL}"""\n', encoding="utf-8"
+ )
+ (tmp_path / "external_systems.py").write_text(
+ TestSqlProseRejection.DOCSTRING_SOURCE, encoding="utf-8"
+ )
+ ids = _ids(DataExtractor().extract(tmp_path, "repowise", make_exclude_predicate()))
+ assert "data::git_metadata" in ids
+ for fabricated in ("data::ranked", "data::path", "data::would"):
+ assert fabricated not in ids
diff --git a/tests/unit/workspace/test_extraction_walk_budget.py b/tests/unit/workspace/test_extraction_walk_budget.py
new file mode 100644
index 000000000..485c0e89c
--- /dev/null
+++ b/tests/unit/workspace/test_extraction_walk_budget.py
@@ -0,0 +1,231 @@
+"""Traversal budget: one tree walk and one boundary detection per repo.
+
+Five extractors each walked the repo and re-read every file whose extension it
+claimed, and boundary detection then ran a sixth walk — per repo, per update.
+Wall time is machine-dependent and makes a poor gate, but walk and read counts
+are exact, so they are what these tests assert.
+
+Measured on the three-repo workspace this change was developed against
+(``repowise`` + ``backend`` + ``frontend``), for ``run_contract_extraction``:
+
+=========================== ======== =======
+metric before after
+=========================== ======== =======
+FileTraverser walks / repo 5 1
+file reads (all repos) 13,889 3,216
+wall time 45.23s 15.46s
+contracts / links 714 / 101 714 / 101
+=========================== ======== =======
+
+Boundary detection keeps its own walk through ``fs_walk.walk_repo``, so the
+per-repo total is 2 rather than 1; what changed there is that it runs once per
+update instead of once for contract extraction and again for the system graph.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from repowise.core.ingestion.traverser import FileTraverser
+from repowise.core.workspace import contracts as contracts_mod
+from repowise.core.workspace import system_graph as system_graph_mod
+from repowise.core.workspace.config import ContractConfig, RepoEntry, WorkspaceConfig
+from repowise.core.workspace.contracts import run_contract_extraction
+
+REPO_SOURCE = {
+ "svc_a/api.py": (
+ 'router = APIRouter(prefix="/api")\n\n\[email protected]("/users")\ndef users(): ...\n'
+ ),
+ "svc_a/db.py": 'Q = "SELECT id FROM users WHERE id = :id"\n',
+ "svc_a/pyproject.toml": '[project]\nname = "svc-a"\n',
+ "svc_b/client.ts": "await fetch('/api/users');\n",
+ "svc_b/package.json": '{"name": "svc-b"}\n',
+ "svc_b/queue.js": "channel.sendToQueue('jobs', payload);\n",
+}
+
+
+def _make_repo(root: Path) -> None:
+ for rel, content in REPO_SOURCE.items():
+ path = root / rel
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content, encoding="utf-8")
+ (root / ".repowise").mkdir(exist_ok=True)
+
+
+def workspace(tmp_path: Path) -> WorkspaceConfig:
+ for alias in ("alpha", "beta"):
+ _make_repo(tmp_path / alias)
+ return WorkspaceConfig(
+ repos=[RepoEntry(path="alpha", alias="alpha"), RepoEntry(path="beta", alias="beta")],
+ contracts=ContractConfig(),
+ )
+
+
+def counters(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:
+ """Count tree walks, file reads and boundary detections."""
+ counts = {"walks": 0, "reads": 0, "boundaries": 0}
+
+ original_traverse = FileTraverser.traverse
+
+ def counted_traverse(self, *args, **kwargs):
+ counts["walks"] += 1
+ return original_traverse(self, *args, **kwargs)
+
+ # Both, because the shared walk reads bytes (so it can hash them) while
+ # other call sites still read text. Counting only one would let a second
+ # read slip in through the other.
+ original_read_text = Path.read_text
+ original_read_bytes = Path.read_bytes
+
+ def counted_read_text(self, *args, **kwargs):
+ counts["reads"] += 1
+ return original_read_text(self, *args, **kwargs)
+
+ def counted_read_bytes(self, *args, **kwargs):
+ counts["reads"] += 1
+ return original_read_bytes(self, *args, **kwargs)
+
+ monkeypatch.setattr(FileTraverser, "traverse", counted_traverse)
+ monkeypatch.setattr(Path, "read_text", counted_read_text)
+ monkeypatch.setattr(Path, "read_bytes", counted_read_bytes)
+
+ from repowise.core.workspace import extractors as extractors_pkg
+ from repowise.core.workspace.extractors import service_boundary
+
+ original_detect = service_boundary.detect_service_boundaries
+
+ def counted_detect(repo_path, *args, **kwargs):
+ counts["boundaries"] += 1
+ return original_detect(repo_path, *args, **kwargs)
+
+ for module in (service_boundary, extractors_pkg, system_graph_mod):
+ monkeypatch.setattr(module, "detect_service_boundaries", counted_detect)
+ return counts
+
+
+async def test_one_tree_walk_per_repo(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ counters: dict[str, int],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ await run_contract_extraction(workspace, tmp_path, [])
+ assert counters["walks"] == 2 # one per repo, not one per extractor per repo
+
+
+async def test_each_file_is_read_once_per_repo(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ counters: dict[str, int],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ await run_contract_extraction(workspace, tmp_path, [])
+ # 4 source files per repo are in the extractors' combined extension set
+ # (api.py, db.py, client.ts, queue.js); the manifests are not source.
+ assert counters["reads"] <= 2 * len(REPO_SOURCE)
+
+
+async def test_boundaries_detected_once_per_repo_when_supplied(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ counters: dict[str, int],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ boundaries = system_graph_mod._detect_boundaries_by_repo(workspace, tmp_path)
+ detections_for_the_shared_map = counters["boundaries"]
+
+ await run_contract_extraction(workspace, tmp_path, [], boundaries)
+
+ # Extraction reuses the map instead of re-detecting per repo.
+ assert counters["boundaries"] == detections_for_the_shared_map
+
+
+async def test_boundaries_are_detected_when_not_supplied(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ counters: dict[str, int],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ await run_contract_extraction(workspace, tmp_path, [])
+ assert counters["boundaries"] == 2 # direct callers still get boundaries
+
+
+async def test_hooks_detect_boundaries_once_per_repo(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ counters: dict[str, int],
+) -> None:
+ """The 2-per-repo -> 1-per-repo budget, through the caller that wires it.
+
+ ``run_cross_repo_hooks`` is where contract extraction and the system-graph
+ build meet; each used to detect boundaries for every repo independently.
+ """
+ from repowise.core.workspace.update import run_cross_repo_hooks
+
+ await run_cross_repo_hooks(workspace, tmp_path, [])
+
+ assert counters["boundaries"] == 2 # two repos, once each — not 4
+
+
+async def test_the_shared_walk_finds_the_same_contracts(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ store = await run_contract_extraction(workspace, tmp_path, [])
+ ids = {c.contract_id for c in store.contracts}
+ assert "http::GET::/api/users" in ids # provider, prefix stitched
+ assert "data::users" in ids # SQL consumer
+ assert "topic::jobs" in ids # queue provider
+
+
+async def test_every_contract_records_its_extraction_layer(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ from repowise.core.workspace.extractors.from_index import (
+ EXTRACTION_LAYER_KEY,
+ LAYER_REGEX,
+ )
+
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ store = await run_contract_extraction(workspace, tmp_path, [])
+ assert store.contracts
+ # No parse cache in these fixture repos, so everything is regex-sourced.
+ assert {c.meta.get(EXTRACTION_LAYER_KEY) for c in store.contracts} == {LAYER_REGEX}
+
+
+async def test_grpc_dialect_provenance_survives_the_layer_stamp(
+ workspace: WorkspaceConfig,
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A gRPC contract keeps its own ``meta["source"]`` and gains a layer."""
+ from repowise.core.workspace.extractors.from_index import (
+ EXTRACTION_LAYER_KEY,
+ LAYER_REGEX,
+ )
+
+ (tmp_path / "alpha" / "svc_a" / "auth.proto").write_text(
+ 'syntax = "proto3";\n\nservice AuthService {\n'
+ " rpc Login (LoginRequest) returns (LoginReply);\n}\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
+ store = await run_contract_extraction(workspace, tmp_path, [])
+
+ grpc = [c for c in store.contracts if c.contract_type == "grpc"]
+ assert grpc, "fixture should produce a gRPC contract"
+ for contract in grpc:
+ assert contract.meta["source"] == "proto" # dialect provenance intact
+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_REGEX
diff --git a/tests/unit/workspace/test_from_index.py b/tests/unit/workspace/test_from_index.py
new file mode 100644
index 000000000..476439b9f
--- /dev/null
+++ b/tests/unit/workspace/test_from_index.py
@@ -0,0 +1,325 @@
+"""Index-backed contract extraction: routes read from parsed decorators.
+
+Two claims are under test here. The first is that reading the decorator a
+parser already recorded removes a class of defect the regex could only chase:
+a route-shaped string in a comment is not a decorator, and an empty path is
+just a string argument, so neither can fabricate or lose a contract.
+
+The second is the inherited-framework claim. Ingestion resolves Flask as well
+as FastAPI, while the workspace regex layer was FastAPI-only. Flask arriving
+here without a new dialect module is the evidence that the rest of the
+framework list comes with the layer rather than being hand-written per
+framework.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from repowise.core.ingestion.models import FileInfo, ParsedFile, Symbol
+from repowise.core.workspace.extractors.base import ScanContext
+from repowise.core.workspace.extractors.from_index import (
+ EXTRACTION_LAYER_KEY,
+ LAYER_INDEX,
+ extract_http_providers,
+ load_repo_index,
+)
+
+
+def _symbol(name: str, decorators: list[str], line: int = 10) -> Symbol:
+ return Symbol(
+ id=f"m.py::{name}",
+ name=name,
+ qualified_name=name,
+ kind="function",
+ signature=f"def {name}()",
+ start_line=line,
+ end_line=line + 3,
+ docstring=None,
+ decorators=decorators,
+ language="python",
+ )
+
+
+def _parsed(symbols: list[Symbol], rel_path: str = "app/routers/chat.py") -> ParsedFile:
+ info = FileInfo(
+ path=rel_path,
+ abs_path=f"/repo/{rel_path}",
+ language="python",
+ size_bytes=100,
+ git_hash="",
+ last_modified=0.0,
+ is_test=False,
+ is_config=False,
+ is_api_contract=True,
+ is_entry_point=False,
+ )
+ return ParsedFile(file_info=info, symbols=symbols, imports=[], exports=[])
+
+
+def _ctx(content: str, rel_path: str = "app/routers/chat.py") -> ScanContext:
+ return ScanContext(
+ repo_alias="backend", rel_path=rel_path, suffix=".py", content=content
+ )
+
+
+def _ids(contracts) -> set[str]:
+ return {c.contract_id for c in contracts}
+
+
+class TestFastApiFromDecorators:
+ CONTENT = 'router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")\n'
+
+ def test_empty_path_route_resolves_to_the_prefix(self) -> None:
+ parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])
+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ assert ids == {"http::POST::/snapshots/{param}/chat"}
+
+ def test_provenance_is_recorded(self) -> None:
+ parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])
+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX
+ assert contract.meta["handler"] == "chat_message"
+
+ def test_layer_key_does_not_collide_with_grpc_dialect_provenance(self) -> None:
+ # The gRPC dialects already own meta["source"] for dialect provenance
+ # ("py_servicer", "go_client", "proto"); the layer must not overwrite
+ # it or the coverage metric mis-partitions every gRPC contract.
+ assert EXTRACTION_LAYER_KEY != "source"
+ parsed = _parsed([_symbol("chat", ['@router.post("")'])])
+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ assert "source" not in contract.meta
+
+ def test_duplicated_decorator_entries_yield_one_contract(self) -> None:
+ # The parser can append the same decorator twice for one symbol.
+ parsed = _parsed(
+ [_symbol("chat", ['@router.post("")', '@router.post("")'])]
+ )
+ assert len(extract_http_providers(_ctx(self.CONTENT), parsed)) == 1
+
+ def test_non_route_decorators_are_ignored(self) -> None:
+ parsed = _parsed(
+ [
+ _symbol(
+ "chat",
+ ['@limiter.limit("240/minute")', "@dataclass", "@classmethod"],
+ )
+ ]
+ )
+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+
+ def test_unknown_router_variable_is_not_a_route(self) -> None:
+ parsed = _parsed([_symbol("cached", ['@cache.get("/x")'])])
+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+
+
+class TestCommentsCannotFabricateRoutes:
+ """The self-extraction defect, structurally: a comment is not a decorator."""
+
+ CONTENT = '''
+router = APIRouter(prefix="/api")
+
+# Recognises @app.get('/path') and @router.post('/orders') declarations.
+EXAMPLE = "@router.delete('/fabricated')"
+
+
+def documented() -> None:
+ """Handles @router.put("/from-a-docstring") style routes."""
+'''
+
+ def test_route_shaped_text_outside_a_decorator_yields_nothing(self) -> None:
+ # No symbol carries a route decorator, so there is no route — however
+ # much route-shaped prose the file contains.
+ parsed = _parsed([_symbol("documented", [])], rel_path="extractors/http.py")
+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+
+ def test_a_real_decorator_in_the_same_file_still_counts(self) -> None:
+ parsed = _parsed(
+ [_symbol("real", ['@router.get("/real")'])], rel_path="extractors/http.py"
+ )
+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ assert ids == {"http::GET::/api/real"}
+
+
+class TestFlaskIsInherited:
+ """Flask providers via the index path — no Flask dialect module exists."""
+
+ CONTENT = 'app = Flask(__name__)\nbp = Blueprint("api", __name__)\n'
+
+ def test_route_decorator_defaults_to_get(self) -> None:
+ parsed = _parsed([_symbol("index", ['@app.route("/health")'])])
+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ assert ids == {"http::GET::/health"}
+
+ def test_methods_kwarg_yields_one_contract_per_verb(self) -> None:
+ parsed = _parsed(
+ [_symbol("users", ['@app.route("/users", methods=["POST", "PUT"])'])]
+ )
+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ assert ids == {"http::POST::/users", "http::PUT::/users"}
+
+ def test_framework_is_labelled_flask(self) -> None:
+ parsed = _parsed([_symbol("index", ['@app.route("/health")'])])
+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ assert contract.meta["framework"] == "flask"
+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX
+
+ def test_blueprint_url_prefix_is_stitched_on(self) -> None:
+ # Flask spells it ``url_prefix=``, and the shared prefix pattern is not
+ # left-anchored to a word boundary, so it matches that too and the
+ # blueprint's mount lands on the route. Incidental, but correct — this
+ # test pins the behaviour so a later tightening of the pattern has to
+ # keep Flask working deliberately.
+ content = 'bp = Blueprint("api", __name__, url_prefix="/api")\n'
+ parsed = _parsed([_symbol("listing", ['@bp.route("/items")'])])
+ ids = _ids(extract_http_providers(_ctx(content), parsed))
+ assert ids == {"http::GET::/api/items"}
+
+ def test_the_regex_layer_has_no_flask_dialect(self) -> None:
+ # Guards the claim in this class's docstring: if a Flask dialect is
+ # added later, this test should be deleted along with the claim.
+ from repowise.core.workspace.extractors.http import PROVIDER_DIALECTS
+
+ assert "flask" not in {d.name for d in PROVIDER_DIALECTS}
+
+
+class TestLoadRepoIndex:
+ def test_missing_cache_returns_none(self, tmp_path) -> None:
+ assert load_repo_index(tmp_path) is None
+
+ def test_unreadable_cache_returns_none(self, tmp_path) -> None:
+ # A repo indexed by another repowise version fails the parser
+ # fingerprint gate and must degrade to the regex path, not raise.
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / ".repowise" / "parse_cache.pkl").write_bytes(b"not a pickle")
+ assert load_repo_index(tmp_path) is None
+
+
+ROUTES_PY = '''\
+from fastapi import APIRouter
+
+router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")
+
+
+# Documents the @router.get("/fabricated") shape this module handles.
+EXAMPLE = "@router.delete('/also-fabricated')"
+
+
+async def chat_message() -> None:
+ """Sends a message. See @router.put("/not-a-route") for the old shape."""
+
+
[email protected]("/conversations")
+async def list_conversations() -> None:
+ ...
+'''
+
+
+def _write_real_parse_cache(repo: Path) -> None:
+ """Parse *repo*'s Python with the real ingestion parser and cache it.
+
+ Uses ``ParseCache`` itself rather than a hand-built pickle, so the entry
+ key, the ``parser_fingerprint`` gate and the content hash are all produced
+ the way ingestion produces them.
+ """
+ from repowise.core.ingestion.models import compute_content_hash
+ from repowise.core.ingestion.parse_cache import ParseCache
+ from repowise.core.ingestion.parser import parse_file
+ from repowise.core.ingestion.traverser import FileTraverser
+
+ cache = ParseCache(repo / ".repowise")
+ for info in FileTraverser(repo).traverse():
+ if not info.path.endswith(".py"):
+ continue
+ source = Path(info.abs_path).read_bytes()
+ cache.put(parse_file(info, source), compute_content_hash(source))
+ cache.save()
+
+
+class TestAgainstARealParseCache:
+ """The plumbing, end to end: real parser, real cache, real hash check.
+
+ Everything else here builds a ``ParsedFile`` by hand, which cannot catch
+ the two agreements this path depends on: that the walk hashes bytes the
+ same way ingestion does, and that a cached entry survives the round trip.
+ """
+
+ @pytest.fixture
+ def repo(self, tmp_path: Path) -> Path:
+ (tmp_path / ".repowise").mkdir()
+ (tmp_path / "routers").mkdir()
+ (tmp_path / "routers" / "chat.py").write_text(ROUTES_PY, encoding="utf-8")
+ _write_real_parse_cache(tmp_path)
+ return tmp_path
+
+ def test_the_cache_round_trips(self, repo: Path) -> None:
+ index = load_repo_index(repo)
+ assert index is not None
+ assert "routers/chat.py" in index
+
+ def test_routes_come_from_the_index(self, repo: Path) -> None:
+ from repowise.core.workspace.extractors import HttpExtractor
+ from repowise.core.workspace.extractors.base import make_exclude_predicate
+
+ index = load_repo_index(repo)
+ contracts = HttpExtractor().extract(
+ repo, "backend", make_exclude_predicate(), None, index=index
+ )
+ providers = [c for c in contracts if c.role == "provider"]
+ assert providers, "index path produced nothing"
+ assert {c.meta[EXTRACTION_LAYER_KEY] for c in providers} == {LAYER_INDEX}
+
+ def test_the_empty_path_route_survives_the_whole_path(self, repo: Path) -> None:
+ from repowise.core.workspace.extractors import HttpExtractor
+ from repowise.core.workspace.extractors.base import make_exclude_predicate
+
+ index = load_repo_index(repo)
+ ids = _ids(
+ HttpExtractor().extract(
+ repo, "backend", make_exclude_predicate(), None, index=index
+ )
+ )
+ assert "http::POST::/snapshots/{param}/chat" in ids
+
+ def test_routes_in_comments_and_docstrings_are_not_extracted(
+ self, repo: Path
+ ) -> None:
+ from repowise.core.workspace.extractors import HttpExtractor
+ from repowise.core.workspace.extractors.base import make_exclude_predicate
+
+ index = load_repo_index(repo)
+ ids = _ids(
+ HttpExtractor().extract(
+ repo, "backend", make_exclude_predicate(), None, index=index
+ )
+ )
+ for fabricated in (
+ "http::GET::/snapshots/{param}/chat/fabricated",
+ "http::DELETE::/snapshots/{param}/chat/also-fabricated",
+ "http::PUT::/snapshots/{param}/chat/not-a-route",
+ ):
+ assert fabricated not in ids
+
+ def test_an_edited_file_falls_back_to_the_regex(self, repo: Path) -> None:
+ from repowise.core.workspace.extractors import HttpExtractor
+ from repowise.core.workspace.extractors.base import make_exclude_predicate
+ from repowise.core.workspace.extractors.from_index import parsed_for
+
+ index = load_repo_index(repo)
+ # Edit the file after it was cached: the entry now describes older
+ # bytes, so it must be rejected rather than trusted.
+ (repo / "routers" / "chat.py").write_text(
+ ROUTES_PY + '\n\[email protected]("/added")\nasync def added() -> None: ...\n',
+ encoding="utf-8",
+ )
+ contracts = HttpExtractor().extract(
+ repo, "backend", make_exclude_predicate(), None, index=index
+ )
+ layers = {c.meta.get(EXTRACTION_LAYER_KEY) for c in contracts if c.role == "provider"}
+ assert LAYER_INDEX not in layers # stale entry refused
+ assert parsed_for(index, "routers/chat.py", "deadbeef") is None
+ # And the newly added route is still found, by the regex path.
+ assert "http::GET::/snapshots/{param}/chat/added" in _ids(contracts)