Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.
Python aicode-complexitycode-healthcode-intelligencecode-qualitydead-codedeveloper-toolsdocumentationgit-analyticsmcpopen-sourcepythonrefactoringstatic-analysistechnical-debt
feat(workspace): read contracts from the parsed index, and walk each repo once (#1581)
Raghav Chamadiya2cf2b86436bf
diff --git a/packages/core/src/repowise/core/workspace/contracts.py b/packages/core/src/repowise/core/workspace/contracts.pyindex e9830d81a..e088b7741 100644--- a/packages/core/src/repowise/core/workspace/contracts.py+++ b/packages/core/src/repowise/core/workspace/contracts.py@@ -17,7 +17,7 @@ from collections import defaultdictfrom dataclasses import asdict, dataclass, fieldfrom datetime import UTC, datetimefrom pathlib import Path-from typing import Any+from typing import TYPE_CHECKING, Anyfrom repowise.core.workspace.config import (WORKSPACE_DATA_DIR,@@ -26,6 +26,9 @@ from repowise.core.workspace.config import ()from repowise.core.workspace.contract_schema import ContractSchema+if TYPE_CHECKING:+ from repowise.core.workspace.extractors.service_boundary import ServiceBoundary+_log = logging.getLogger("repowise.workspace.contracts")# ---------------------------------------------------------------------------@@ -674,17 +677,22 @@ async def run_contract_extraction(ws_config: WorkspaceConfig,workspace_root: Path,changed_repos: list[str],+ boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,) -> ContractStore:"""Full contract extraction pipeline.Called from :func:`run_cross_repo_hooks` during ``repowise update --workspace``.- 1. For each repo: scan files with each extractor (via ``to_thread``)- 2. Detect service boundaries per repo- 3. Assign service to each contract- 4. Run matching engine- 5. Merge manual links from ``WorkspaceConfig``- 6. Save ``contracts.json``+ 1. For each repo: walk once, then run every extractor over that file list+ 2. Assign service to each contract from the repo's boundaries+ 3. Run matching engine+ 4. Merge manual links from ``WorkspaceConfig``+ 5. Save ``contracts.json``++ *boundaries_by_repo* is the workspace-wide boundary map computed once by+ :func:`run_cross_repo_hooks` and shared with the system-graph build, which+ needs the same answer. When None (a direct call, or a test), boundaries are+ detected here instead."""from .extractors import (DataExtractor,@@ -695,7 +703,12 @@ async def run_contract_extraction(assign_service,detect_service_boundaries,)- from .extractors.base import make_exclude_predicate+ from .extractors.base import iter_source_files, make_exclude_predicate+ from .extractors.from_index import (+ EXTRACTION_LAYER_KEY,+ LAYER_REGEX,+ load_repo_index,+ )contract_config = ws_config.contractsexclude = make_exclude_predicate(tuple(contract_config.exclude_globs))@@ -716,8 +729,10 @@ async def run_contract_extraction(async def _extract_one_repo(alias: str, repo_path: Path) -> list[Contract]:contracts: list[Contract] = []- # Service boundary detection- boundaries = await asyncio.to_thread(detect_service_boundaries, repo_path)+ if boundaries_by_repo is not None:+ boundaries = boundaries_by_repo.get(alias, [])+ else:+ boundaries = await asyncio.to_thread(detect_service_boundaries, repo_path)# Run enabled extractorsextractors = []@@ -731,11 +746,45 @@ async def run_contract_extraction(extractors.append(TopicExtractor())if contract_config.detect_data:extractors.append(DataExtractor())+ if not extractors:+ return contracts++ # One walk per repo, shared by every extractor. Each used to walk and+ # re-read the tree itself, so a file claimed by N extractors was read N+ # times; the union of their extensions is walked once here instead.+ wanted: frozenset[str] = frozenset()+ for extractor in extractors:+ wanted |= extractor.source_extensions()+ # The walk records each file's content hash as it reads it, which is+ # what lets the index path tell a matching parse from a stale one+ # without a second read.+ content_hashes: dict[str, str] = {}+ files = await asyncio.to_thread(+ lambda: list(iter_source_files(repo_path, wanted, exclude, content_hashes))+ )++ # Symbols ingestion already parsed for this repo, when its parse cache+ # is usable. None means the regex dialects are the only path, which is+ # also the answer for languages with no AST tier. Only the HTTP+ # extractor consults it, so nothing is loaded when it is disabled.+ index = None+ if contract_config.detect_http:+ index = await asyncio.to_thread(load_repo_index, repo_path)for extractor in extractors:- found = await asyncio.to_thread(extractor.extract, repo_path, alias, exclude)+ kwargs = (+ {"index": index, "content_hashes": content_hashes}+ if isinstance(extractor, HttpExtractor)+ else {}+ )+ found = await asyncio.to_thread(+ lambda e=extractor, kw=kwargs: e.extract(+ repo_path, alias, exclude, files, **kw+ )+ )for c in found:c.service = assign_service(c.file_path, boundaries)+ c.meta.setdefault(EXTRACTION_LAYER_KEY, LAYER_REGEX)contracts.extend(found)return contractsdiff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.pyindex 078a9f714..3fb48c7e9 100644--- a/packages/core/src/repowise/core/workspace/extractors/base.py+++ b/packages/core/src/repowise/core/workspace/extractors/base.py@@ -12,7 +12,7 @@ descend into sibling/vendored repos rooted under the workspace.from __future__ import annotationsimport os-from collections.abc import Callable, Iterator, Mapping+from collections.abc import Callable, Iterator, Mapping, Sequencefrom dataclasses import dataclass, fieldfrom fnmatch import fnmatchfrom pathlib import Path@@ -39,6 +39,15 @@ _TEST_FILE_PATTERNS = ()+# Repowise's own contract-extractor sources. Their docstrings and comments spell+# out the syntax each dialect matches (``@app.get("/path")``, ``topic::orders``,+# ``AuthServiceStub``), so scanning them yields contracts for endpoints and+# queues that do not exist — on this repo, 14 of them. A regex over raw text+# cannot tell an example in a comment from a live route; excluding the tree that+# is nothing *but* examples is the cheap half of that fix.+_SELF_EXCLUDE_GLOBS = ("*/repowise/core/workspace/extractors/*",)++def is_test_path(rel_path: str) -> bool:"""True when *rel_path* (POSIX) lives in a test tree or is a test file."""parts = rel_path.split("/")@@ -55,16 +64,18 @@ def make_exclude_predicate() -> Callable[[str], bool]:"""Build a ``rel_path -> bool`` skip predicate for contract extraction.- Skips the default test/spec trees (unless *exclude_tests* is False) plus any+ Skips the default test/spec trees (unless *exclude_tests* is False), the+ extractors' own source tree (see :data:`_SELF_EXCLUDE_GLOBS`), plus anyuser-supplied ``extra_globs`` (matched against the full POSIX path and thebare filename)."""+ globs = _SELF_EXCLUDE_GLOBS + tuple(extra_globs)def skip(rel_path: str) -> bool:if exclude_tests and is_test_path(rel_path):return Truename = rel_path.rsplit("/", 1)[-1]- return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in extra_globs)+ return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in globs)return skip@@ -87,10 +98,34 @@ class ScanContext:mounts: Mapping[str, str] = field(default_factory=dict)+# One scanned source file: ``(rel_path, suffix, content)``.+SourceFile = tuple[str, str, str]+++def select_files(+ repo_path: Path,+ extensions: frozenset[str],+ exclude: Callable[[str], bool] | None,+ files: Sequence[SourceFile] | None,+ content_hashes: dict[str, str] | None = None,+) -> list[SourceFile]:+ """Filter an already-walked *files* list, or walk *repo_path* when it is None.++ Five extractors each used to walk the repo independently, so a repo was+ traversed and every matching file re-read once per extractor. The+ orchestrator now walks once and passes the result down; *files* being None+ keeps the standalone call (and every direct test) working unchanged.+ """+ if files is None:+ return list(iter_source_files(repo_path, extensions, exclude, content_hashes))+ return [f for f in files if f[1] in extensions]++def iter_source_files(repo_root: Path,extensions: frozenset[str],exclude: Callable[[str], bool] | None = None,+ content_hashes: dict[str, str] | None = None,) -> Iterator[tuple[str, str, str]]:"""Yield ``(rel_path, suffix, content)`` for each scannable source file.@@ -109,6 +144,14 @@ def iter_source_files(Only files whose lower-cased suffix is in *extensions* are yielded.Oversized, binary, generated, and ignored files are already excluded by thetraverser. Unreadable files are silently skipped.++ When *content_hashes* is given it is filled with ``rel_path -> sha256`` of+ each file's raw bytes — the same key the ingestion parse cache is stored+ under, so a caller can tell a matching parse from a stale one. Files are+ read as bytes and decoded here rather than via ``read_text`` so that hash+ describes the bytes on disk; the newline translation ``read_text`` would+ have applied is reproduced verbatim, leaving every dialect's input+ unchanged."""from repowise.core.ingestion.traverser import FileTraverser@@ -124,7 +167,16 @@ def iter_source_files(if exclude is not None and exclude(info.path):continuetry:- content = Path(info.abs_path).read_text(encoding="utf-8", errors="replace")+ raw = Path(info.abs_path).read_bytes()except OSError:continue+ content = raw.decode("utf-8", errors="replace")+ # Universal newlines, as ``read_text`` applied before this read moved+ # to bytes. Dialect regexes are written against ``\n``.+ if "\r" in content:+ content = content.replace("\r\n", "\n").replace("\r", "\n")+ if content_hashes is not None:+ from repowise.core.ingestion.models import compute_content_hash++ content_hashes[info.path] = compute_content_hash(raw)yield info.path, suffix, contentdiff --git a/packages/core/src/repowise/core/workspace/extractors/data/__init__.py b/packages/core/src/repowise/core/workspace/extractors/data/__init__.pyindex 09cafa98c..6f058c500 100644--- a/packages/core/src/repowise/core/workspace/extractors/data/__init__.py+++ b/packages/core/src/repowise/core/workspace/extractors/data/__init__.py@@ -13,7 +13,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from ..base import ScanContext, iter_source_files+from ..base import ScanContext, select_filesfrom .ddl import DdlDialectfrom .dialect import DataDialectfrom .names import normalize_table_name@@ -27,11 +27,13 @@ from .orm_models import (from .sql_strings import SqlStringsDialectif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom pathlib import Pathfrom repowise.core.workspace.contracts import Contract+ from ..base import SourceFile+# Table-ownership recognisers (DDL + one per ORM family).PROVIDER_DIALECTS: tuple[DataDialect, ...] = (DdlDialect(),@@ -59,18 +61,25 @@ class DataExtractor:provider_dialects: tuple[DataDialect, ...] = PROVIDER_DIALECTSconsumer_dialects: tuple[DataDialect, ...] = CONSUMER_DIALECTS+ @classmethod+ def source_extensions(cls) -> frozenset[str]:+ """Every extension this extractor's dialects claim."""+ return _union_extensions(cls.provider_dialects) | _union_extensions(+ cls.consumer_dialects+ )+def extract(self,repo_path: Path,repo_alias: str = "",exclude: Callable[[str], bool] | None = None,+ files: Sequence[SourceFile] | None = None,) -> list[Contract]:"""Scan all source files in *repo_path* and return Contract instances."""- all_exts = _union_extensions(self.provider_dialects) | _union_extensions(- self.consumer_dialects- )contracts: list[Contract] = []- for rel_path, suffix, content in iter_source_files(repo_path, all_exts, exclude):+ for rel_path, suffix, content in select_files(+ repo_path, self.source_extensions(), exclude, files+ ):ctx = ScanContext(repo_alias, rel_path, suffix, content)for dialect in self.provider_dialects:if suffix in dialect.extensions:diff --git a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.pyindex 01d5ef223..695ace2b1 100644--- a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py+++ b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py@@ -84,6 +84,33 @@ _SQL_KEYWORDS = frozenset(_MIN_LITERAL_LEN = 12+# A SQL literal *opens* with SQL. Prose that happens to satisfy the verb shape+# does not: a docstring reading "The incremental update path re-parses ... would+# SELECT the whole table ... vanished from a file" clears every gate above and+# yielded the tables ``path`` and ``would``. Anchoring to the start rejects+# prose as a class instead of blocklisting the English words it leaks.+#+# The opener set is deliberately wider than the statement verbs. A query is+# routinely preceded by a named-query comment (``-- name: get_user``, the+# aiosql/yesql/sqlc/HugSQL convention), a ``CREATE TABLE x AS SELECT ... FROM+# y`` whose read of ``y`` is a real consumer, an ``EXPLAIN``, or a T-SQL+# ``DECLARE``/``SET``/``BEGIN`` preamble. Requiring a bare statement verb threw+# all of those away; the verb gate below still decides whether it is SQL.+_SQL_COMMENT_PREFIX_RE = re.compile(r"^(?:\s|--[^\n]*\n|/\*.*?\*/)+", re.DOTALL)+_SQL_OPENER_RE = re.compile(+ r"^[(\s]*(?:"+ r"WITH|SELECT|INSERT|UPDATE|DELETE|MERGE|REPLACE|UPSERT"+ r"|CREATE|EXPLAIN|TRUNCATE|CALL|EXEC(?:UTE)?|DECLARE|SET|BEGIN|COPY"+ r")\b",+ re.IGNORECASE,+)+++def _opens_with_sql(literal: str) -> bool:+ """True when *literal* begins with SQL, ignoring any leading comment."""+ stripped = _SQL_COMMENT_PREFIX_RE.sub("", literal, count=1)+ return bool(_SQL_OPENER_RE.match(stripped))+def _verb_for_statement(stmt: object) -> str:kind = type(stmt).__name__.lower()@@ -121,6 +148,7 @@ class SqlStringsDialect:s = next(g for g in m.groups() if g is not None)if (len(s) >= _MIN_LITERAL_LEN+ and _opens_with_sql(s)and _SQL_VERB_RE.search(s)and (_UPPER_VERB_RE.search(s) or _SQL_PUNCT_RE.search(s))):@@ -156,10 +184,19 @@ class SqlStringsDialect:if stmt is None:return Noneverb = _verb_for_statement(stmt)+ # A CTE alias is a name bound by this statement, not a table it+ # reads. sqlglot models a reference to one as exp.Table all the+ # same, so ``WITH ranked AS (...) ... FROM ranked`` reported a+ # table named ``ranked``. Only unqualified references shadow.+ cte_names = {+ cte.alias.lower() for cte in stmt.find_all(exp.CTE) if cte.alias+ }for table in stmt.find_all(exp.Table):name = table.nameif not name:continue+ if not table.db and name.lower() in cte_names:+ continueraw = f"{table.db}.{name}" if table.db else namefound.append((raw, verb))return found if found else Nonediff --git a/packages/core/src/repowise/core/workspace/extractors/from_index.py b/packages/core/src/repowise/core/workspace/extractors/from_index.pynew file mode 100644index 000000000..0d54b2f21--- /dev/null+++ b/packages/core/src/repowise/core/workspace/extractors/from_index.py@@ -0,0 +1,262 @@+"""Index-backed contract extraction: read the symbols ingestion already parsed.++Workspace mode re-derived route tables from raw text with regexes, running+*after* the ingestion pipeline had parsed the very files it then re-read from+disk (``update.py`` indexes every stale repo, and only then calls+``run_cross_repo_hooks``). This module reads that parse instead.++The source is the per-repo parse cache (``<repo>/.repowise/parse_cache.pkl``),+which stores a :class:`~repowise.core.ingestion.models.ParsedFile` per file —+symbols, their kinds, their line ranges, and ``Symbol.decorators`` holding the+verbatim decorator text (``@router.post("")``). Reading a decorator node cannot+pick up a route-shaped string from a comment or a docstring, and cannot lose an+empty path, so two of the defect classes that motivated this module are gone by+construction rather than by a better regex.++**What the index does not carry.** A router's mount prefix comes from a call+expression (``APIRouter(prefix="/x")``, ``include_router(r, prefix="/y")``), not+from a decorator, and the cache stores extraction results rather than the tree.+Prefix resolution therefore still reads the file text via :mod:`.http.mounts`.+The route's *identity* is what moves onto the parse; its prefix is stitched on+exactly as before.++**Availability is not guaranteed.** :class:`ParseCache` gates every entry on+``parser_fingerprint()``, so a repo indexed by a different repowise version+loads as empty — measured on this workspace, two of three repos. Callers must+treat :func:`load_repo_index` returning ``None`` as "use the regex path", which+is also the honest answer for languages with no AST tier at all.+"""++from __future__ import annotations++import logging+import re+from typing import TYPE_CHECKING++from .http.dialect import build_provider_contract+from .http.mounts import compose_prefix, router_prefixes++if TYPE_CHECKING:+ from pathlib import Path++ from repowise.core.ingestion.models import ParsedFile+ from repowise.core.workspace.contracts import Contract++ from .base import ScanContext++_log = logging.getLogger(__name__)++# Recorded on every contract as ``meta[EXTRACTION_LAYER_KEY]`` so the coverage+# metric can report which layer produced it. Deliberately not ``meta["source"]``:+# eleven gRPC dialects already use that key for *dialect* provenance+# ("py_servicer", "go_client", "proto", ...), and reusing it would have left+# every gRPC contract classified as neither index- nor regex-sourced.+EXTRACTION_LAYER_KEY = "extraction_layer"+LAYER_INDEX = "index"+LAYER_REGEX = "regex"++# ---------------------------------------------------------------------------+# Decorator parsing+# ---------------------------------------------------------------------------++# A decorator's head and its first string argument, from the verbatim text the+# parser stores. ``@router.post("")`` -> ("router.post", ""). The argument is+# ``*`` not ``+``: an empty path is the idiomatic collection root on a prefixed+# router and must survive to be stitched onto that prefix.+_DECORATOR_RE = re.compile(r"""^@([\w.]+)\s*\(\s*(?:(?P<q>['"])(?P<arg>[^'"]*)(?P=q))?""")++# HTTP verbs usable as the trailing segment of a decorator head+# (``@router.get`` / ``@app.post``).+_VERB_METHODS = frozenset({"get", "post", "put", "patch", "delete", "head", "options"})++# Flask/Django-style ``@app.route("/x", methods=["POST"])``. Ingestion resolves+# Flask (framework edges + dynamic hints) while the workspace regex layer was+# FastAPI-only, so this is the framework the index path inherits for free.+_METHODS_KW_RE = re.compile(r"""methods\s*=\s*\[([^\]]*)\]""")+_METHOD_LITERAL_RE = re.compile(r"""['"]([A-Za-z]+)['"]""")++_ROUTE_HEADS = frozenset({"route"})++# The languages anything here reads decorators for. Widening this means+# widening :func:`extract_http_providers` to match.+INDEX_SUFFIXES = frozenset({".py"})+++def _decorator_head_and_arg(decorator: str) -> tuple[str, str | None]:+ """Split verbatim decorator text into its dotted head and first string arg.++ ``@router.post("/x")`` -> ``("router.post", "/x")``;+ ``@router.post("")`` -> ``("router.post", "")``;+ ``@dataclass`` -> ``("dataclass", None)``.+ """+ m = _DECORATOR_RE.match(decorator.strip())+ if m is None:+ return decorator.strip().lstrip("@"), None+ return m.group(1), m.group("arg")+++def _routes_in_decorator(decorator: str) -> list[tuple[str, str, str]]:+ """Return ``(router_var, METHOD, raw_path)`` for a route decorator.++ A ``@app.route(...)`` carrying ``methods=[...]`` yields one entry per verb+ (Flask's way of declaring several on one handler); a verb-named decorator+ yields exactly one. Anything else yields nothing.+ """+ head, arg = _decorator_head_and_arg(decorator)+ if arg is None or "." not in head:+ return []+ var, _, tail = head.rpartition(".")+ tail = tail.lower()++ if tail in _VERB_METHODS:+ return [(var, tail.upper(), arg)]++ if tail in _ROUTE_HEADS:+ mk = _METHODS_KW_RE.search(decorator)+ verbs = [v.upper() for v in _METHOD_LITERAL_RE.findall(mk.group(1))] if mk else []+ # Flask defaults to GET when ``methods`` is absent.+ return [(var, verb, arg) for verb in (verbs or ["GET"])]++ return []+++# ---------------------------------------------------------------------------+# Reading the parse cache+# ---------------------------------------------------------------------------+++def load_repo_index(+ repo_root: Path,+ suffixes: frozenset[str] = INDEX_SUFFIXES,+) -> dict[str, ParsedFile] | None:+ """Return ``rel_path -> ParsedFile`` from the repo's parse cache.++ Only entries whose path ends in one of *suffixes* are deserialized. The+ cache holds every file the repo indexed (27 MB of pickled ``ParsedFile``+ graphs on this repo) and only the languages read here are ever consulted,+ so unpickling the rest would be pure memory cost — multiplied by the repos+ running concurrently.++ ``None`` when the cache is absent or unusable — a repo indexed by another+ repowise version fails the ``parser_fingerprint`` gate and loads empty,+ which is indistinguishable from "never indexed" and gets the same answer:+ fall back to the regex path.+ """+ import pickle++ from repowise.core.ingestion.parse_cache import ParseCache++ cache_dir = repo_root / ".repowise"+ if not (cache_dir / "parse_cache.pkl").is_file():+ return None++ cache = ParseCache(cache_dir)+ try:+ cache.load()+ except Exception:+ _log.debug("Parse cache unreadable for %s", repo_root, exc_info=True)+ return None++ # ParseCache exposes lookup by (FileInfo, hash) but not enumeration, and+ # this needs the whole map. Reaching for the private dict is the coupling+ # that buys that; log loudly rather than go dark if it ever moves.+ entries = getattr(cache, "_entries", None)+ if entries is None:+ _log.warning(+ "ParseCache no longer exposes _entries; index-backed contract "+ "extraction is disabled and everything falls back to regex"+ )+ return None+ if not entries:+ _log.info(+ "Parse cache for %s loaded no entries (version mismatch or empty); "+ "contract extraction falls back to the regex path",+ repo_root,+ )+ return None++ out: dict[str, ParsedFile] = {}+ for (rel_path, content_hash), blob in entries.items():+ if not rel_path.endswith(tuple(suffixes)):+ continue+ try:+ parsed = pickle.loads(blob)+ except Exception: # a single corrupt entry must not lose the repo+ _log.debug("Skipping unpicklable parse-cache entry %s", rel_path)+ continue+ # The cache key carries the hash the entry was parsed from; keep it on+ # the object so :func:`parsed_for` can reject a stale entry.+ parsed.content_hash = content_hash+ out[rel_path] = parsed+ return out or None+++def parsed_for(+ index: dict[str, ParsedFile] | None,+ rel_path: str,+ content_hash: str | None,+) -> ParsedFile | None:+ """The parse for *rel_path*, but only if it describes the bytes on disk.++ Parse-cache entries are keyed by content hash. An entry whose hash does not+ match the file just walked describes an older version of it, and trusting+ that would report routes which no longer exist (or miss ones that now do).+ Without a hash to check against there is nothing to trust, so the caller+ gets ``None`` and stays on the regex path.+ """+ if index is None or content_hash is None:+ return None+ parsed = index.get(rel_path)+ if parsed is None:+ return None+ if parsed.content_hash != content_hash:+ _log.debug("Parse-cache entry for %s is stale; using the regex path", rel_path)+ return None+ return parsed+++# ---------------------------------------------------------------------------+# Provider extraction+# ---------------------------------------------------------------------------++def extract_http_providers(+ ctx: ScanContext,+ parsed: ParsedFile,+) -> list[Contract]:+ """HTTP provider contracts from *parsed*'s decorated symbols.++ Routes come from ``Symbol.decorators``; the mount prefix still comes from+ the file text (see the module docstring), so ``ctx.content`` is required.+ """+ prefixes = router_prefixes(ctx.content, "APIRouter|FastAPI|Flask|Blueprint")+ known = set(prefixes) | {"app", "router", "bp", "blueprint"}++ out: list[Contract] = []+ seen: set[tuple[str, str]] = set()+ for symbol in parsed.symbols:+ # ``Symbol.decorators`` can repeat an entry (the parser appends per+ # matching capture), so dedupe per symbol before building contracts.+ for decorator in dict.fromkeys(symbol.decorators):+ for var, method, raw_path in _routes_in_decorator(decorator):+ if var not in known:+ continue+ path = compose_prefix(prefixes.get(var, ""), raw_path)+ path = compose_prefix(ctx.mounts.get(var, ""), path)+ key = (method, path)+ if key in seen:+ continue+ seen.add(key)+ framework = "flask" if _is_route_head(decorator) else "fastapi"+ contract = build_provider_contract(+ ctx, method=method, path_raw=path, framework=framework+ )+ if contract is not None:+ contract.meta[EXTRACTION_LAYER_KEY] = LAYER_INDEX+ contract.meta["handler"] = symbol.name+ out.append(contract)+ return out+++def _is_route_head(decorator: str) -> bool:+ head, _ = _decorator_head_and_arg(decorator)+ return head.rpartition(".")[2].lower() in _ROUTE_HEADSdiff --git a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.pyindex 96b37ba73..a83a42d1a 100644--- a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py+++ b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py@@ -11,7 +11,7 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from ..base import ScanContext, iter_source_files+from ..base import ScanContext, select_filesfrom .csharp import CSharpGrpcDialectfrom .dialect import GrpcDialectfrom .go import GoGrpcDialect@@ -21,11 +21,13 @@ from .python import PythonGrpcDialectfrom .typescript import TypeScriptGrpcDialectif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom pathlib import Pathfrom repowise.core.workspace.contracts import Contract+ from ..base import SourceFile+# One dialect per language/IDL; extension sets are disjoint, so exactly one runs# per file.DIALECTS: tuple[GrpcDialect, ...] = (@@ -50,16 +52,22 @@ class GrpcExtractor:dialects: tuple[GrpcDialect, ...] = DIALECTS+ @classmethod+ def source_extensions(cls) -> frozenset[str]:+ """Every extension this extractor's dialects claim."""+ return _union_extensions(cls.dialects)+def extract(self,repo_path: Path,repo_alias: str = "",exclude: Callable[[str], bool] | None = None,+ files: Sequence[SourceFile] | None = None,) -> list[Contract]:- all_exts = _union_extensions(self.dialects)-contracts: list[Contract] = []- for rel_path, suffix, content in iter_source_files(repo_path, all_exts, exclude):+ for rel_path, suffix, content in select_files(+ repo_path, self.source_extensions(), exclude, files+ ):ctx = ScanContext(repo_alias, rel_path, suffix, content)for dialect in self.dialects:if suffix in dialect.extensions:diff --git a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py b/packages/core/src/repowise/core/workspace/extractors/http/__init__.pyindex 491a3925c..8725c6253 100644--- a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py+++ b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py@@ -12,7 +12,8 @@ from __future__ import annotationsfrom typing import TYPE_CHECKING-from ..base import ScanContext, iter_source_files+from ..base import ScanContext, select_files+from ..langs import PYTHONfrom .aspnet import AspNetDialectfrom .csharp_http import CSharpHttpDialectfrom .dialect import HttpDialect@@ -29,11 +30,13 @@ from .rust_clients import RustClientsDialectfrom .spring import SpringDialectif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom pathlib import Pathfrom repowise.core.workspace.contracts import Contract+ from ..base import SourceFile+# Route-declaration recognisers (one framework each).PROVIDER_DIALECTS: tuple[HttpDialect, ...] = (ExpressDialect(),@@ -54,6 +57,12 @@ CONSUMER_DIALECTS: tuple[HttpDialect, ...] = ()+# Provider dialects the index path fully replaces for a file it has parsed.+# Only the Python route regex is superseded today; every other framework still+# runs its dialect, because nothing reads their decorators yet.+_INDEX_BACKED_DIALECTS = frozenset({"fastapi"})++def _union_extensions(dialects: tuple[HttpDialect, ...]) -> frozenset[str]:out: set[str] = set()for d in dialects:@@ -67,31 +76,70 @@ class HttpExtractor:provider_dialects: tuple[HttpDialect, ...] = PROVIDER_DIALECTSconsumer_dialects: tuple[HttpDialect, ...] = CONSUMER_DIALECTS+ @classmethod+ def source_extensions(cls) -> frozenset[str]:+ """Every extension this extractor's dialects claim."""+ return _union_extensions(cls.provider_dialects) | _union_extensions(+ cls.consumer_dialects+ )+def extract(self,repo_path: Path,repo_alias: str = "",exclude: Callable[[str], bool] | None = None,+ files: Sequence[SourceFile] | None = None,+ index: dict[str, object] | None = None,+ content_hashes: dict[str, str] | None = None,) -> list[Contract]:"""Scan all source files in *repo_path* and return Contract instances.Files are read once into memory so a first pass can collect repo-widerouter mounts (``include_router(prefix=...)`` / ``app.use('/x', router)``)before the extraction pass stitches them onto each provider route.++ *files* is an already-walked ``(rel_path, suffix, content)`` list from+ the orchestrator's single traversal; None walks the repo directly.+ *index* maps ``rel_path -> ParsedFile`` from the repo's parse cache. A+ file present there gets its Python routes from the parsed decorators+ rather than from the FastAPI text regex — a decorator node cannot carry+ a route from a comment, and cannot lose an empty path. *content_hashes*+ is the walk's ``rel_path -> sha256`` map, which is what proves a cached+ parse describes the file as it is now; without it the index is unused."""- all_exts = _union_extensions(self.provider_dialects) | _union_extensions(- self.consumer_dialects+ own_hashes: dict[str, str] | None = None+ if files is None and index is not None and content_hashes is None:+ own_hashes = content_hashes = {}+ scanned = select_files(+ repo_path, self.source_extensions(), exclude, files, own_hashes)+ mounts = self._collect_mounts(scanned)- files = list(iter_source_files(repo_path, all_exts, exclude))- mounts = self._collect_mounts(files)+ from ..from_index import extract_http_providers, parsed_forcontracts: list[Contract] = []- for rel_path, suffix, content in files:+ for rel_path, suffix, content in scanned:ctx = ScanContext(repo_alias, rel_path, suffix, content, mounts)+ # Only a parse that matches the bytes just walked supersedes the+ # regex; a stale or missing entry leaves the dialect in charge.+ parsed = (+ parsed_for(index, rel_path, (content_hashes or {}).get(rel_path))+ if suffix in PYTHON+ else None+ )+ # Run the index pass first: it only supersedes the text dialect for+ # a file it actually produced routes for. A parse that yielded no+ # decorated symbols (a grammar the parser stumbled on, a route+ # shape the queries do not capture) must not silently delete the+ # routes the regex can still see in the file's text.+ from_parse = extract_http_providers(ctx, parsed) if parsed else []+ contracts.extend(from_parse)for dialect in self.provider_dialects:- if suffix in dialect.extensions:- contracts.extend(dialect.extract(ctx))+ if suffix not in dialect.extensions:+ continue+ if from_parse and dialect.name in _INDEX_BACKED_DIALECTS:+ continue # superseded by the index pass above+ contracts.extend(dialect.extract(ctx))for dialect in self.consumer_dialects:if suffix in dialect.extensions:contracts.extend(dialect.extract(ctx))diff --git a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.pyindex d4acb6255..3b37f6ad5 100644--- a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py+++ b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py@@ -23,8 +23,12 @@ if TYPE_CHECKING:# router prefix can be resolved; only known routers (those bound to APIRouter /# FastAPI in-file, plus the conventional ``app`` / ``router`` names) are kept, so# an unrelated ``@cache.get(...)`` decorator is not mistaken for a route.+# The path is ``*`` not ``+``: ``@router.post("")`` on a prefixed router is the+# idiomatic way to serve the collection root, and requiring one character made+# those routes invisible entirely. The empty path contributes nothing on its own,+# so ``build_provider_contract`` still drops it when no prefix stitches on._FASTAPI_RE = re.compile(- rf"""@(\w+)\.({METHODS})\s*\(\s*['"]([^'"]+)['"]""",+ rf"""@(\w+)\.({METHODS})\s*\(\s*['"]([^'"]*)['"]""",re.IGNORECASE,)diff --git a/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py b/packages/core/src/repowise/core/workspace/extractors/socket_extractor.pyindex 48c7469e6..56b28e4c2 100644--- a/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py+++ b/packages/core/src/repowise/core/workspace/extractors/socket_extractor.py@@ -12,7 +12,7 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import TYPE_CHECKING-from .base import iter_source_files+from .base import select_filesfrom .http.paths import (extract_path_from_url,is_unusable_consumer_path,@@ -22,10 +22,12 @@ from .http.paths import (from .langs import CSHARP, PYTHONif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom repowise.core.workspace.contracts import Contract+ from .base import SourceFile+_log = logging.getLogger("repowise.workspace.extractors.socket")_EXTENSIONS = CSHARP | PYTHON@@ -141,18 +143,26 @@ def _normalize_socket_identity(raw: str, prefix: str = "") -> str | None:class SocketExtractor:"""Extract socket/websocket contracts from source files."""+ @classmethod+ def source_extensions(cls) -> frozenset[str]:+ """Every extension this extractor scans."""+ return _EXTENSIONS+def extract(self,repo_path: Path,repo_alias: str = "",exclude: Callable[[str], bool] | None = None,+ files: Sequence[SourceFile] | None = None,) -> list[Contract]:from repowise.core.workspace.contracts import Contractcontracts: list[Contract] = []seen: set[tuple[str, str, str]] = set()- for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS, exclude):+ for rel_path, _suffix, content in select_files(+ repo_path, _EXTENSIONS, exclude, files+ ):for pdef in _CLIENT_PATTERNS + _PROVIDER_PATTERNS:if pdef.context_regex is not None and not pdef.context_regex.search(content):continuediff --git a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.pyindex 7466036cd..a5882dd98 100644--- a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py+++ b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py@@ -14,13 +14,15 @@ from typing import TYPE_CHECKINGfrom repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTRY-from .base import iter_source_files+from .base import select_filesif TYPE_CHECKING:- from collections.abc import Callable+ from collections.abc import Callable, Sequencefrom repowise.core.workspace.contracts import Contract+ from .base import SourceFile+_log = logging.getLogger("repowise.workspace.extractors.topic")# ---------------------------------------------------------------------------@@ -222,11 +224,17 @@ _ALL_PATTERNS = _KAFKA_PATTERNS + _RABBITMQ_PATTERNS + _NATS_PATTERNSclass TopicExtractor:"""Extract message topic/queue contracts from source files."""+ @classmethod+ def source_extensions(cls) -> frozenset[str]:+ """Every extension this extractor scans."""+ return _EXTENSIONS+def extract(self,repo_path: Path,repo_alias: str = "",exclude: Callable[[str], bool] | None = None,+ files: Sequence[SourceFile] | None = None,) -> list[Contract]:from repowise.core.workspace.contracts import Contract@@ -236,7 +244,9 @@ class TopicExtractor:# File discovery is gitignore- and nested-repo-aware (see# ``iter_source_files``) so a workspace repo whose root contains nested# repos or large ignored trees is not scanned end-to-end.- for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS, exclude):+ for rel_path, _suffix, content in select_files(+ repo_path, _EXTENSIONS, exclude, files+ ):for pdef in _ALL_PATTERNS:for match in pdef.regex.finditer(content):topic_name = match.group(pdef.topic_group).strip()diff --git a/packages/core/src/repowise/core/workspace/system_graph.py b/packages/core/src/repowise/core/workspace/system_graph.pyindex 4b339a6ea..2b5eeadf0 100644--- a/packages/core/src/repowise/core/workspace/system_graph.py+++ b/packages/core/src/repowise/core/workspace/system_graph.py@@ -471,15 +471,21 @@ async def run_system_graph_build(workspace_root: Path,store: ContractStore,overlay: CrossRepoOverlay,+ boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,) -> SystemGraph:"""Build and persist the system graph from the latest contracts + overlay.Called from ``run_cross_repo_hooks`` after contract extraction andcross-repo analysis. Boundary detection (a filesystem walk) runs off-thread.++ *boundaries_by_repo* is the map ``run_cross_repo_hooks`` already computed+ for contract extraction. Both consumers need the same answer, so passing it+ in halves the boundary walks per update; None re-detects for direct callers."""- boundaries_by_repo = await asyncio.to_thread(- _detect_boundaries_by_repo, ws_config, workspace_root- )+ if boundaries_by_repo is None:+ boundaries_by_repo = await asyncio.to_thread(+ _detect_boundaries_by_repo, ws_config, workspace_root+ )diagnostics = build_diagnostics(store.contracts, store.contract_links)graph = build_system_graph(diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.pyindex b7394bd8f..a3376f563 100644--- a/packages/core/src/repowise/core/workspace/update.py+++ b/packages/core/src/repowise/core/workspace/update.py@@ -16,7 +16,10 @@ from contextlib import suppressfrom dataclasses import dataclassfrom datetime import UTC, datetimefrom pathlib import Path-from typing import Any+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from repowise.core.workspace.extractors.service_boundary import ServiceBoundary# Per-repo update lock — the shared single-flight guard (one implementation# for the CLI update command and this workspace updater, which used to carry@@ -904,7 +907,22 @@ async def run_cross_repo_hooks(from .conformance import run_conformance_checkfrom .contracts import ContractStore, load_contract_store, run_contract_extractionfrom .cross_repo import CrossRepoOverlay, run_cross_repo_analysis- from .system_graph import SystemGraph, run_system_graph_build+ from .system_graph import (+ SystemGraph,+ _detect_boundaries_by_repo,+ run_system_graph_build,+ )++ # Service boundaries, detected once for the whole workspace. Contract+ # extraction and the system-graph build both need them and each used to walk+ # every repo for its own copy.+ boundaries_by_repo: dict[str, list[ServiceBoundary]] = {}+ try:+ boundaries_by_repo = await asyncio.to_thread(+ _detect_boundaries_by_repo, ws_config, workspace_root+ )+ except Exception:+ _log.warning("Service boundary detection failed", exc_info=True)overlay = CrossRepoOverlay()try:@@ -920,7 +938,9 @@ async def run_cross_repo_hooks(# Contract extraction (overwrites contracts.json).store = ContractStore()try:- store = await run_contract_extraction(ws_config, workspace_root, changed_repos)+ store = await run_contract_extraction(+ ws_config, workspace_root, changed_repos, boundaries_by_repo or None+ )except Exception:_log.warning("Contract extraction failed", exc_info=True)@@ -928,7 +948,9 @@ async def run_cross_repo_hooks(# view reads. Built last so it folds in the contracts and overlay above.system_graph: SystemGraph | None = Nonetry:- system_graph = await run_system_graph_build(ws_config, workspace_root, store, overlay)+ system_graph = await run_system_graph_build(+ ws_config, workspace_root, store, overlay, boundaries_by_repo or None+ )except Exception:_log.warning("System graph build failed", exc_info=True)diff --git a/tests/unit/workspace/test_dialect_regressions.py b/tests/unit/workspace/test_dialect_regressions.pynew file mode 100644index 000000000..f71461276--- /dev/null+++ b/tests/unit/workspace/test_dialect_regressions.py@@ -0,0 +1,249 @@+"""Per-dialect regression locks for contracts that were fabricated or lost.++The dialects under ``workspace/extractors/`` had no direct tests: coverage was+indirect, through ``test_contracts.py`` and ``test_extractor_traversal.py``+exercising the top-level extractors. Four defects reached the live workspace+artifact through that gap, and each one gets a test here that fails against the+pre-fix code:++1. ``@router.post("")`` on a prefixed router was invisible — the decorator+ regex required at least one path character. It cost 2 of the 3 real SSE+ endpoints in the hosted backend.+2. The extractors' own sources were scanned, so the example syntax in their+ docstrings became 14 live contracts (``topic::orders``, ``http::GET::/path``,+ ``grpc::AuthService/*``).+3. ``WITH ranked AS (...)`` reported a table named ``ranked`` — sqlglot models a+ CTE reference as a table.+4. A docstring reading "...update path..." / "...update would..." satisfied the+ SQL verb gate and became the tables ``path`` and ``would``.+"""++from __future__ import annotations++from pathlib import Path++from repowise.core.workspace.extractors.base import (+ ScanContext,+ make_exclude_predicate,+)+from repowise.core.workspace.extractors.data import DataExtractor+from repowise.core.workspace.extractors.data.sql_strings import SqlStringsDialect+from repowise.core.workspace.extractors.http.fastapi import FastApiDialect+++def _ctx(content: str, rel_path: str = "app/routers/chat.py") -> ScanContext:+ return ScanContext(+ repo_alias="backend", rel_path=rel_path, suffix=".py", content=content+ )+++def _ids(contracts) -> set[str]:+ return {c.contract_id for c in contracts}+++class TestFastApiEmptyPath:+ """Regression 1: ``@router.post("")`` mounted at a router prefix."""++ # Mirrors backend/app/routers/workspace_chat.py:34,162 and chat.py:49,352.+ SOURCE = '''+router = APIRouter(prefix="/snapshots/{snapshot_id}/chat", tags=["chat"])+++async def chat_message(request: Request) -> None:+ ...++[email protected]("/conversations")+async def list_conversations() -> None:+ ...+'''++ def test_empty_path_route_is_extracted_at_the_router_prefix(self) -> None:+ ids = _ids(FastApiDialect().extract(_ctx(self.SOURCE)))+ assert "http::POST::/snapshots/{param}/chat" in ids++ def test_sibling_routes_still_extracted(self) -> None:+ ids = _ids(FastApiDialect().extract(_ctx(self.SOURCE)))+ assert "http::GET::/snapshots/{param}/chat/conversations" in ids++ def test_empty_path_without_a_prefix_contributes_nothing(self) -> None:+ # No prefix to stitch onto, so there is no path to record. Dropped by+ # build_provider_contract rather than silently becoming bare "/".+ src = '''+app = FastAPI()+++async def nothing() -> None:+ ...+'''+ assert FastApiDialect().extract(_ctx(src)) == []++ def test_non_router_decorator_is_not_a_route(self) -> None:+ # The empty-path relaxation must not turn every @x.get() into a route.+ src = '''+router = APIRouter(prefix="/api")++[email protected]("/not-a-route")+def cached() -> None:+ ...+'''+ assert FastApiDialect().extract(_ctx(src)) == []+++class TestExtractorSelfExclusion:+ """Regression 2: the extractors' own example syntax became live contracts."""++ SELF_PATHS = (+ "packages/core/src/repowise/core/workspace/extractors/http/fastapi.py",+ "packages/core/src/repowise/core/workspace/extractors/http/mounts.py",+ "packages/core/src/repowise/core/workspace/extractors/topic_extractor.py",+ "packages/core/src/repowise/core/workspace/extractors/grpc/python.py",+ "packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py",+ )++ def test_extractor_sources_are_skipped(self) -> None:+ skip = make_exclude_predicate()+ for path in self.SELF_PATHS:+ assert skip(path), path++ def test_the_rest_of_the_workspace_package_is_still_scanned(self) -> None:+ # The exclusion is the extractors tree only, not workspace/ at large.+ skip = make_exclude_predicate()+ assert not skip("packages/core/src/repowise/core/workspace/contracts.py")++ def test_ordinary_source_is_unaffected(self) -> None:+ skip = make_exclude_predicate()+ assert not skip("app/routers/chat.py")+ assert not skip("src/lib/api/client.ts")++ def test_user_supplied_globs_still_apply(self) -> None:+ skip = make_exclude_predicate(("vendor/*",))+ assert skip("vendor/thing.py")+ assert not skip("app/thing.py")+++class TestSqlCteAlias:+ """Regression 3: ``WITH ranked AS (...)`` emitted a table named ``ranked``."""++ # Reduced from packages/core/src/repowise/core/persistence/crud/git.py:258.+ LITERAL = """+WITH ranked AS (+ SELECT id, PERCENT_RANK() OVER (ORDER BY commit_count_90d) AS prank+ FROM git_metadata+ WHERE repository_id = :repo_id+)+UPDATE git_metadata+SET churn_percentile = (SELECT prank FROM ranked WHERE ranked.id = git_metadata.id)+"""++ def test_cte_alias_is_not_a_table(self) -> None:+ tables = {t for t, _verb in SqlStringsDialect()._tables_in(self.LITERAL)}+ assert "ranked" not in tables++ def test_the_real_table_is_still_found(self) -> None:+ tables = {t for t, _verb in SqlStringsDialect()._tables_in(self.LITERAL)}+ assert "git_metadata" in tables++ def test_a_real_table_sharing_a_cte_name_elsewhere_is_unaffected(self) -> None:+ # Shadowing is per-statement: no CTE here, so ``ranked`` is a table.+ tables = {+ t for t, _verb in SqlStringsDialect()._tables_in("SELECT id FROM ranked")+ }+ assert "ranked" in tables+++class TestSqlProseRejection:+ """Regression 4: a docstring became the tables ``path`` and ``would``."""++ # Reduced from the batch_upsert_symbols docstring in+ # packages/core/src/repowise/core/persistence/crud/external_systems.py.+ DOCSTRING_SOURCE = '''+def batch_upsert_symbols(file_paths, symbols):+ """Make ``wiki_symbols`` for *file_paths* match a fresh parse (*symbols*).++ The incremental update path re-parses changed files but historically never+ persisted their symbols. Calling the repo-wide helper per update would+ SELECT the whole table. This prunes rows for symbols that vanished from a+ still-existing file.+ """+'''++ def test_docstring_prose_yields_no_tables(self) -> None:+ ctx = _ctx(self.DOCSTRING_SOURCE, rel_path="crud/external_systems.py")+ assert SqlStringsDialect().extract(ctx) == []++ def test_the_leaked_words_are_gone(self) -> None:+ ctx = _ctx(self.DOCSTRING_SOURCE, rel_path="crud/external_systems.py")+ ids = _ids(SqlStringsDialect().extract(ctx))+ assert "data::path" not in ids+ assert "data::would" not in ids++ def test_real_sql_in_the_same_file_is_still_extracted(self) -> None:+ src = self.DOCSTRING_SOURCE + '''+ rows = conn.execute("SELECT id, name FROM wiki_symbols WHERE repo_id = :r")+'''+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "crud/external_systems.py")))+ assert "data::wiki_symbols" in ids++ def test_named_query_comment_still_opens_the_gate(self) -> None:+ # aiosql / yesql / sqlc / HugSQL all prefix a query with a name comment.+ src = '''+q = """+-- name: get_user+SELECT id FROM accounts WHERE id = :id+"""+'''+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))+ assert "data::accounts" in ids++ def test_block_comment_prefix_still_opens_the_gate(self) -> None:+ src = 'q = "/* cached */ SELECT id FROM accounts WHERE id = 1"\n'+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))+ assert "data::accounts" in ids++ def test_create_table_as_select_records_the_table_it_reads(self) -> None:+ src = 'q = "CREATE TABLE snapshot AS SELECT id FROM accounts WHERE x = 1"\n'+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))+ assert "data::accounts" in ids++ def test_explain_still_opens_the_gate(self) -> None:+ src = 'q = "EXPLAIN ANALYZE SELECT id FROM accounts WHERE id = 1"\n'+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))+ assert "data::accounts" in ids++ def test_prose_beginning_with_a_dash_is_still_rejected(self) -> None:+ # The comment strip must not become a way back in for prose.+ src = '''+def f():+ """- The incremental update path would SELECT rows from a file."""+'''+ assert SqlStringsDialect().extract(_ctx(src, "f.py")) == []++ def test_indented_sql_literal_still_opens_the_gate(self) -> None:+ # Multi-line SQL literals usually start with a newline and indentation.+ src = '''+q = """+ SELECT id FROM projects WHERE owner = :owner+"""+'''+ ids = _ids(SqlStringsDialect().extract(_ctx(src, "q.py")))+ assert "data::projects" in ids+++class TestDataExtractorEndToEnd:+ """The two SQL fixes hold through the DataExtractor, not just the dialect."""++ def test_prose_and_cte_do_not_reach_contracts(self, tmp_path: Path) -> None:+ (tmp_path / "git.py").write_text(+ f'SQL = """{TestSqlCteAlias.LITERAL}"""\n', encoding="utf-8"+ )+ (tmp_path / "external_systems.py").write_text(+ TestSqlProseRejection.DOCSTRING_SOURCE, encoding="utf-8"+ )+ ids = _ids(DataExtractor().extract(tmp_path, "repowise", make_exclude_predicate()))+ assert "data::git_metadata" in ids+ for fabricated in ("data::ranked", "data::path", "data::would"):+ assert fabricated not in idsdiff --git a/tests/unit/workspace/test_extraction_walk_budget.py b/tests/unit/workspace/test_extraction_walk_budget.pynew file mode 100644index 000000000..485c0e89c--- /dev/null+++ b/tests/unit/workspace/test_extraction_walk_budget.py@@ -0,0 +1,231 @@+"""Traversal budget: one tree walk and one boundary detection per repo.++Five extractors each walked the repo and re-read every file whose extension it+claimed, and boundary detection then ran a sixth walk — per repo, per update.+Wall time is machine-dependent and makes a poor gate, but walk and read counts+are exact, so they are what these tests assert.++Measured on the three-repo workspace this change was developed against+(``repowise`` + ``backend`` + ``frontend``), for ``run_contract_extraction``:++=========================== ======== =======+metric before after+=========================== ======== =======+FileTraverser walks / repo 5 1+file reads (all repos) 13,889 3,216+wall time 45.23s 15.46s+contracts / links 714 / 101 714 / 101+=========================== ======== =======++Boundary detection keeps its own walk through ``fs_walk.walk_repo``, so the+per-repo total is 2 rather than 1; what changed there is that it runs once per+update instead of once for contract extraction and again for the system graph.+"""++from __future__ import annotations++from pathlib import Path++import pytest++from repowise.core.ingestion.traverser import FileTraverser+from repowise.core.workspace import contracts as contracts_mod+from repowise.core.workspace import system_graph as system_graph_mod+from repowise.core.workspace.config import ContractConfig, RepoEntry, WorkspaceConfig+from repowise.core.workspace.contracts import run_contract_extraction++REPO_SOURCE = {+ "svc_a/api.py": (+ 'router = APIRouter(prefix="/api")\n\n\[email protected]("/users")\ndef users(): ...\n'+ ),+ "svc_a/db.py": 'Q = "SELECT id FROM users WHERE id = :id"\n',+ "svc_a/pyproject.toml": '[project]\nname = "svc-a"\n',+ "svc_b/client.ts": "await fetch('/api/users');\n",+ "svc_b/package.json": '{"name": "svc-b"}\n',+ "svc_b/queue.js": "channel.sendToQueue('jobs', payload);\n",+}+++def _make_repo(root: Path) -> None:+ for rel, content in REPO_SOURCE.items():+ path = root / rel+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(content, encoding="utf-8")+ (root / ".repowise").mkdir(exist_ok=True)+++def workspace(tmp_path: Path) -> WorkspaceConfig:+ for alias in ("alpha", "beta"):+ _make_repo(tmp_path / alias)+ return WorkspaceConfig(+ repos=[RepoEntry(path="alpha", alias="alpha"), RepoEntry(path="beta", alias="beta")],+ contracts=ContractConfig(),+ )+++def counters(monkeypatch: pytest.MonkeyPatch) -> dict[str, int]:+ """Count tree walks, file reads and boundary detections."""+ counts = {"walks": 0, "reads": 0, "boundaries": 0}++ original_traverse = FileTraverser.traverse++ def counted_traverse(self, *args, **kwargs):+ counts["walks"] += 1+ return original_traverse(self, *args, **kwargs)++ # Both, because the shared walk reads bytes (so it can hash them) while+ # other call sites still read text. Counting only one would let a second+ # read slip in through the other.+ original_read_text = Path.read_text+ original_read_bytes = Path.read_bytes++ def counted_read_text(self, *args, **kwargs):+ counts["reads"] += 1+ return original_read_text(self, *args, **kwargs)++ def counted_read_bytes(self, *args, **kwargs):+ counts["reads"] += 1+ return original_read_bytes(self, *args, **kwargs)++ monkeypatch.setattr(FileTraverser, "traverse", counted_traverse)+ monkeypatch.setattr(Path, "read_text", counted_read_text)+ monkeypatch.setattr(Path, "read_bytes", counted_read_bytes)++ from repowise.core.workspace import extractors as extractors_pkg+ from repowise.core.workspace.extractors import service_boundary++ original_detect = service_boundary.detect_service_boundaries++ def counted_detect(repo_path, *args, **kwargs):+ counts["boundaries"] += 1+ return original_detect(repo_path, *args, **kwargs)++ for module in (service_boundary, extractors_pkg, system_graph_mod):+ monkeypatch.setattr(module, "detect_service_boundaries", counted_detect)+ return counts+++async def test_one_tree_walk_per_repo(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ counters: dict[str, int],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ await run_contract_extraction(workspace, tmp_path, [])+ assert counters["walks"] == 2 # one per repo, not one per extractor per repo+++async def test_each_file_is_read_once_per_repo(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ counters: dict[str, int],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ await run_contract_extraction(workspace, tmp_path, [])+ # 4 source files per repo are in the extractors' combined extension set+ # (api.py, db.py, client.ts, queue.js); the manifests are not source.+ assert counters["reads"] <= 2 * len(REPO_SOURCE)+++async def test_boundaries_detected_once_per_repo_when_supplied(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ counters: dict[str, int],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ boundaries = system_graph_mod._detect_boundaries_by_repo(workspace, tmp_path)+ detections_for_the_shared_map = counters["boundaries"]++ await run_contract_extraction(workspace, tmp_path, [], boundaries)++ # Extraction reuses the map instead of re-detecting per repo.+ assert counters["boundaries"] == detections_for_the_shared_map+++async def test_boundaries_are_detected_when_not_supplied(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ counters: dict[str, int],+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ await run_contract_extraction(workspace, tmp_path, [])+ assert counters["boundaries"] == 2 # direct callers still get boundaries+++async def test_hooks_detect_boundaries_once_per_repo(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ counters: dict[str, int],+) -> None:+ """The 2-per-repo -> 1-per-repo budget, through the caller that wires it.++ ``run_cross_repo_hooks`` is where contract extraction and the system-graph+ build meet; each used to detect boundaries for every repo independently.+ """+ from repowise.core.workspace.update import run_cross_repo_hooks++ await run_cross_repo_hooks(workspace, tmp_path, [])++ assert counters["boundaries"] == 2 # two repos, once each — not 4+++async def test_the_shared_walk_finds_the_same_contracts(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ store = await run_contract_extraction(workspace, tmp_path, [])+ ids = {c.contract_id for c in store.contracts}+ assert "http::GET::/api/users" in ids # provider, prefix stitched+ assert "data::users" in ids # SQL consumer+ assert "topic::jobs" in ids # queue provider+++async def test_every_contract_records_its_extraction_layer(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ from repowise.core.workspace.extractors.from_index import (+ EXTRACTION_LAYER_KEY,+ LAYER_REGEX,+ )++ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ store = await run_contract_extraction(workspace, tmp_path, [])+ assert store.contracts+ # No parse cache in these fixture repos, so everything is regex-sourced.+ assert {c.meta.get(EXTRACTION_LAYER_KEY) for c in store.contracts} == {LAYER_REGEX}+++async def test_grpc_dialect_provenance_survives_the_layer_stamp(+ workspace: WorkspaceConfig,+ tmp_path: Path,+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A gRPC contract keeps its own ``meta["source"]`` and gains a layer."""+ from repowise.core.workspace.extractors.from_index import (+ EXTRACTION_LAYER_KEY,+ LAYER_REGEX,+ )++ (tmp_path / "alpha" / "svc_a" / "auth.proto").write_text(+ 'syntax = "proto3";\n\nservice AuthService {\n'+ " rpc Login (LoginRequest) returns (LoginReply);\n}\n",+ encoding="utf-8",+ )+ monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)+ store = await run_contract_extraction(workspace, tmp_path, [])++ grpc = [c for c in store.contracts if c.contract_type == "grpc"]+ assert grpc, "fixture should produce a gRPC contract"+ for contract in grpc:+ assert contract.meta["source"] == "proto" # dialect provenance intact+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_REGEXdiff --git a/tests/unit/workspace/test_from_index.py b/tests/unit/workspace/test_from_index.pynew file mode 100644index 000000000..476439b9f--- /dev/null+++ b/tests/unit/workspace/test_from_index.py@@ -0,0 +1,325 @@+"""Index-backed contract extraction: routes read from parsed decorators.++Two claims are under test here. The first is that reading the decorator a+parser already recorded removes a class of defect the regex could only chase:+a route-shaped string in a comment is not a decorator, and an empty path is+just a string argument, so neither can fabricate or lose a contract.++The second is the inherited-framework claim. Ingestion resolves Flask as well+as FastAPI, while the workspace regex layer was FastAPI-only. Flask arriving+here without a new dialect module is the evidence that the rest of the+framework list comes with the layer rather than being hand-written per+framework.+"""++from __future__ import annotations++from pathlib import Path++import pytest++from repowise.core.ingestion.models import FileInfo, ParsedFile, Symbol+from repowise.core.workspace.extractors.base import ScanContext+from repowise.core.workspace.extractors.from_index import (+ EXTRACTION_LAYER_KEY,+ LAYER_INDEX,+ extract_http_providers,+ load_repo_index,+)+++def _symbol(name: str, decorators: list[str], line: int = 10) -> Symbol:+ return Symbol(+ id=f"m.py::{name}",+ name=name,+ qualified_name=name,+ kind="function",+ signature=f"def {name}()",+ start_line=line,+ end_line=line + 3,+ docstring=None,+ decorators=decorators,+ language="python",+ )+++def _parsed(symbols: list[Symbol], rel_path: str = "app/routers/chat.py") -> ParsedFile:+ info = FileInfo(+ path=rel_path,+ abs_path=f"/repo/{rel_path}",+ language="python",+ size_bytes=100,+ git_hash="",+ last_modified=0.0,+ is_test=False,+ is_config=False,+ is_api_contract=True,+ is_entry_point=False,+ )+ return ParsedFile(file_info=info, symbols=symbols, imports=[], exports=[])+++def _ctx(content: str, rel_path: str = "app/routers/chat.py") -> ScanContext:+ return ScanContext(+ repo_alias="backend", rel_path=rel_path, suffix=".py", content=content+ )+++def _ids(contracts) -> set[str]:+ return {c.contract_id for c in contracts}+++class TestFastApiFromDecorators:+ CONTENT = 'router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")\n'++ def test_empty_path_route_resolves_to_the_prefix(self) -> None:+ parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))+ assert ids == {"http::POST::/snapshots/{param}/chat"}++ def test_provenance_is_recorded(self) -> None:+ parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX+ assert contract.meta["handler"] == "chat_message"++ def test_layer_key_does_not_collide_with_grpc_dialect_provenance(self) -> None:+ # The gRPC dialects already own meta["source"] for dialect provenance+ # ("py_servicer", "go_client", "proto"); the layer must not overwrite+ # it or the coverage metric mis-partitions every gRPC contract.+ assert EXTRACTION_LAYER_KEY != "source"+ parsed = _parsed([_symbol("chat", ['@router.post("")'])])+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]+ assert "source" not in contract.meta++ def test_duplicated_decorator_entries_yield_one_contract(self) -> None:+ # The parser can append the same decorator twice for one symbol.+ parsed = _parsed(+ [_symbol("chat", ['@router.post("")', '@router.post("")'])]+ )+ assert len(extract_http_providers(_ctx(self.CONTENT), parsed)) == 1++ def test_non_route_decorators_are_ignored(self) -> None:+ parsed = _parsed(+ [+ _symbol(+ "chat",+ ['@limiter.limit("240/minute")', "@dataclass", "@classmethod"],+ )+ ]+ )+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []++ def test_unknown_router_variable_is_not_a_route(self) -> None:+ parsed = _parsed([_symbol("cached", ['@cache.get("/x")'])])+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []+++class TestCommentsCannotFabricateRoutes:+ """The self-extraction defect, structurally: a comment is not a decorator."""++ CONTENT = '''+router = APIRouter(prefix="/api")++# Recognises @app.get('/path') and @router.post('/orders') declarations.+EXAMPLE = "@router.delete('/fabricated')"+++def documented() -> None:+ """Handles @router.put("/from-a-docstring") style routes."""+'''++ def test_route_shaped_text_outside_a_decorator_yields_nothing(self) -> None:+ # No symbol carries a route decorator, so there is no route — however+ # much route-shaped prose the file contains.+ parsed = _parsed([_symbol("documented", [])], rel_path="extractors/http.py")+ assert extract_http_providers(_ctx(self.CONTENT), parsed) == []++ def test_a_real_decorator_in_the_same_file_still_counts(self) -> None:+ parsed = _parsed(+ [_symbol("real", ['@router.get("/real")'])], rel_path="extractors/http.py"+ )+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))+ assert ids == {"http::GET::/api/real"}+++class TestFlaskIsInherited:+ """Flask providers via the index path — no Flask dialect module exists."""++ CONTENT = 'app = Flask(__name__)\nbp = Blueprint("api", __name__)\n'++ def test_route_decorator_defaults_to_get(self) -> None:+ parsed = _parsed([_symbol("index", ['@app.route("/health")'])])+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))+ assert ids == {"http::GET::/health"}++ def test_methods_kwarg_yields_one_contract_per_verb(self) -> None:+ parsed = _parsed(+ [_symbol("users", ['@app.route("/users", methods=["POST", "PUT"])'])]+ )+ ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))+ assert ids == {"http::POST::/users", "http::PUT::/users"}++ def test_framework_is_labelled_flask(self) -> None:+ parsed = _parsed([_symbol("index", ['@app.route("/health")'])])+ contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]+ assert contract.meta["framework"] == "flask"+ assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX++ def test_blueprint_url_prefix_is_stitched_on(self) -> None:+ # Flask spells it ``url_prefix=``, and the shared prefix pattern is not+ # left-anchored to a word boundary, so it matches that too and the+ # blueprint's mount lands on the route. Incidental, but correct — this+ # test pins the behaviour so a later tightening of the pattern has to+ # keep Flask working deliberately.+ content = 'bp = Blueprint("api", __name__, url_prefix="/api")\n'+ parsed = _parsed([_symbol("listing", ['@bp.route("/items")'])])+ ids = _ids(extract_http_providers(_ctx(content), parsed))+ assert ids == {"http::GET::/api/items"}++ def test_the_regex_layer_has_no_flask_dialect(self) -> None:+ # Guards the claim in this class's docstring: if a Flask dialect is+ # added later, this test should be deleted along with the claim.+ from repowise.core.workspace.extractors.http import PROVIDER_DIALECTS++ assert "flask" not in {d.name for d in PROVIDER_DIALECTS}+++class TestLoadRepoIndex:+ def test_missing_cache_returns_none(self, tmp_path) -> None:+ assert load_repo_index(tmp_path) is None++ def test_unreadable_cache_returns_none(self, tmp_path) -> None:+ # A repo indexed by another repowise version fails the parser+ # fingerprint gate and must degrade to the regex path, not raise.+ (tmp_path / ".repowise").mkdir()+ (tmp_path / ".repowise" / "parse_cache.pkl").write_bytes(b"not a pickle")+ assert load_repo_index(tmp_path) is None+++ROUTES_PY = '''\+from fastapi import APIRouter++router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")+++# Documents the @router.get("/fabricated") shape this module handles.+EXAMPLE = "@router.delete('/also-fabricated')"+++async def chat_message() -> None:+ """Sends a message. See @router.put("/not-a-route") for the old shape."""++[email protected]("/conversations")+async def list_conversations() -> None:+ ...+'''+++def _write_real_parse_cache(repo: Path) -> None:+ """Parse *repo*'s Python with the real ingestion parser and cache it.++ Uses ``ParseCache`` itself rather than a hand-built pickle, so the entry+ key, the ``parser_fingerprint`` gate and the content hash are all produced+ the way ingestion produces them.+ """+ from repowise.core.ingestion.models import compute_content_hash+ from repowise.core.ingestion.parse_cache import ParseCache+ from repowise.core.ingestion.parser import parse_file+ from repowise.core.ingestion.traverser import FileTraverser++ cache = ParseCache(repo / ".repowise")+ for info in FileTraverser(repo).traverse():+ if not info.path.endswith(".py"):+ continue+ source = Path(info.abs_path).read_bytes()+ cache.put(parse_file(info, source), compute_content_hash(source))+ cache.save()+++class TestAgainstARealParseCache:+ """The plumbing, end to end: real parser, real cache, real hash check.++ Everything else here builds a ``ParsedFile`` by hand, which cannot catch+ the two agreements this path depends on: that the walk hashes bytes the+ same way ingestion does, and that a cached entry survives the round trip.+ """++ @pytest.fixture+ def repo(self, tmp_path: Path) -> Path:+ (tmp_path / ".repowise").mkdir()+ (tmp_path / "routers").mkdir()+ (tmp_path / "routers" / "chat.py").write_text(ROUTES_PY, encoding="utf-8")+ _write_real_parse_cache(tmp_path)+ return tmp_path++ def test_the_cache_round_trips(self, repo: Path) -> None:+ index = load_repo_index(repo)+ assert index is not None+ assert "routers/chat.py" in index++ def test_routes_come_from_the_index(self, repo: Path) -> None:+ from repowise.core.workspace.extractors import HttpExtractor+ from repowise.core.workspace.extractors.base import make_exclude_predicate++ index = load_repo_index(repo)+ contracts = HttpExtractor().extract(+ repo, "backend", make_exclude_predicate(), None, index=index+ )+ providers = [c for c in contracts if c.role == "provider"]+ assert providers, "index path produced nothing"+ assert {c.meta[EXTRACTION_LAYER_KEY] for c in providers} == {LAYER_INDEX}++ def test_the_empty_path_route_survives_the_whole_path(self, repo: Path) -> None:+ from repowise.core.workspace.extractors import HttpExtractor+ from repowise.core.workspace.extractors.base import make_exclude_predicate++ index = load_repo_index(repo)+ ids = _ids(+ HttpExtractor().extract(+ repo, "backend", make_exclude_predicate(), None, index=index+ )+ )+ assert "http::POST::/snapshots/{param}/chat" in ids++ def test_routes_in_comments_and_docstrings_are_not_extracted(+ self, repo: Path+ ) -> None:+ from repowise.core.workspace.extractors import HttpExtractor+ from repowise.core.workspace.extractors.base import make_exclude_predicate++ index = load_repo_index(repo)+ ids = _ids(+ HttpExtractor().extract(+ repo, "backend", make_exclude_predicate(), None, index=index+ )+ )+ for fabricated in (+ "http::GET::/snapshots/{param}/chat/fabricated",+ "http::DELETE::/snapshots/{param}/chat/also-fabricated",+ "http::PUT::/snapshots/{param}/chat/not-a-route",+ ):+ assert fabricated not in ids++ def test_an_edited_file_falls_back_to_the_regex(self, repo: Path) -> None:+ from repowise.core.workspace.extractors import HttpExtractor+ from repowise.core.workspace.extractors.base import make_exclude_predicate+ from repowise.core.workspace.extractors.from_index import parsed_for++ index = load_repo_index(repo)+ # Edit the file after it was cached: the entry now describes older+ # bytes, so it must be rejected rather than trusted.+ (repo / "routers" / "chat.py").write_text(+ ROUTES_PY + '\n\[email protected]("/added")\nasync def added() -> None: ...\n',+ encoding="utf-8",+ )+ contracts = HttpExtractor().extract(+ repo, "backend", make_exclude_predicate(), None, index=index+ )+ layers = {c.meta.get(EXTRACTION_LAYER_KEY) for c in contracts if c.role == "provider"}+ assert LAYER_INDEX not in layers # stale entry refused+ assert parsed_for(index, "routers/chat.py", "deadbeef") is None+ # And the newly added route is still found, by the regex path.+ assert "http::GET::/snapshots/{param}/chat/added" in _ids(contracts)