Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(workspace): give the cross-repo hooks read access to the index (#1826)

Raghav Chamadiya363a476a7dfb
diff --git a/packages/core/src/repowise/core/workspace/contracts.py b/packages/core/src/repowise/core/workspace/contracts.py
index 6021b8539..aa3fb843a 100644
--- a/packages/core/src/repowise/core/workspace/contracts.py
+++ b/packages/core/src/repowise/core/workspace/contracts.py
@@ -30,6 +30,7 @@ from repowise.core.workspace.contract_schema import ContractSchema
if TYPE_CHECKING:
from repowise.core.workspace.extractors.service_boundary import ServiceBoundary
+ from repowise.core.workspace.repo_index import WorkspaceIndex
_log = logging.getLogger("repowise.workspace.contracts")
@@ -719,6 +720,7 @@ async def run_contract_extraction(
changed_repos: list[str],
boundaries_by_repo: dict[str, list[ServiceBoundary]] | None = None,
previous_store: ContractStore | None = None,
+ workspace_index: WorkspaceIndex | None = None,
) -> ContractStore:
"""Full contract extraction pipeline.
@@ -736,6 +738,10 @@ async def run_contract_extraction(
needs the same answer. When None (a direct call, or a test), boundaries are
detected here instead.
+ *workspace_index* is the open read side of each repo's ``wiki.db``, held by
+ :func:`run_cross_repo_hooks` across every phase. When None, or when a repo
+ has no entry in it, that repo is extracted from text alone.
+
*previous_store* is the artifact as it stands on disk, the source of any
carried-forward rows. When None nothing is reused and every repo is
extracted — the behaviour before incremental extraction existed, and what a
@@ -789,12 +795,7 @@ async def run_contract_extraction(
detect_service_boundaries,
)
from .extractors.base import iter_source_files, make_exclude_predicate
- from .extractors.from_index import (
- ALL_INDEX_SUFFIXES,
- EXTRACTION_LAYER_KEY,
- LAYER_REGEX,
- load_repo_index,
- )
+ from .extractors.from_index import EXTRACTION_LAYER_KEY, LAYER_REGEX
contract_config = ws_config.contracts
exclude = make_exclude_predicate(tuple(contract_config.exclude_globs))
@@ -901,21 +902,14 @@ async def run_contract_extraction(
wanted: frozenset[str] = frozenset()
for extractor in extractors:
wanted |= extractor.source_extensions()
- # The walk records each file's content hash as it reads it, which is
- # what lets the index path tell a matching parse from a stale one
- # without a second read.
- content_hashes: dict[str, str] = {}
files = await asyncio.to_thread(
- lambda: list(iter_source_files(repo_path, wanted, exclude, content_hashes))
+ lambda: list(iter_source_files(repo_path, wanted, exclude))
)
- # Symbols ingestion already parsed for this repo, when its parse cache
- # is usable. None means the regex dialects are the only path, which is
- # also the answer for languages with no AST tier. Only the HTTP
- # extractor consults it, so nothing is loaded when it is disabled.
- index = None
- if contract_config.detect_http:
- index = await asyncio.to_thread(load_repo_index, repo_path, ALL_INDEX_SUFFIXES)
+ # The symbols ingestion persisted for this repo. None means the regex
+ # dialects are the only path, which is also the answer for a repo that
+ # has never been indexed. Only the HTTP extractor consults it.
+ repo_index = workspace_index.get(alias) if workspace_index is not None else None
# Counters the HTTP extractor fills in as it goes. The unresolved-path
# count is the honest half of any recall figure: it says how many real
@@ -929,7 +923,7 @@ async def run_contract_extraction(
for extractor in extractors:
kwargs = (
- {"index": index, "content_hashes": content_hashes, "stats": stats}
+ {"repo_index": repo_index, "stats": stats}
if isinstance(extractor, HttpExtractor)
else {}
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index 3fb48c7e9..2dd675417 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -16,6 +16,10 @@ from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from fnmatch import fnmatch
from pathlib import Path
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from repowise.core.workspace.repo_index import RepoIndex
# Path segments that mark a test/mock tree. A route handler or topic publisher
# that exists only under one of these is a fixture, not a real service contract,
@@ -89,6 +93,9 @@ class ScanContext:
file text. ``mounts`` is the repo-wide ``router-variable -> mount-prefix``
map a provider dialect uses to recover cross-file route prefixes (see
:mod:`.http.mounts`); empty for single-file extraction.
+
+ ``index`` is the repo's read-only symbol table, when the repo has one.
+ Every dialect may ignore it and read ``content`` exactly as before.
"""
repo_alias: str
@@ -96,6 +103,7 @@ class ScanContext:
suffix: str
content: str
mounts: Mapping[str, str] = field(default_factory=dict)
+ index: RepoIndex | None = None
# One scanned source file: ``(rel_path, suffix, content)``.
@@ -107,7 +115,6 @@ def select_files(
extensions: frozenset[str],
exclude: Callable[[str], bool] | None,
files: Sequence[SourceFile] | None,
- content_hashes: dict[str, str] | None = None,
) -> list[SourceFile]:
"""Filter an already-walked *files* list, or walk *repo_path* when it is None.
@@ -117,7 +124,7 @@ def select_files(
keeps the standalone call (and every direct test) working unchanged.
"""
if files is None:
- return list(iter_source_files(repo_path, extensions, exclude, content_hashes))
+ return list(iter_source_files(repo_path, extensions, exclude))
return [f for f in files if f[1] in extensions]
@@ -125,7 +132,6 @@ def iter_source_files(
repo_root: Path,
extensions: frozenset[str],
exclude: Callable[[str], bool] | None = None,
- content_hashes: dict[str, str] | None = None,
) -> Iterator[tuple[str, str, str]]:
"""Yield ``(rel_path, suffix, content)`` for each scannable source file.
@@ -145,13 +151,9 @@ def iter_source_files(
Oversized, binary, generated, and ignored files are already excluded by the
traverser. Unreadable files are silently skipped.
- When *content_hashes* is given it is filled with ``rel_path -> sha256`` of
- each file's raw bytes — the same key the ingestion parse cache is stored
- under, so a caller can tell a matching parse from a stale one. Files are
- read as bytes and decoded here rather than via ``read_text`` so that hash
- describes the bytes on disk; the newline translation ``read_text`` would
- have applied is reproduced verbatim, leaving every dialect's input
- unchanged.
+ Files are read as bytes and decoded here rather than via ``read_text``; the
+ newline translation ``read_text`` would have applied is reproduced verbatim
+ below, leaving every dialect's input unchanged.
"""
from repowise.core.ingestion.traverser import FileTraverser
@@ -175,8 +177,4 @@ def iter_source_files(
# to bytes. Dialect regexes are written against ``\n``.
if "\r" in content:
content = content.replace("\r\n", "\n").replace("\r", "\n")
- if content_hashes is not None:
- from repowise.core.ingestion.models import compute_content_hash
-
- content_hashes[info.path] = compute_content_hash(raw)
yield info.path, suffix, content
diff --git a/packages/core/src/repowise/core/workspace/extractors/from_index.py b/packages/core/src/repowise/core/workspace/extractors/from_index.py
index 781045128..bf947c177 100644
--- a/packages/core/src/repowise/core/workspace/extractors/from_index.py
+++ b/packages/core/src/repowise/core/workspace/extractors/from_index.py
@@ -5,26 +5,27 @@ Workspace mode re-derived route tables from raw text with regexes, running
disk (``update.py`` indexes every stale repo, and only then calls
``run_cross_repo_hooks``). This module reads that parse instead.
-The source is the per-repo parse cache (``<repo>/.repowise/parse_cache.pkl``),
-which stores a :class:`~repowise.core.ingestion.models.ParsedFile` per file —
-symbols, their kinds, their line ranges, and ``Symbol.decorators`` holding the
-verbatim decorator text (``@router.post("")``). Reading a decorator node cannot
-pick up a route-shaped string from a comment or a docstring, and cannot lose an
-empty path, so two of the defect classes that motivated this module are gone by
-construction rather than by a better regex.
-
-**What the index does not carry.** A router's mount prefix comes from a call
-expression (``APIRouter(prefix="/x")``, ``include_router(r, prefix="/y")``), not
-from a decorator, and the cache stores extraction results rather than the tree.
-Prefix resolution therefore still reads the file text via :mod:`.http.mounts`.
-The route's *identity* is what moves onto the parse; its prefix is stitched on
-exactly as before.
-
-**Availability is not guaranteed.** :class:`ParseCache` gates every entry on
+The source is the repo's ``wiki.db`` symbol table, reached through
+:class:`~repowise.core.workspace.repo_index.RepoIndex`. It replaces the per-repo
+parse cache this module used to read: that cache gates every entry on
``parser_fingerprint()``, so a repo indexed by a different repowise version
-loads as empty — measured on this workspace, two of three repos. Callers must
-treat :func:`load_repo_index` returning ``None`` as "use the regex path", which
-is also the honest answer for languages with no AST tier at all.
+loads empty — measured at three of three repos on a live workspace, which left
+the whole index path inert. The database carries no such gate.
+
+**What the index carries is identity, not syntax.** A symbol row is a name, a
+kind and a line span; it holds no decorator text. A route's *identity* is the
+handler symbol that span names, and its declaration is read from the unbroken
+run of decorators immediately above the span (:func:`_decorators_above`) rather
+than from anywhere in the file. That is what stops the route-shaped prose in a
+comment — the defect class that motivated this module — from becoming a
+contract. It is a narrowing, not a proof: a docstring line beginning with ``@``
+directly above a definition would still be read as one.
+
+A router's mount prefix comes from a call expression
+(``APIRouter(prefix="/x")``, ``include_router(r, prefix="/y")``), never from a
+declaration above a handler, so prefix resolution still reads the file text via
+:mod:`.http.mounts`. The route's identity moves onto the index; its prefix is
+stitched on exactly as before.
"""
from __future__ import annotations
@@ -38,10 +39,10 @@ from .http.mounts import compose_prefix, router_prefixes
from .langs import JS_TS
if TYPE_CHECKING:
- from pathlib import Path
+ from collections.abc import Sequence
- from repowise.core.ingestion.models import ParsedFile
from repowise.core.workspace.contracts import Contract
+ from repowise.core.workspace.repo_index import IndexedSymbol
from .base import ScanContext
@@ -60,8 +61,8 @@ LAYER_REGEX = "regex"
# Decorator parsing
# ---------------------------------------------------------------------------
-# A decorator's head and its first string argument, from the verbatim text the
-# parser stores. ``@router.post("")`` -> ("router.post", ""). The argument is
+# A decorator's head and its first string argument, from the verbatim text
+# above a symbol. ``@router.post("")`` -> ("router.post", ""). The argument is
# ``*`` not ``+``: an empty path is the idiomatic collection root on a prefixed
# router and must survive to be stitched onto that prefix.
_DECORATOR_RE = re.compile(r"""^@([\w.]+)\s*\(\s*(?:(?P<q>['"])(?P<arg>[^'"]*)(?P=q))?""")
@@ -78,6 +79,14 @@ _METHOD_LITERAL_RE = re.compile(r"""['"]([A-Za-z]+)['"]""")
_ROUTE_HEADS = frozenset({"route"})
+# How far above a definition the decorator walk may reach. Bounds the scan on a
+# file whose indexed spans no longer line up with its text.
+_DECORATOR_LOOKBACK = 40
+
+# Symbol kinds a route decorator can sit above. Classes are included because
+# Flask-style pluggable views decorate one, which the retired path also caught.
+_HANDLER_KINDS = frozenset({"function", "method", "class"})
+
# The languages anything here reads decorators for. Widening this means
# widening :func:`extract_http_providers` to match.
INDEX_SUFFIXES = frozenset({".py"})
@@ -86,14 +95,10 @@ INDEX_SUFFIXES = frozenset({".py"})
# languages HTTP clients are written in, so the wrapper-confirmation pass can
# bound "this symbol's own body" to a parsed extent instead of guessing it by
# counting braces. Kept separate from INDEX_SUFFIXES because the two passes read
-# different things off the same parse: providers read ``Symbol.decorators``,
-# consumers read line ranges.
+# different things off the same rows: providers read the declaration above a
+# span, consumers read the span itself.
CONSUMER_INDEX_SUFFIXES = frozenset(JS_TS)
-# What :func:`load_repo_index` deserializes for both passes together. One load
-# per repo feeds both; asking for them separately would read the cache twice.
-ALL_INDEX_SUFFIXES = INDEX_SUFFIXES | CONSUMER_INDEX_SUFFIXES
-
def _decorator_head_and_arg(decorator: str) -> tuple[str, str | None]:
"""Split verbatim decorator text into its dotted head and first string arg.
@@ -133,140 +138,147 @@ def _routes_in_decorator(decorator: str) -> list[tuple[str, str, str]]:
return []
+def _decorators_above(lines: list[str], start_line: int) -> list[str]:
+ """Verbatim decorator text stacked directly above a definition.
+
+ ``start_line`` is 1-indexed and names the ``def`` line: the parser takes a
+ symbol's span from the definition node, not from Python's enclosing
+ ``decorated_definition``, so any decorators sit above it.
+
+ The run above the definition must be decorators and nothing else. Stopping
+ at the first ordinary line — not merely at a blank one — is what keeps a
+ decorated *class* from lending its route to the first method under it, and
+ a neighbouring handler from lending its route to the next.
+ """
+ out: list[str] = []
+ pending: list[str] = []
+ depth = 0 # unclosed parens below, so a walk inside a multi-line decorator
+ i = start_line - 2 # 0-indexed line directly above the definition
+ floor = max(0, i - _DECORATOR_LOOKBACK)
+ while i >= floor:
+ line = lines[i]
+ stripped = line.strip()
+ if not stripped or stripped.startswith("#"):
+ break
+ # A line belongs to the stack only as a decorator head or as the
+ # argument list of one still open below it. Anything else is ordinary
+ # code and ends the run.
+ opened = depth + line.count(")") - line.count("(")
+ if opened <= 0 and not stripped.startswith("@"):
+ break
+ depth = opened
+ pending.append(line)
+ if depth == 0:
+ out.append("\n".join(reversed(pending)))
+ pending = []
+ i -= 1
+ return out
+
+
# ---------------------------------------------------------------------------
-# Reading the parse cache
+# Reading the index
# ---------------------------------------------------------------------------
-def load_repo_index(
- repo_root: Path,
- suffixes: frozenset[str] = INDEX_SUFFIXES,
-) -> dict[str, ParsedFile] | None:
- """Return ``rel_path -> ParsedFile`` from the repo's parse cache.
+def symbols_for_content(ctx: ScanContext, line_count: int) -> list[IndexedSymbol]:
+ """This file's indexed symbols, but only if they still describe *ctx.content*.
- Only entries whose path ends in one of *suffixes* are deserialized. The
- cache holds every file the repo indexed (27 MB of pickled ``ParsedFile``
- graphs on this repo) and only the languages read here are ever consulted,
- so unpickling the rest would be pure memory cost — multiplied by the repos
- running concurrently.
+ The index carries no per-file content hash, so currency is proven against
+ the text itself. Two checks, because they catch opposite edits: a span
+ reaching past the end of the file was parsed from a longer version of it,
+ and a definition the index does not place is code added since it was
+ written. The second matters most — the index pass supersedes the text
+ dialect for a file it produced routes for, so an unseen definition would be
+ a route silently dropped rather than one merely not upgraded.
- ``None`` when the cache is absent or unusable — a repo indexed by another
- repowise version fails the ``parser_fingerprint`` gate and loads empty,
- which is indistinguishable from "never indexed" and gets the same answer:
- fall back to the regex path.
+ Neither catches an edit that shifts lines without changing the definition
+ set. A shifted span reads no decorator and yields no route, so the cost is
+ a miss, not a fabrication.
"""
- import pickle
-
- from repowise.core.ingestion.parse_cache import ParseCache
-
- cache_dir = repo_root / ".repowise"
- if not (cache_dir / "parse_cache.pkl").is_file():
- return None
-
- cache = ParseCache(cache_dir)
- try:
- cache.load()
- except Exception:
- _log.debug("Parse cache unreadable for %s", repo_root, exc_info=True)
- return None
-
- # ParseCache exposes lookup by (FileInfo, hash) but not enumeration, and
- # this needs the whole map. Reaching for the private dict is the coupling
- # that buys that; log loudly rather than go dark if it ever moves.
- entries = getattr(cache, "_entries", None)
- if entries is None:
- _log.warning(
- "ParseCache no longer exposes _entries; index-backed contract "
- "extraction is disabled and everything falls back to regex"
- )
- return None
- if not entries:
- _log.info(
- "Parse cache for %s loaded no entries (version mismatch or empty); "
- "contract extraction falls back to the regex path",
- repo_root,
- )
- return None
+ if ctx.index is None:
+ return []
+ symbols = ctx.index.symbols_for_file(ctx.rel_path)
+ if any(s.end_line > line_count for s in symbols):
+ _log.debug("Indexed spans for %s overrun the file; using the regex path", ctx.rel_path)
+ return []
+ if ctx.suffix in INDEX_SUFFIXES and not _definitions_are_indexed(ctx.content, symbols):
+ _log.debug("Indexed symbols for %s miss a definition; using the regex path", ctx.rel_path)
+ return []
+ return symbols
- out: dict[str, ParsedFile] = {}
- for (rel_path, content_hash), blob in entries.items():
- if not rel_path.endswith(tuple(suffixes)):
- continue
- try:
- parsed = pickle.loads(blob)
- except Exception: # a single corrupt entry must not lose the repo
- _log.debug("Skipping unpicklable parse-cache entry %s", rel_path)
- continue
- # The cache key carries the hash the entry was parsed from; keep it on
- # the object so :func:`parsed_for` can reject a stale entry.
- parsed.content_hash = content_hash
- out[rel_path] = parsed
- return out or None
-
-
-def parsed_for(
- index: dict[str, ParsedFile] | None,
- rel_path: str,
- content_hash: str | None,
-) -> ParsedFile | None:
- """The parse for *rel_path*, but only if it describes the bytes on disk.
-
- Parse-cache entries are keyed by content hash. An entry whose hash does not
- match the file just walked describes an older version of it, and trusting
- that would report routes which no longer exist (or miss ones that now do).
- Without a hash to check against there is nothing to trust, so the caller
- gets ``None`` and stays on the regex path.
+
+# A Python definition line, wherever it is indented.
+_DEF_LINE_RE = re.compile(r"^[ \t]*(?:async[ \t]+)?def[ \t]", re.MULTILINE)
+
+
+def _definitions_are_indexed(content: str, symbols: Sequence[IndexedSymbol]) -> bool:
+ """True when every ``def`` in *content* is one the index knows about.
+
+ A definition is known if a symbol starts on that line, or if it is nested
+ inside a symbol's span — the parser records a closure as part of its
+ enclosing function rather than on its own row.
"""
- if index is None or content_hash is None:
- return None
- parsed = index.get(rel_path)
- if parsed is None:
- return None
- if parsed.content_hash != content_hash:
- _log.debug("Parse-cache entry for %s is stale; using the regex path", rel_path)
- return None
- return parsed
+ starts = {s.start_line for s in symbols}
+ spans = [(s.start_line, s.end_line) for s in symbols]
+ for m in _DEF_LINE_RE.finditer(content):
+ line = content.count("\n", 0, m.start()) + 1
+ if line in starts:
+ continue
+ if not any(start < line <= end for start, end in spans):
+ return False
+ return True
# ---------------------------------------------------------------------------
# Provider extraction
# ---------------------------------------------------------------------------
+
def extract_http_providers(
ctx: ScanContext,
- parsed: ParsedFile,
+ symbols: Sequence[IndexedSymbol],
+ lines: list[str],
) -> list[Contract]:
- """HTTP provider contracts from *parsed*'s decorated symbols.
+ """HTTP provider contracts from the decorators above *symbols*.
- Routes come from ``Symbol.decorators``; the mount prefix still comes from
- the file text (see the module docstring), so ``ctx.content`` is required.
+ *lines* is ``ctx.content`` already split; the mount prefix still comes from
+ the whole file text (see the module docstring), so ``ctx.content`` is read
+ as well.
"""
+ # Routes before prefixes: ``router_prefixes`` scans the whole file, so it
+ # is worth paying for only once a route exists to stitch a prefix onto.
+ routes = [
+ (symbol, decorator, route)
+ for symbol in symbols
+ if symbol.kind in _HANDLER_KINDS
+ for decorator in _decorators_above(lines, symbol.start_line)
+ for route in _routes_in_decorator(decorator)
+ ]
+ if not routes:
+ return []
+
prefixes = router_prefixes(ctx.content, "APIRouter|FastAPI|Flask|Blueprint")
known = set(prefixes) | {"app", "router", "bp", "blueprint"}
out: list[Contract] = []
seen: set[tuple[str, str]] = set()
- for symbol in parsed.symbols:
- # ``Symbol.decorators`` can repeat an entry (the parser appends per
- # matching capture), so dedupe per symbol before building contracts.
- for decorator in dict.fromkeys(symbol.decorators):
- for var, method, raw_path in _routes_in_decorator(decorator):
- if var not in known:
- continue
- path = compose_prefix(prefixes.get(var, ""), raw_path)
- path = compose_prefix(ctx.mounts.get(var, ""), path)
- key = (method, path)
- if key in seen:
- continue
- seen.add(key)
- framework = "flask" if _is_route_head(decorator) else "fastapi"
- contract = build_provider_contract(
- ctx, method=method, path_raw=path, framework=framework
- )
- if contract is not None:
- contract.meta[EXTRACTION_LAYER_KEY] = LAYER_INDEX
- contract.meta["handler"] = symbol.name
- out.append(contract)
+ for symbol, decorator, (var, method, raw_path) in routes:
+ if var not in known:
+ continue
+ path = compose_prefix(prefixes.get(var, ""), raw_path)
+ path = compose_prefix(ctx.mounts.get(var, ""), path)
+ key = (method, path)
+ if key in seen:
+ continue
+ seen.add(key)
+ framework = "flask" if _is_route_head(decorator) else "fastapi"
+ contract = build_provider_contract(
+ ctx, method=method, path_raw=path, framework=framework
+ )
+ if contract is not None:
+ contract.meta[EXTRACTION_LAYER_KEY] = LAYER_INDEX
+ contract.meta["handler"] = symbol.name
+ out.append(contract)
return out
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
index 5f2a57173..111c812b2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
from pathlib import Path
from repowise.core.workspace.contracts import Contract
+ from repowise.core.workspace.repo_index import RepoIndex
from ..base import SourceFile
@@ -96,8 +97,7 @@ class HttpExtractor:
repo_alias: str = "",
exclude: Callable[[str], bool] | None = None,
files: Sequence[SourceFile] | None = None,
- index: dict[str, object] | None = None,
- content_hashes: dict[str, str] | None = None,
+ repo_index: RepoIndex | None = None,
stats: dict[str, int] | None = None,
) -> list[Contract]:
"""Scan all source files in *repo_path* and return Contract instances.
@@ -108,12 +108,10 @@ class HttpExtractor:
*files* is an already-walked ``(rel_path, suffix, content)`` list from
the orchestrator's single traversal; None walks the repo directly.
- *index* maps ``rel_path -> ParsedFile`` from the repo's parse cache. A
- file present there gets its Python routes from the parsed decorators
- rather than from the FastAPI text regex — a decorator node cannot carry
- a route from a comment, and cannot lose an empty path. *content_hashes*
- is the walk's ``rel_path -> sha256`` map, which is what proves a cached
- parse describes the file as it is now; without it the index is unused.
+ *repo_index* is the repo's read-only symbol table. A file it has
+ symbols for gets its Python routes from the declarations above those
+ symbols rather than from the FastAPI text regex, which is what stops a
+ route in a comment becoming a contract and an empty path being lost.
*stats* is an optional out-dict of counters. ``http_consumer_unresolved``
counts calls to a *confirmed* HTTP wrapper whose path argument could not
@@ -121,35 +119,34 @@ class HttpExtractor:
cannot be named. They are counted rather than dropped, so a recall
figure built from these contracts states its own denominator.
"""
- own_hashes: dict[str, str] | None = None
- if files is None and index is not None and content_hashes is None:
- own_hashes = content_hashes = {}
- scanned = select_files(
- repo_path, self.source_extensions(), exclude, files, own_hashes
- )
+ scanned = select_files(repo_path, self.source_extensions(), exclude, files)
mounts = self._collect_mounts(scanned)
- from ..from_index import CONSUMER_INDEX_SUFFIXES, extract_http_providers, parsed_for
+ from ..from_index import (
+ CONSUMER_INDEX_SUFFIXES,
+ extract_http_providers,
+ symbols_for_content,
+ )
from .index_clients import extract_consumers
contracts: list[Contract] = []
for rel_path, suffix, content in scanned:
- ctx = ScanContext(repo_alias, rel_path, suffix, content, mounts)
- # Only a parse that matches the bytes just walked supersedes the
- # regex; a stale or missing entry leaves the dialect in charge.
- parsed = (
- parsed_for(index, rel_path, (content_hashes or {}).get(rel_path))
- if suffix in PYTHON or suffix in CONSUMER_INDEX_SUFFIXES
- else None
+ ctx = ScanContext(repo_alias, rel_path, suffix, content, mounts, repo_index)
+ indexed = suffix in PYTHON or suffix in CONSUMER_INDEX_SUFFIXES
+ # Only spans that can still describe this file's text supersede the
+ # regex; a missing or overrun entry leaves the dialect in charge.
+ symbols = (
+ symbols_for_content(ctx, content.count("\n") + 1) if indexed else []
)
+ lines = content.split("\n") if symbols else []
# Run the index pass first: it only supersedes the text dialect for
- # a file it actually produced routes for. A parse that yielded no
+ # a file it actually produced routes for. A file that yielded no
# decorated symbols (a grammar the parser stumbled on, a route
# shape the queries do not capture) must not silently delete the
# routes the regex can still see in the file's text.
from_parse = (
- extract_http_providers(ctx, parsed)
- if parsed is not None and suffix in PYTHON
+ extract_http_providers(ctx, symbols, lines)
+ if symbols and suffix in PYTHON
else []
)
contracts.extend(from_parse)
@@ -157,8 +154,8 @@ class HttpExtractor:
# Consumers, under the same per-file supersede rule: calls at the
# sites of wrappers confirmed to reach an HTTP sink.
consumers_from_parse: list[Contract] = []
- if parsed is not None and suffix in CONSUMER_INDEX_SUFFIXES:
- consumers_from_parse, unresolved = extract_consumers(ctx, parsed)
+ if symbols and suffix in CONSUMER_INDEX_SUFFIXES:
+ consumers_from_parse, unresolved = extract_consumers(ctx, symbols)
contracts.extend(consumers_from_parse)
if stats is not None and unresolved:
stats["http_consumer_unresolved"] = (
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/index_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/index_clients.py
index 47b1b5de0..06ab8d8a3 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/index_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/index_clients.py
@@ -28,11 +28,14 @@ from .wrappers import (
confirm_wrappers,
mask_source,
sink_call_names,
+ sink_patterns,
)
if TYPE_CHECKING:
- from repowise.core.ingestion.models import ParsedFile
+ from collections.abc import Sequence
+
from repowise.core.workspace.contracts import Contract
+ from repowise.core.workspace.repo_index import IndexedSymbol
from ..base import ScanContext
@@ -160,7 +163,7 @@ def _literal_url(arg: str) -> str | None:
return inner if _CONCRETE_URL_RE.match(inner) else None
-def _declaration_sites(parsed: ParsedFile) -> set[tuple[int, str]]:
+def _declaration_sites(symbols: Sequence[IndexedSymbol]) -> set[tuple[int, str]]:
"""``(line, name)`` for each callable's own declaration.
A method declaration is syntactically a call — ``fetch<T>(path: string)`` —
@@ -170,16 +173,18 @@ def _declaration_sites(parsed: ParsedFile) -> set[tuple[int, str]]:
"""
return {
(s.start_line, s.name)
- for s in parsed.symbols
+ for s in symbols
if s.kind in {"function", "method", "constructor"}
}
-def _confirmed_ranges(parsed: ParsedFile, confirmed: set[str]) -> list[tuple[int, int]]:
+def _confirmed_ranges(
+ symbols: Sequence[IndexedSymbol], confirmed: set[str]
+) -> list[tuple[int, int]]:
"""Line extents of the symbols confirmed to reach a sink."""
return [
(s.start_line, s.end_line)
- for s in parsed.symbols
+ for s in symbols
if s.name in confirmed and s.kind in {"function", "method", "constructor"}
]
@@ -190,7 +195,7 @@ def _within(ranges: list[tuple[int, int]], line: int) -> bool:
def extract_consumers(
ctx: ScanContext,
- parsed: ParsedFile,
+ symbols: Sequence[IndexedSymbol],
budget: int = DEFAULT_HOP_BUDGET,
) -> tuple[list[Contract], int]:
"""Consumer contracts at confirmed wrapper call sites, plus an unresolved count.
@@ -201,7 +206,14 @@ def extract_consumers(
endpoint calls this layer cannot name; reporting the number is what keeps
the recall figure honest rather than flattering.
"""
- confirmed = confirm_wrappers(parsed, ctx.content, ctx.suffix, budget)
+ # No sink anywhere in the raw text means no wrapper can be confirmed and no
+ # receiverless sink call can exist, so the two O(n) masking passes below
+ # would run to find nothing. Masking only removes matches, so a miss here is
+ # a miss there. Most files in a repo take this exit.
+ if not any(p.search(ctx.content) for p in sink_patterns(ctx.suffix)):
+ return [], 0
+
+ confirmed = confirm_wrappers(symbols, ctx.content, ctx.suffix, budget)
sinks = sink_call_names(ctx.suffix)
if not confirmed and not sinks:
return [], 0
@@ -212,7 +224,7 @@ def extract_consumers(
# argument scanner. Masking preserves offsets and length, so slices taken
# against this text are the real argument text.
content = mask_source(ctx.content, ctx.suffix)
- declarations = _declaration_sites(parsed)
+ declarations = _declaration_sites(symbols)
# Pass 1: every call site of a confirmed wrapper, split into resolved
# (concrete literal) and unresolved. The set of wrappers proven to take a
@@ -227,7 +239,7 @@ def extract_consumers(
# proved, and a call that never parsed proved nothing either way. Gating
# these would let a scanner failure zero itself out.
parse_failures = 0
- confirmed_ranges = _confirmed_ranges(parsed, confirmed)
+ confirmed_ranges = _confirmed_ranges(symbols, confirmed)
for m in _CALL_RE.finditer(content):
name = m.group("name")
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/wrappers.py b/packages/core/src/repowise/core/workspace/extractors/http/wrappers.py
index 107c256e8..2cf65175c 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/wrappers.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/wrappers.py
@@ -39,7 +39,9 @@ import re
from typing import TYPE_CHECKING
if TYPE_CHECKING:
- from repowise.core.ingestion.models import ParsedFile, Symbol
+ from collections.abc import Sequence
+
+ from repowise.core.workspace.repo_index import IndexedSymbol
# Hop budget for wrapper confirmation. 2 is not arbitrary: the real chain in
# ``frontend/src/lib/api/client.ts`` is
@@ -239,7 +241,7 @@ def sink_call_names(suffix: str) -> frozenset[str]:
return _SINK_CALL_NAMES_BY_SUFFIX.get(suffix.lower(), frozenset())
-def symbol_body(lines: list[str], symbol: Symbol) -> str:
+def symbol_body(lines: list[str], symbol: IndexedSymbol) -> str:
"""The source text of *symbol*'s body, excluding its own declaration.
The declaration must be excluded because a symbol *named* like a sink
@@ -267,7 +269,7 @@ def symbol_body(lines: list[str], symbol: Symbol) -> str:
return "\n".join([head, *rest]) if head else "\n".join(rest)
-def _callable_symbols(parsed: ParsedFile) -> dict[str, list[Symbol]]:
+def _callable_symbols(symbols: Sequence[IndexedSymbol]) -> dict[str, list[IndexedSymbol]]:
"""Callable symbols in this file, indexed by bare name.
A name can be declared more than once (two classes with a ``get`` method);
@@ -276,15 +278,15 @@ def _callable_symbols(parsed: ParsedFile) -> dict[str, list[Symbol]]:
the alternative — dropping ambiguous names — would lose real calls in every
file with two similarly-shaped classes.
"""
- out: dict[str, list[Symbol]] = {}
- for sym in parsed.symbols:
+ out: dict[str, list[IndexedSymbol]] = {}
+ for sym in symbols:
if sym.kind in _CALLABLE_KINDS:
out.setdefault(sym.name, []).append(sym)
return out
def confirm_wrappers(
- parsed: ParsedFile,
+ symbols: Sequence[IndexedSymbol],
content: str,
suffix: str,
budget: int = DEFAULT_HOP_BUDGET,
@@ -305,7 +307,7 @@ def confirm_wrappers(
# confirming a wrapper on it would reintroduce name-guessing by another
# route. Masking preserves offsets, so symbol line ranges still apply.
lines = mask_source(content, suffix, strings=True).split("\n")
- by_name = _callable_symbols(parsed)
+ by_name = _callable_symbols(symbols)
if not by_name:
return set()
@@ -315,26 +317,26 @@ def confirm_wrappers(
failed_at: dict[str, int] = {}
confirmed: set[str] = set()
- def reaches_sink(sym: Symbol, hops: int, stack: frozenset[str]) -> bool:
- if sym.id in stack: # a cycle cannot introduce a new sink
+ def reaches_sink(sym: IndexedSymbol, hops: int, stack: frozenset[str]) -> bool:
+ if sym.symbol_id in stack: # a cycle cannot introduce a new sink
return False
- if failed_at.get(sym.id, -1) >= hops:
+ if failed_at.get(sym.symbol_id, -1) >= hops:
return False
body = symbol_body(lines, sym)
if not body:
- failed_at[sym.id] = max(failed_at.get(sym.id, -1), hops)
+ failed_at[sym.symbol_id] = max(failed_at.get(sym.symbol_id, -1), hops)
return False
if any(p.search(body) for p in patterns):
return True
if hops <= 0:
- failed_at[sym.id] = max(failed_at.get(sym.id, -1), hops)
+ failed_at[sym.symbol_id] = max(failed_at.get(sym.symbol_id, -1), hops)
return False
- inner = stack | {sym.id}
+ inner = stack | {sym.symbol_id}
for m in _LOCAL_CALL_RE.finditer(body):
for callee in by_name.get(m.group(1), ()):
- if callee.id != sym.id and reaches_sink(callee, hops - 1, inner):
+ if callee.symbol_id != sym.symbol_id and reaches_sink(callee, hops - 1, inner):
return True
- failed_at[sym.id] = max(failed_at.get(sym.id, -1), hops)
+ failed_at[sym.symbol_id] = max(failed_at.get(sym.symbol_id, -1), hops)
return False
for name, syms in by_name.items():
diff --git a/packages/core/src/repowise/core/workspace/registry.py b/packages/core/src/repowise/core/workspace/registry.py
index a613beea9..a023f44cd 100644
--- a/packages/core/src/repowise/core/workspace/registry.py
+++ b/packages/core/src/repowise/core/workspace/registry.py
@@ -39,6 +39,26 @@ class RepoContext:
_engine: Any = field(default=None, repr=False) # AsyncEngine, for dispose
+def repo_db_path(repo_path: Path) -> Path:
+ """Where a repo's index database lives."""
+ return repo_path / ".repowise" / "wiki.db"
+
+
+async def open_repo_db(repo_path: Path) -> tuple[Any, async_sessionmaker[AsyncSession]]:
+ """Open a repo's ``wiki.db``, returning ``(engine, session_factory)``.
+
+ SQLite creates the file if it is absent, so check :func:`repo_db_path`
+ first when "never indexed" needs a different answer than "empty". The
+ caller owns the engine and must dispose it. Shared with :mod:`.repo_index`,
+ which needs the same connection without the search and vector stores.
+ """
+ from repowise.core.persistence.database import create_engine, get_db_url, init_db
+
+ engine = create_engine(get_db_url(f"sqlite:///{repo_db_path(repo_path).as_posix()}"))
+ await init_db(engine)
+ return engine, async_sessionmaker(engine, expire_on_commit=False, class_=AsyncSession)
+
+
# ---------------------------------------------------------------------------
# RepoRegistry — lazy loading + LRU eviction
# ---------------------------------------------------------------------------
@@ -150,37 +170,15 @@ class RepoRegistry:
raise ValueError(f"No repo with alias '{alias}' in workspace config")
repo_path = (self._workspace_root / entry.path).resolve()
- db_path = repo_path / ".repowise" / "wiki.db"
-
- if not db_path.exists():
- _log.warning("No wiki.db for repo '%s' at %s", alias, db_path)
- from sqlalchemy.ext.asyncio import (
- AsyncSession as _AsyncSession,
- )
- from sqlalchemy.ext.asyncio import (
- async_sessionmaker as _async_sessionmaker,
- )
-
- from repowise.core.persistence.database import (
- create_engine,
- get_db_url,
- init_db,
- )
from repowise.core.persistence.search import FullTextSearch
from repowise.core.persistence.vector_store import InMemoryVectorStore
from repowise.core.providers.embedding.base import KeylessEmbedder
- db_url = get_db_url(f"sqlite:///{db_path.as_posix()}")
-
- engine = create_engine(db_url)
- await init_db(engine)
-
- session_factory = _async_sessionmaker(
- engine,
- expire_on_commit=False,
- class_=_AsyncSession,
- )
+ db_path = repo_db_path(repo_path)
+ if not db_path.exists():
+ _log.warning("No wiki.db for repo '%s' at %s", alias, db_path)
+ engine, session_factory = await open_repo_db(repo_path)
fts = FullTextSearch(engine)
await fts.ensure_index()
diff --git a/packages/core/src/repowise/core/workspace/repo_index.py b/packages/core/src/repowise/core/workspace/repo_index.py
new file mode 100644
index 000000000..7afd5e122
--- /dev/null
+++ b/packages/core/src/repowise/core/workspace/repo_index.py
@@ -0,0 +1,234 @@
+"""Read-only index access for the cross-repo hooks.
+
+The hooks run immediately after ingestion writes each repo's
+``.repowise/wiki.db`` and, until now, re-derived from raw text what that
+database already holds. This is the read side: one connection per repo, opened
+once and held across every hook phase.
+
+Read-only in intent, not in mechanism — the shared :func:`.registry.open_repo_db`
+runs the same schema check the MCP read path does.
+
+Line ranges here are the *index's*, so a file edited after its repo was
+indexed can hand back a span that no longer describes the bytes on disk. A
+caller slicing source with these numbers must bound them against the content
+it holds — see :func:`.extractors.from_index.symbols_for_content`.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+ from sqlalchemy.ext.asyncio import AsyncSession
+
+_log = logging.getLogger("repowise.workspace.repo_index")
+
+# Node-id prefix ingestion gives an import target it could not resolve inside
+# the repo — a third-party package or a sibling workspace repo.
+EXTERNAL_PREFIX = "external:"
+
+
+@dataclass(frozen=True)
+class IndexedSymbol:
+ """One ``wiki_symbols`` row: what ingestion parsed, without the body."""
+
+ symbol_id: str # "<rel_path>::<name>", the ingestion Symbol.id
+ name: str
+ qualified_name: str
+ kind: str
+ signature: str
+ file_path: str
+ start_line: int # 1-indexed, inclusive
+ end_line: int # 1-indexed, inclusive
+ visibility: str
+
+
+@dataclass(frozen=True)
+class ExternalImport:
+ """An ``imports`` edge from a repo file to an unresolved external target."""
+
+ source_file: str
+ external_name: str # the target with its ``external:`` prefix stripped
+ imported_names: tuple[str, ...]
+
+
+class RepoIndex:
+ """Read-only accessor over one repo's ``wiki.db``.
+
+ Built by :func:`open_workspace_index`; the caller owns :meth:`close`. Every
+ accessor is a pure in-memory read of what :meth:`_load` fetched, so a
+ dialect may call one from the worker thread it runs in.
+ """
+
+ def __init__(self, alias: str, repo_path: Path, session: AsyncSession, engine: Any) -> None:
+ self.alias = alias
+ self.repo_path = repo_path
+ self._session = session
+ self._engine = engine
+ self._by_file: dict[str, list[IndexedSymbol]] = {}
+ self._externals: list[ExternalImport] = []
+
+ # -- Loading -----------------------------------------------------------
+
+ async def _load(self, repo_id: str) -> None:
+ from sqlalchemy import select
+
+ from repowise.core.persistence.models import GraphEdge, WikiSymbol
+
+ rows = await self._session.execute(
+ select(
+ WikiSymbol.symbol_id,
+ WikiSymbol.name,
+ WikiSymbol.qualified_name,
+ WikiSymbol.kind,
+ WikiSymbol.signature,
+ WikiSymbol.file_path,
+ WikiSymbol.start_line,
+ WikiSymbol.end_line,
+ WikiSymbol.visibility,
+ ).where(WikiSymbol.repository_id == repo_id)
+ )
+ for row in rows:
+ self._by_file.setdefault(row.file_path, []).append(IndexedSymbol(*row))
+ # Outermost first, so the first span containing a line is the class and
+ # the last is the method.
+ for symbols in self._by_file.values():
+ symbols.sort(key=lambda s: (s.start_line, -s.end_line))
+
+ edges = await self._session.execute(
+ select(
+ GraphEdge.source_node_id,
+ GraphEdge.target_node_id,
+ GraphEdge.imported_names_json,
+ ).where(
+ GraphEdge.repository_id == repo_id,
+ GraphEdge.edge_type == "imports",
+ GraphEdge.target_node_id.startswith(EXTERNAL_PREFIX),
+ )
+ )
+ for source, target, names_json in edges:
+ try:
+ names = json.loads(names_json or "[]")
+ except ValueError:
+ names = []
+ if not isinstance(names, list):
+ names = []
+ self._externals.append(
+ ExternalImport(
+ source_file=source,
+ external_name=target[len(EXTERNAL_PREFIX) :],
+ imported_names=tuple(n for n in names if isinstance(n, str)),
+ )
+ )
+
+ # -- Public API --------------------------------------------------------
+
+ def symbols_for_file(self, rel_path: str) -> list[IndexedSymbol]:
+ """Symbols declared in *rel_path* (POSIX, repo-relative), outermost first."""
+ return self._by_file.get(rel_path, [])
+
+ def symbol_at(self, rel_path: str, line: int) -> IndexedSymbol | None:
+ """The innermost symbol whose span contains *line*, or None."""
+ best: IndexedSymbol | None = None
+ for sym in self._by_file.get(rel_path, ()):
+ if sym.start_line <= line <= sym.end_line and (
+ best is None or sym.start_line >= best.start_line
+ ):
+ best = sym
+ return best
+
+ def external_import_edges(self) -> list[ExternalImport]:
+ """Every ``imports`` edge leaving the repo, with the names it consumes."""
+ return self._externals
+
+ def public_symbols(self) -> list[IndexedSymbol]:
+ """Symbols ingestion marked public, across the whole repo."""
+ return [s for syms in self._by_file.values() for s in syms if s.visibility == "public"]
+
+ async def close(self) -> None:
+ await self._session.close()
+ await self._engine.dispose()
+
+
+class WorkspaceIndex:
+ """The open :class:`RepoIndex` for each workspace repo that has one."""
+
+ def __init__(self, repos: dict[str, RepoIndex]) -> None:
+ self._repos = repos
+
+ def get(self, alias: str) -> RepoIndex | None:
+ return self._repos.get(alias)
+
+ async def close(self) -> None:
+ for repo in self._repos.values():
+ try:
+ await repo.close()
+ except Exception:
+ _log.warning("Error closing index for '%s'", repo.alias, exc_info=True)
+ self._repos.clear()
+
+
+async def open_repo_index(alias: str, repo_path: Path) -> RepoIndex | None:
+ """Open *repo_path*'s ``wiki.db`` read side, or None when it has no index."""
+ from sqlalchemy import select
+
+ from repowise.core.persistence.crud.repository import get_repository_by_path
+ from repowise.core.persistence.models import Repository
+
+ from .registry import open_repo_db, repo_db_path
+
+ if not repo_db_path(repo_path).is_file():
+ return None
+ engine, session_factory = await open_repo_db(repo_path)
+
+ session = session_factory()
+ ok = False
+ try:
+ # A wiki.db normally holds one repository row; prefer the one recorded
+ # at this checkout when it holds more, since test runs and moved
+ # checkouts leave others behind. Newest first so the fallback is a
+ # decision rather than whatever order the scan returns.
+ repo = await get_repository_by_path(session, str(repo_path))
+ if repo is None:
+ repo = (
+ await session.execute(
+ select(Repository).order_by(Repository.updated_at.desc()).limit(1)
+ )
+ ).scalar_one_or_none()
+ if repo is None:
+ raise LookupError(f"No repository row in {repo_path}")
+ index = RepoIndex(alias, repo_path, session, engine)
+ await index._load(repo.id)
+ ok = True
+ finally:
+ # BaseException too: a cancellation landing in _load would otherwise
+ # leak the engine and its connection.
+ if not ok:
+ await session.close()
+ await engine.dispose()
+ return index
+
+
+async def open_workspace_index(ws_config: Any, workspace_root: Path) -> WorkspaceIndex:
+ """Open every indexed repo in the workspace. A repo that fails is skipped."""
+ entries = list(ws_config.repos)
+ opened = await asyncio.gather(
+ *(
+ open_repo_index(e.alias, (workspace_root / e.path).resolve())
+ for e in entries
+ ),
+ return_exceptions=True,
+ )
+ repos: dict[str, RepoIndex] = {}
+ for entry, index in zip(entries, opened, strict=True):
+ if isinstance(index, BaseException):
+ _log.warning("Could not open the index for '%s'", entry.alias, exc_info=index)
+ elif index is not None:
+ repos[entry.alias] = index
+ return WorkspaceIndex(repos)
diff --git a/packages/core/src/repowise/core/workspace/update.py b/packages/core/src/repowise/core/workspace/update.py
index 6c07afb1f..2320946ed 100644
--- a/packages/core/src/repowise/core/workspace/update.py
+++ b/packages/core/src/repowise/core/workspace/update.py
@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from repowise.core.workspace.extractors.service_boundary import ServiceBoundary
+ from repowise.core.workspace.repo_index import WorkspaceIndex
# Per-repo update lock — the shared single-flight guard (one implementation
# for the CLI update command and this workspace updater, which used to carry
@@ -944,6 +945,36 @@ async def run_cross_repo_hooks(
if len(ws_config.repos) < 2:
return
+ from .repo_index import WorkspaceIndex, open_workspace_index
+
+ timings = PhaseTimingRecorder()
+
+ # The read side of every repo's index, opened once here and closed in the
+ # finally below, so one connection per repo serves all five phases.
+ # Indexing has just written these databases; without this the phases below
+ # re-derive from raw text what they already hold. Only the HTTP extractor
+ # reads it today, so nothing is opened when that is off.
+ workspace_index = (
+ await open_workspace_index(ws_config, workspace_root)
+ if ws_config.contracts.detect_http
+ else WorkspaceIndex({})
+ )
+ try:
+ await _run_phases(
+ ws_config, workspace_root, changed_repos, timings, workspace_index
+ )
+ finally:
+ await workspace_index.close()
+
+
+async def _run_phases(
+ ws_config: WorkspaceConfig,
+ workspace_root: Path,
+ changed_repos: list[str],
+ timings: PhaseTimingRecorder,
+ workspace_index: WorkspaceIndex,
+) -> None:
+ """The five cross-repo phases, over an already-open workspace index."""
from .breaking_change import run_breaking_change_detection
from .conformance import run_conformance_check
from .contracts import ContractStore, load_contract_store, run_contract_extraction
@@ -954,8 +985,6 @@ async def run_cross_repo_hooks(
run_system_graph_build,
)
- timings = PhaseTimingRecorder()
-
# Service boundaries, detected once for the whole workspace. Contract
# extraction and the system-graph build both need them and each used to walk
# every repo for its own copy.
@@ -983,7 +1012,7 @@ async def run_cross_repo_hooks(
previous_store = load_contract_store(workspace_root) or ContractStore()
# Phases 1 and 2 are independent: they read disjoint inputs (git history and
- # manifests vs source files and the parse cache), write different artifacts
+ # manifests vs source files and the symbol index), write different artifacts
# (cross_repo_edges.json vs contracts.json), and share no mutable state —
# boundaries_by_repo is computed above and passed in read-only. Both push
# their blocking work through asyncio.to_thread, so gather genuinely
@@ -1008,6 +1037,7 @@ async def run_cross_repo_hooks(
changed_repos,
boundaries_by_repo or None,
previous_store,
+ workspace_index,
),
),
return_exceptions=True,
diff --git a/tests/unit/workspace/_repo_index.py b/tests/unit/workspace/_repo_index.py
new file mode 100644
index 000000000..1534d78f1
--- /dev/null
+++ b/tests/unit/workspace/_repo_index.py
@@ -0,0 +1,80 @@
+"""Build a real per-repo index for the contract-extraction tests.
+
+Writes an actual ``.repowise/wiki.db`` with the rows ingestion would have
+persisted and opens it through :func:`open_repo_index`, so the tests exercise
+the query path — repository resolution, ``file_path`` keys, external edges —
+rather than a stand-in for it.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import TYPE_CHECKING, Any
+
+from repowise.core.persistence.models import GraphEdge, Repository, WikiSymbol
+from repowise.core.workspace.registry import open_repo_db
+from repowise.core.workspace.repo_index import RepoIndex, open_repo_index
+
+if TYPE_CHECKING:
+ from collections.abc import Mapping, Sequence
+ from pathlib import Path
+
+_REPO_ID = "testrepo"
+
+
+async def make_repo_index(
+ repo: Path,
+ symbols_by_file: Mapping[str, Sequence[Any]],
+ *,
+ alias: str = "backend",
+ external_edges: Sequence[tuple[str, str, list[str]]] = (),
+) -> RepoIndex:
+ """Persist *symbols_by_file* to ``repo``'s wiki.db and open it.
+
+ Values are ingestion ``Symbol`` objects; they are mapped onto ``WikiSymbol``
+ exactly as :func:`repowise.core.persistence.crud.external_systems` does.
+ *external_edges* are ``(source_file, target_node_id, imported_names)``;
+ ``imported_names`` is written verbatim so a malformed payload can be tested.
+ """
+ (repo / ".repowise").mkdir(parents=True, exist_ok=True)
+ engine, session_factory = await open_repo_db(repo)
+ async with session_factory() as session:
+ session.add(
+ Repository(id=_REPO_ID, name=alias, url="", local_path=str(repo))
+ )
+ await session.flush() # graph_edges carries an FK onto the row above
+ for rel_path, symbols in symbols_by_file.items():
+ for sym in symbols:
+ session.add(
+ WikiSymbol(
+ repository_id=_REPO_ID,
+ file_path=rel_path,
+ symbol_id=sym.id,
+ name=sym.name,
+ qualified_name=sym.qualified_name,
+ kind=sym.kind,
+ signature=sym.signature,
+ start_line=sym.start_line,
+ end_line=sym.end_line,
+ docstring=sym.docstring,
+ visibility=sym.visibility,
+ language=sym.language,
+ parent_name=sym.parent_name,
+ )
+ )
+ for source, target, names in external_edges:
+ session.add(
+ GraphEdge(
+ repository_id=_REPO_ID,
+ source_node_id=source,
+ target_node_id=target,
+ edge_type="imports",
+ imported_names_json=json.dumps(names),
+ )
+ )
+ await session.commit()
+ await engine.dispose()
+
+ index = await open_repo_index(alias, repo)
+ assert index is not None
+ return index
diff --git a/tests/unit/workspace/test_extraction_walk_budget.py b/tests/unit/workspace/test_extraction_walk_budget.py
index 485c0e89c..d1814082e 100644
--- a/tests/unit/workspace/test_extraction_walk_budget.py
+++ b/tests/unit/workspace/test_extraction_walk_budget.py
@@ -201,7 +201,7 @@ async def test_every_contract_records_its_extraction_layer(
monkeypatch.setattr(contracts_mod, "save_contract_store", lambda store, root: root)
store = await run_contract_extraction(workspace, tmp_path, [])
assert store.contracts
- # No parse cache in these fixture repos, so everything is regex-sourced.
+ # No wiki.db in these fixture repos, so everything is regex-sourced.
assert {c.meta.get(EXTRACTION_LAYER_KEY) for c in store.contracts} == {LAYER_REGEX}
diff --git a/tests/unit/workspace/test_from_index.py b/tests/unit/workspace/test_from_index.py
index 476439b9f..acd0fe6b1 100644
--- a/tests/unit/workspace/test_from_index.py
+++ b/tests/unit/workspace/test_from_index.py
@@ -1,15 +1,21 @@
-"""Index-backed contract extraction: routes read from parsed decorators.
+"""Index-backed contract extraction: routes read at indexed handler symbols.
-Two claims are under test here. The first is that reading the decorator a
-parser already recorded removes a class of defect the regex could only chase:
-a route-shaped string in a comment is not a decorator, and an empty path is
-just a string argument, so neither can fabricate or lose a contract.
+Three claims are under test here. The first is that anchoring a route to the
+declaration directly above an indexed symbol removes a class of defect the
+regex could only chase: a route-shaped string in a comment or a docstring is
+not a declaration above a handler, and an empty path is just a string argument,
+so neither can fabricate or lose a contract.
The second is the inherited-framework claim. Ingestion resolves Flask as well
as FastAPI, while the workspace regex layer was FastAPI-only. Flask arriving
here without a new dialect module is the evidence that the rest of the
framework list comes with the layer rather than being hand-written per
framework.
+
+The third is that the substrate is the repo's ``wiki.db``. The parse cache this
+module used to read loaded zero of three repos on a live workspace, so the
+end-to-end class below writes real rows and reads them back through the real
+query path.
"""
from __future__ import annotations
@@ -18,51 +24,60 @@ from pathlib import Path
import pytest
-from repowise.core.ingestion.models import FileInfo, ParsedFile, Symbol
+from repowise.core.ingestion.models import FileInfo, Symbol
+from repowise.core.ingestion.parser import ASTParser
from repowise.core.workspace.extractors.base import ScanContext
from repowise.core.workspace.extractors.from_index import (
EXTRACTION_LAYER_KEY,
LAYER_INDEX,
extract_http_providers,
- load_repo_index,
+ symbols_for_content,
)
+from repowise.core.workspace.repo_index import IndexedSymbol
+from ._repo_index import make_repo_index
-def _symbol(name: str, decorators: list[str], line: int = 10) -> Symbol:
- return Symbol(
- id=f"m.py::{name}",
- name=name,
- qualified_name=name,
- kind="function",
- signature=f"def {name}()",
- start_line=line,
- end_line=line + 3,
- docstring=None,
- decorators=decorators,
- language="python",
- )
+REL_PATH = "app/routers/chat.py"
-def _parsed(symbols: list[Symbol], rel_path: str = "app/routers/chat.py") -> ParsedFile:
- info = FileInfo(
- path=rel_path,
- abs_path=f"/repo/{rel_path}",
- language="python",
- size_bytes=100,
- git_hash="",
- last_modified=0.0,
- is_test=False,
- is_config=False,
- is_api_contract=True,
- is_entry_point=False,
- )
- return ParsedFile(file_info=info, symbols=symbols, imports=[], exports=[])
+def _case(
+ prelude: str, *handlers: tuple[str, list[str]]
+) -> tuple[ScanContext, list[IndexedSymbol], list[str]]:
+ """Assemble file text plus the symbol rows the index would hold for it.
+
+ Each handler contributes its decorator lines and a two-line ``def``. The
+ symbol's ``start_line`` is the ``def`` line, which is where the parser puts
+ it — decorators live above the span, never inside it.
+ """
+ lines = prelude.split("\n")
+ symbols: list[IndexedSymbol] = []
+ for name, decorators in handlers:
+ lines.append("")
+ lines.extend(decorators)
+ start = len(lines) + 1
+ lines.append(f"async def {name}() -> None:")
+ lines.append(" ...")
+ symbols.append(
+ IndexedSymbol(
+ symbol_id=f"{REL_PATH}::{name}",
+ name=name,
+ qualified_name=name,
+ kind="function",
+ signature=f"def {name}()",
+ file_path=REL_PATH,
+ start_line=start,
+ end_line=start + 1,
+ visibility="public",
+ )
+ )
+ content = "\n".join(lines)
+ ctx = ScanContext("backend", REL_PATH, ".py", content)
+ return ctx, symbols, lines
-def _ctx(content: str, rel_path: str = "app/routers/chat.py") -> ScanContext:
- return ScanContext(
- repo_alias="backend", rel_path=rel_path, suffix=".py", content=content
- )
+def _extract(prelude: str, *handlers: tuple[str, list[str]]):
+ ctx, symbols, lines = _case(prelude, *handlers)
+ return extract_http_providers(ctx, symbols, lines)
def _ids(contracts) -> set[str]:
@@ -70,16 +85,14 @@ def _ids(contracts) -> set[str]:
class TestFastApiFromDecorators:
- CONTENT = 'router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")\n'
+ PRELUDE = 'router = APIRouter(prefix="/snapshots/{snapshot_id}/chat")'
def test_empty_path_route_resolves_to_the_prefix(self) -> None:
- parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])
- ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ ids = _ids(_extract(self.PRELUDE, ("chat_message", ['@router.post("")'])))
assert ids == {"http::POST::/snapshots/{param}/chat"}
def test_provenance_is_recorded(self) -> None:
- parsed = _parsed([_symbol("chat_message", ['@router.post("")'])])
- contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ contract = _extract(self.PRELUDE, ("chat_message", ['@router.post("")']))[0]
assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX
assert contract.meta["handler"] == "chat_message"
@@ -88,81 +101,164 @@ class TestFastApiFromDecorators:
# ("py_servicer", "go_client", "proto"); the layer must not overwrite
# it or the coverage metric mis-partitions every gRPC contract.
assert EXTRACTION_LAYER_KEY != "source"
- parsed = _parsed([_symbol("chat", ['@router.post("")'])])
- contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ contract = _extract(self.PRELUDE, ("chat", ['@router.post("")']))[0]
assert "source" not in contract.meta
- def test_duplicated_decorator_entries_yield_one_contract(self) -> None:
- # The parser can append the same decorator twice for one symbol.
- parsed = _parsed(
- [_symbol("chat", ['@router.post("")', '@router.post("")'])]
+ def test_a_multi_line_decorator_is_read_whole(self) -> None:
+ ids = _ids(
+ _extract(
+ self.PRELUDE,
+ (
+ "chat",
+ ["@router.post(", ' "/messages",', " status_code=201,", ")"],
+ ),
+ )
+ )
+ assert ids == {"http::POST::/snapshots/{param}/chat/messages"}
+
+ def test_a_stacked_decorator_does_not_hide_the_route(self) -> None:
+ ids = _ids(
+ _extract(
+ self.PRELUDE,
+ ("chat", ['@limiter.limit("120/minute")', '@router.get("/history")']),
+ )
)
- assert len(extract_http_providers(_ctx(self.CONTENT), parsed)) == 1
+ assert ids == {"http::GET::/snapshots/{param}/chat/history"}
def test_non_route_decorators_are_ignored(self) -> None:
- parsed = _parsed(
+ assert (
+ _extract(
+ self.PRELUDE,
+ ("chat", ['@limiter.limit("240/minute")', "@dataclass", "@classmethod"]),
+ )
+ == []
+ )
+
+ def test_unknown_router_variable_is_not_a_route(self) -> None:
+ assert _extract(self.PRELUDE, ("cached", ['@cache.get("/x")'])) == []
+
+ def test_the_same_route_twice_yields_one_contract(self) -> None:
+ contracts = _extract(
+ self.PRELUDE,
+ ("chat", ['@router.post("")']),
+ ("chat_again", ['@router.post("")']),
+ )
+ assert len(contracts) == 1
+
+ def test_a_decorated_class_does_not_lend_its_route_to_a_method(self) -> None:
+ # PEP 8 asks for a blank line here and nothing enforces one. Without a
+ # stop at ordinary code the walk climbs past ``class Foo:`` and binds
+ # the class's route to the first method under it.
+ content = '\n'.join(
[
- _symbol(
- "chat",
- ['@limiter.limit("240/minute")', "@dataclass", "@classmethod"],
- )
+ 'app = Flask(__name__)',
+ '@app.route("/items")',
+ "class Foo:",
+ " def get(self) -> None:",
+ " ...",
]
)
- assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+ ctx = ScanContext("backend", REL_PATH, ".py", content)
+ method = IndexedSymbol(
+ symbol_id=f"{REL_PATH}::Foo::get",
+ name="get",
+ qualified_name="Foo.get",
+ kind="method",
+ signature="def get(self)",
+ file_path=REL_PATH,
+ start_line=4,
+ end_line=5,
+ visibility="public",
+ )
+ assert extract_http_providers(ctx, [method], content.split("\n")) == []
- def test_unknown_router_variable_is_not_a_route(self) -> None:
- parsed = _parsed([_symbol("cached", ['@cache.get("/x")'])])
- assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+ def test_a_neighbouring_handler_does_not_lend_its_route(self) -> None:
+ content = '\n'.join(
+ [
+ 'router = APIRouter()',
+ '@router.get("/first")',
+ "async def first() -> None:",
+ " ...",
+ "async def second() -> None:",
+ " ...",
+ ]
+ )
+ ctx = ScanContext("backend", REL_PATH, ".py", content)
+ second = IndexedSymbol(
+ symbol_id=f"{REL_PATH}::second",
+ name="second",
+ qualified_name="second",
+ kind="function",
+ signature="def second()",
+ file_path=REL_PATH,
+ start_line=5,
+ end_line=6,
+ visibility="public",
+ )
+ assert extract_http_providers(ctx, [second], content.split("\n")) == []
+
+ def test_a_non_handler_symbol_carries_no_route(self) -> None:
+ # A route decorator can only sit above a callable; a module constant
+ # that happens to follow one must not become a contract.
+ ctx, symbols, lines = _case(self.PRELUDE, ("chat", ['@router.post("/x")']))
+ constants = [
+ IndexedSymbol(
+ symbol_id=f"{REL_PATH}::EXAMPLE",
+ name="EXAMPLE",
+ qualified_name="EXAMPLE",
+ kind="constant",
+ signature="",
+ file_path=REL_PATH,
+ start_line=symbols[0].start_line,
+ end_line=symbols[0].start_line,
+ visibility="public",
+ )
+ ]
+ assert extract_http_providers(ctx, constants, lines) == []
class TestCommentsCannotFabricateRoutes:
- """The self-extraction defect, structurally: a comment is not a decorator."""
+ """The self-extraction defect, structurally: prose is not a declaration."""
- CONTENT = '''
-router = APIRouter(prefix="/api")
+ PRELUDE = '''router = APIRouter(prefix="/api")
# Recognises @app.get('/path') and @router.post('/orders') declarations.
-EXAMPLE = "@router.delete('/fabricated')"
-
+EXAMPLE = "@router.delete('/fabricated')"'''
-def documented() -> None:
- """Handles @router.put("/from-a-docstring") style routes."""
-'''
-
- def test_route_shaped_text_outside_a_decorator_yields_nothing(self) -> None:
- # No symbol carries a route decorator, so there is no route — however
- # much route-shaped prose the file contains.
- parsed = _parsed([_symbol("documented", [])], rel_path="extractors/http.py")
- assert extract_http_providers(_ctx(self.CONTENT), parsed) == []
+ def test_route_shaped_text_above_no_handler_yields_nothing(self) -> None:
+ assert _extract(self.PRELUDE, ("documented", [])) == []
def test_a_real_decorator_in_the_same_file_still_counts(self) -> None:
- parsed = _parsed(
- [_symbol("real", ['@router.get("/real")'])], rel_path="extractors/http.py"
- )
- ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ ids = _ids(_extract(self.PRELUDE, ("real", ['@router.get("/real")'])))
assert ids == {"http::GET::/api/real"}
+ def test_a_commented_out_decorator_is_not_a_route(self) -> None:
+ # The comment stop is what keeps the walk out of the previous symbol's
+ # body, and it makes a commented-out decorator inert on the way.
+ ids = _ids(_extract(self.PRELUDE, ("plain", ['# @router.get("/commented")'])))
+ assert ids == set()
+
class TestFlaskIsInherited:
"""Flask providers via the index path — no Flask dialect module exists."""
- CONTENT = 'app = Flask(__name__)\nbp = Blueprint("api", __name__)\n'
+ PRELUDE = 'app = Flask(__name__)\nbp = Blueprint("api", __name__)'
def test_route_decorator_defaults_to_get(self) -> None:
- parsed = _parsed([_symbol("index", ['@app.route("/health")'])])
- ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
+ ids = _ids(_extract(self.PRELUDE, ("index", ['@app.route("/health")'])))
assert ids == {"http::GET::/health"}
def test_methods_kwarg_yields_one_contract_per_verb(self) -> None:
- parsed = _parsed(
- [_symbol("users", ['@app.route("/users", methods=["POST", "PUT"])'])]
+ ids = _ids(
+ _extract(
+ self.PRELUDE,
+ ("users", ['@app.route("/users", methods=["POST", "PUT"])']),
+ )
)
- ids = _ids(extract_http_providers(_ctx(self.CONTENT), parsed))
assert ids == {"http::POST::/users", "http::PUT::/users"}
def test_framework_is_labelled_flask(self) -> None:
- parsed = _parsed([_symbol("index", ['@app.route("/health")'])])
- contract = extract_http_providers(_ctx(self.CONTENT), parsed)[0]
+ contract = _extract(self.PRELUDE, ("index", ['@app.route("/health")']))[0]
assert contract.meta["framework"] == "flask"
assert contract.meta[EXTRACTION_LAYER_KEY] == LAYER_INDEX
@@ -172,9 +268,8 @@ class TestFlaskIsInherited:
# blueprint's mount lands on the route. Incidental, but correct — this
# test pins the behaviour so a later tightening of the pattern has to
# keep Flask working deliberately.
- content = 'bp = Blueprint("api", __name__, url_prefix="/api")\n'
- parsed = _parsed([_symbol("listing", ['@bp.route("/items")'])])
- ids = _ids(extract_http_providers(_ctx(content), parsed))
+ prelude = 'bp = Blueprint("api", __name__, url_prefix="/api")'
+ ids = _ids(_extract(prelude, ("listing", ['@bp.route("/items")'])))
assert ids == {"http::GET::/api/items"}
def test_the_regex_layer_has_no_flask_dialect(self) -> None:
@@ -185,16 +280,9 @@ class TestFlaskIsInherited:
assert "flask" not in {d.name for d in PROVIDER_DIALECTS}
-class TestLoadRepoIndex:
- def test_missing_cache_returns_none(self, tmp_path) -> None:
- assert load_repo_index(tmp_path) is None
-
- def test_unreadable_cache_returns_none(self, tmp_path) -> None:
- # A repo indexed by another repowise version fails the parser
- # fingerprint gate and must degrade to the regex path, not raise.
- (tmp_path / ".repowise").mkdir()
- (tmp_path / ".repowise" / "parse_cache.pkl").write_bytes(b"not a pickle")
- assert load_repo_index(tmp_path) is None
+def test_no_index_on_the_context_means_no_symbols() -> None:
+ ctx = ScanContext("backend", REL_PATH, ".py", "x = 1\n")
+ assert symbols_for_content(ctx, 1) == []
ROUTES_PY = '''\
@@ -218,84 +306,69 @@ async def list_conversations() -> None:
'''
-def _write_real_parse_cache(repo: Path) -> None:
- """Parse *repo*'s Python with the real ingestion parser and cache it.
+def _parse_symbols(rel_path: str, source: str) -> list[Symbol]:
+ """Symbols for *source* from the real parser, as ingestion records them."""
+ info = FileInfo(
+ path=rel_path,
+ abs_path=f"/repo/{rel_path}",
+ language="python",
+ size_bytes=len(source),
+ git_hash="",
+ last_modified=0.0,
+ is_test=False,
+ is_config=False,
+ is_api_contract=True,
+ is_entry_point=False,
+ )
+ return ASTParser().parse_file(info, source.encode()).symbols
- Uses ``ParseCache`` itself rather than a hand-built pickle, so the entry
- key, the ``parser_fingerprint`` gate and the content hash are all produced
- the way ingestion produces them.
- """
- from repowise.core.ingestion.models import compute_content_hash
- from repowise.core.ingestion.parse_cache import ParseCache
- from repowise.core.ingestion.parser import parse_file
- from repowise.core.ingestion.traverser import FileTraverser
-
- cache = ParseCache(repo / ".repowise")
- for info in FileTraverser(repo).traverse():
- if not info.path.endswith(".py"):
- continue
- source = Path(info.abs_path).read_bytes()
- cache.put(parse_file(info, source), compute_content_hash(source))
- cache.save()
-
-
-class TestAgainstARealParseCache:
- """The plumbing, end to end: real parser, real cache, real hash check.
-
- Everything else here builds a ``ParsedFile`` by hand, which cannot catch
- the two agreements this path depends on: that the walk hashes bytes the
- same way ingestion does, and that a cached entry survives the round trip.
+
+class TestAgainstARealIndex:
+ """The plumbing, end to end: real parser, real wiki.db, real query path.
+
+ Everything above builds symbol rows by hand, which cannot catch the two
+ agreements this path depends on — that the parser's line numbers leave a
+ decorator above the span, and that the rows survive the database round trip
+ keyed by the same relative path the walk yields.
"""
@pytest.fixture
- def repo(self, tmp_path: Path) -> Path:
- (tmp_path / ".repowise").mkdir()
+ async def repo(self, tmp_path: Path):
(tmp_path / "routers").mkdir()
(tmp_path / "routers" / "chat.py").write_text(ROUTES_PY, encoding="utf-8")
- _write_real_parse_cache(tmp_path)
- return tmp_path
+ rel = "routers/chat.py"
+ index = await make_repo_index(tmp_path, {rel: _parse_symbols(rel, ROUTES_PY)})
+ yield tmp_path, index
+ await index.close()
- def test_the_cache_round_trips(self, repo: Path) -> None:
- index = load_repo_index(repo)
- assert index is not None
- assert "routers/chat.py" in index
-
- def test_routes_come_from_the_index(self, repo: Path) -> None:
+ def _extract_all(self, repo: Path, index):
from repowise.core.workspace.extractors import HttpExtractor
from repowise.core.workspace.extractors.base import make_exclude_predicate
- index = load_repo_index(repo)
- contracts = HttpExtractor().extract(
- repo, "backend", make_exclude_predicate(), None, index=index
+ return HttpExtractor().extract(
+ repo, "backend", make_exclude_predicate(), None, index
)
- providers = [c for c in contracts if c.role == "provider"]
+
+ async def test_the_rows_round_trip(self, repo) -> None:
+ _path, index = repo
+ assert index.symbols_for_file("routers/chat.py")
+
+ async def test_routes_come_from_the_index(self, repo) -> None:
+ path, index = repo
+ providers = [c for c in self._extract_all(path, index) if c.role == "provider"]
assert providers, "index path produced nothing"
assert {c.meta[EXTRACTION_LAYER_KEY] for c in providers} == {LAYER_INDEX}
- def test_the_empty_path_route_survives_the_whole_path(self, repo: Path) -> None:
- from repowise.core.workspace.extractors import HttpExtractor
- from repowise.core.workspace.extractors.base import make_exclude_predicate
-
- index = load_repo_index(repo)
- ids = _ids(
- HttpExtractor().extract(
- repo, "backend", make_exclude_predicate(), None, index=index
- )
- )
+ async def test_the_empty_path_route_survives_the_whole_path(self, repo) -> None:
+ path, index = repo
+ ids = _ids(self._extract_all(path, index))
assert "http::POST::/snapshots/{param}/chat" in ids
- def test_routes_in_comments_and_docstrings_are_not_extracted(
- self, repo: Path
+ async def test_routes_in_comments_and_docstrings_are_not_extracted(
+ self, repo
) -> None:
- from repowise.core.workspace.extractors import HttpExtractor
- from repowise.core.workspace.extractors.base import make_exclude_predicate
-
- index = load_repo_index(repo)
- ids = _ids(
- HttpExtractor().extract(
- repo, "backend", make_exclude_predicate(), None, index=index
- )
- )
+ path, index = repo
+ ids = _ids(self._extract_all(path, index))
for fabricated in (
"http::GET::/snapshots/{param}/chat/fabricated",
"http::DELETE::/snapshots/{param}/chat/also-fabricated",
@@ -303,23 +376,28 @@ class TestAgainstARealParseCache:
):
assert fabricated not in ids
- def test_an_edited_file_falls_back_to_the_regex(self, repo: Path) -> None:
- from repowise.core.workspace.extractors import HttpExtractor
- from repowise.core.workspace.extractors.base import make_exclude_predicate
- from repowise.core.workspace.extractors.from_index import parsed_for
-
- index = load_repo_index(repo)
- # Edit the file after it was cached: the entry now describes older
- # bytes, so it must be rejected rather than trusted.
- (repo / "routers" / "chat.py").write_text(
- ROUTES_PY + '\n\[email protected]("/added")\nasync def added() -> None: ...\n',
+ async def test_a_truncated_file_falls_back_to_the_regex(self, repo) -> None:
+ path, index = repo
+ # Cut the file down after it was indexed: the spans now overrun it, so
+ # they must be refused rather than used to slice the wrong lines.
+ (path / "routers" / "chat.py").write_text(
+ 'from fastapi import APIRouter\n\nrouter = APIRouter(prefix="/api")\n',
encoding="utf-8",
)
- contracts = HttpExtractor().extract(
- repo, "backend", make_exclude_predicate(), None, index=index
+ contracts = self._extract_all(path, index)
+ layers = {
+ c.meta.get(EXTRACTION_LAYER_KEY) for c in contracts if c.role == "provider"
+ }
+ assert LAYER_INDEX not in layers
+
+ async def test_a_route_added_after_indexing_is_still_found(self, repo) -> None:
+ path, index = repo
+ (path / "routers" / "chat.py").write_text(
+ ROUTES_PY + '\n\[email protected]("/added")\nasync def added() -> None: ...\n',
+ encoding="utf-8",
)
- layers = {c.meta.get(EXTRACTION_LAYER_KEY) for c in contracts if c.role == "provider"}
- assert LAYER_INDEX not in layers # stale entry refused
- assert parsed_for(index, "routers/chat.py", "deadbeef") is None
- # And the newly added route is still found, by the regex path.
- assert "http::GET::/snapshots/{param}/chat/added" in _ids(contracts)
+ # The appended route sits outside every indexed span, so the index
+ # cannot see it; the regex dialect still can, and both run for a file
+ # the index pass produced no routes for.
+ ids = _ids(self._extract_all(path, index))
+ assert "http::GET::/snapshots/{param}/chat/added" in ids
diff --git a/tests/unit/workspace/test_incremental_extraction.py b/tests/unit/workspace/test_incremental_extraction.py
index 4e1a6b416..ff1d31d12 100644
--- a/tests/unit/workspace/test_incremental_extraction.py
+++ b/tests/unit/workspace/test_incremental_extraction.py
@@ -88,9 +88,9 @@ def extracted(monkeypatch: pytest.MonkeyPatch) -> list[str]:
original_iter = base.iter_source_files
- def recording_iter(repo_path, wanted, exclude, content_hashes=None):
+ def recording_iter(repo_path, wanted, exclude=None):
seen.append(Path(repo_path).name)
- return original_iter(repo_path, wanted, exclude, content_hashes)
+ return original_iter(repo_path, wanted, exclude)
monkeypatch.setattr(base, "iter_source_files", recording_iter)
return seen
diff --git a/tests/unit/workspace/test_index_clients.py b/tests/unit/workspace/test_index_clients.py
index 006172e7c..08416540f 100644
--- a/tests/unit/workspace/test_index_clients.py
+++ b/tests/unit/workspace/test_index_clients.py
@@ -10,11 +10,12 @@ Two consequences are what these tests pin:
The second is the false positive the name-matching approach cannot avoid, and
is the thing this layer is bought for.
-Symbols come from the real :class:`ASTParser` rather than hand-built
-:class:`Symbol` objects. Wrapper confirmation depends on each symbol's
-``start_line``/``end_line`` being the extent ingestion actually recorded, so
-fabricating those here would test this module against my own arithmetic instead
-of against the parse it consumes in production.
+Symbols come from the real :class:`ASTParser`, mapped onto the
+:class:`IndexedSymbol` rows ingestion persists, rather than hand-built spans.
+Wrapper confirmation depends on each symbol's ``start_line``/``end_line`` being
+the extent ingestion actually recorded, so fabricating those here would test
+this module against my own arithmetic instead of against the parse it consumes
+in production.
"""
from __future__ import annotations
@@ -30,9 +31,13 @@ from repowise.core.workspace.extractors.from_index import EXTRACTION_LAYER_KEY,
from repowise.core.workspace.extractors.http.index_clients import extract_consumers
from repowise.core.workspace.extractors.http.js_clients import JsClientsDialect
from repowise.core.workspace.extractors.http.wrappers import confirm_wrappers
+from repowise.core.workspace.repo_index import IndexedSymbol
+from ._repo_index import make_repo_index
-def _parse(path: str, source: str):
+
+def _ingestion_symbols(path: str, source: str):
+ """The symbols the real parser records for *source*."""
info = FileInfo(
path=path,
abs_path=f"C:/fake/{path}",
@@ -45,15 +50,33 @@ def _parse(path: str, source: str):
is_api_contract=False,
is_entry_point=False,
)
- return ASTParser().parse_file(info, source.encode())
+ return ASTParser().parse_file(info, source.encode()).symbols
+
+
+def _symbols(path: str, source: str) -> list[IndexedSymbol]:
+ """Those symbols as the rows ingestion persists, which is what this reads."""
+ return [
+ IndexedSymbol(
+ symbol_id=sym.id,
+ name=sym.name,
+ qualified_name=sym.qualified_name,
+ kind=sym.kind,
+ signature=sym.signature,
+ file_path=path,
+ start_line=sym.start_line,
+ end_line=sym.end_line,
+ visibility=sym.visibility,
+ )
+ for sym in _ingestion_symbols(path, source)
+ ]
def _run(source: str, path: str = "src/lib/api/client.ts"):
"""Extract via the index path; returns ``(contracts, unresolved, ctx)``."""
- parsed = _parse(path, source)
+ symbols = _symbols(path, source)
suffix = "." + path.rsplit(".", 1)[1]
ctx = ScanContext("frontend", path, suffix, source, {})
- contracts, unresolved = extract_consumers(ctx, parsed)
+ contracts, unresolved = extract_consumers(ctx, symbols)
return contracts, unresolved, ctx
@@ -61,6 +84,13 @@ def _ids(contracts) -> set[str]:
return {c.contract_id for c in contracts}
+async def _index_for(repo, path: str, source: str):
+ """A real per-repo index holding just *path*'s parsed symbols."""
+ return await make_repo_index(
+ repo, {path: _ingestion_symbols(path, source)}, alias="frontend"
+ )
+
+
# The shape frontend/src/lib/api/client.ts really uses: a private `request`
# holding the sink, a generic `fetch<T>` delegating to it, and API methods
# calling `this.fetch<T>(...)`. Three hops from call site to sink.
@@ -209,8 +239,8 @@ class TestNameLookalikesAreRejected:
assert unresolved == 0
def test_they_are_not_confirmed_as_wrappers(self):
- parsed = _parse("src/lib/cache.ts", DECOY_TS)
- assert confirm_wrappers(parsed, DECOY_TS, ".ts") == set()
+ symbols = _symbols("src/lib/cache.ts", DECOY_TS)
+ assert confirm_wrappers(symbols, DECOY_TS, ".ts") == set()
def test_the_regex_dialect_does_emit_them(self):
"""Documents the defect being removed, so the gain is not hypothetical."""
@@ -299,10 +329,9 @@ class TestUnresolvedPathsAreCounted:
class TestFallbackRemainsReachable:
"""A repo with no usable index keeps the regex path.
- The parse cache is version-gated, so a repo indexed by an older repowise
- loads zero entries — measured on two of three repos in this workspace. The
- regex dialect is therefore not a legacy path but the only path for those
- repos, and it has to stay reachable.
+ A repo that has never been indexed has no ``wiki.db`` to read, so the regex
+ dialect is not a legacy path but the only path for it, and it has to stay
+ reachable.
"""
# Plain `fetch("/x")` — the one shape the text dialect handles well.
@@ -316,28 +345,32 @@ class TestFallbackRemainsReachable:
"frontend",
None,
files=[("src/legacy.ts", ".ts", self.PLAIN)],
- index=index,
- content_hashes={},
+ repo_index=index,
)
def test_without_an_index_the_regex_dialect_still_extracts(self, tmp_path):
got = self._extract(tmp_path, index=None)
assert "http::GET::/legacy/ping" in _ids(got)
- def test_an_index_that_cannot_read_a_file_does_not_delete_its_contracts(
+ async def test_an_index_missing_this_file_does_not_delete_its_contracts(
self, tmp_path
):
- """An index present but missing this file must not suppress the dialect.
-
- This is the rule that stops a bad or partial parse silently deleting
- contracts the regex can still see.
+ """An index present but holding no rows for this file must not suppress
+ the dialect. This is the rule that stops a partial index silently
+ deleting contracts the regex can still see.
"""
- got = self._extract(tmp_path, index={"src/other.ts": _parse("src/other.ts", "")})
+ index = await make_repo_index(
+ tmp_path, {"src/other.ts": []}, alias="frontend"
+ )
+ try:
+ got = self._extract(tmp_path, index=index)
+ finally:
+ await index.close()
assert "http::GET::/legacy/ping" in _ids(got)
def test_a_language_with_no_sink_patterns_confirms_nothing(self):
- parsed = _parse("src/main.go", "package main\n")
- assert confirm_wrappers(parsed, "package main\n", ".go") == set()
+ symbols = _symbols("src/main.go", "package main\n")
+ assert confirm_wrappers(symbols, "package main\n", ".go") == set()
def test_one_line_wrapper_bodies_are_not_lost(self):
"""A concise wrapper puts its body on the declaration line."""
@@ -347,8 +380,8 @@ class TestFallbackRemainsReachable:
" go() { return this.ping(); }\n"
"}\n"
)
- parsed = _parse("src/lib/one.ts", source)
- assert "ping" in confirm_wrappers(parsed, source, ".ts")
+ symbols = _symbols("src/lib/one.ts", source)
+ assert "ping" in confirm_wrappers(symbols, source, ".ts")
class TestHopBudget:
@@ -356,12 +389,12 @@ class TestHopBudget:
def test_budget_one_still_reaches_through_the_two_hop_chain(self):
"""call site -> fetch -> request -> sink: the target is 1 hop from the sink."""
- parsed = _parse("src/lib/api/client.ts", CLIENT_TS)
- assert "fetch" in confirm_wrappers(parsed, CLIENT_TS, ".ts", budget=1)
+ symbols = _symbols("src/lib/api/client.ts", CLIENT_TS)
+ assert "fetch" in confirm_wrappers(symbols, CLIENT_TS, ".ts", budget=1)
def test_budget_zero_confirms_only_the_symbol_holding_the_sink(self):
- parsed = _parse("src/lib/api/client.ts", CLIENT_TS)
- assert confirm_wrappers(parsed, CLIENT_TS, ".ts", budget=0) == {"request"}
+ symbols = _symbols("src/lib/api/client.ts", CLIENT_TS)
+ assert confirm_wrappers(symbols, CLIENT_TS, ".ts", budget=0) == {"request"}
def test_a_chain_longer_than_the_budget_is_not_confirmed(self):
source = (
@@ -373,8 +406,8 @@ class TestHopBudget:
" e() { return this.d('/x/y'); }\n"
"}\n"
)
- parsed = _parse("src/lib/deep.ts", source)
- confirmed = confirm_wrappers(parsed, source, ".ts", budget=2)
+ symbols = _symbols("src/lib/deep.ts", source)
+ confirmed = confirm_wrappers(symbols, source, ".ts", budget=2)
assert "a" in confirmed and "b" in confirmed and "c" in confirmed
assert "d" not in confirmed
@@ -387,8 +420,8 @@ class TestCycleSafety:
" b(p: string) { return this.a(p); }\n"
"}\n"
)
- parsed = _parse("src/lib/r.ts", source)
- assert confirm_wrappers(parsed, source, ".ts") == set()
+ symbols = _symbols("src/lib/r.ts", source)
+ assert confirm_wrappers(symbols, source, ".ts") == set()
class TestSseEndpointsLinkToTheirConsumers:
@@ -587,7 +620,7 @@ class TestNothingFoundIsSilentlyDropped:
class TestSupersedeCannotSubtract:
"""The index pass removes its own duplicates, never another shape."""
- def test_an_axios_call_survives_beside_a_confirmed_wrapper(self, tmp_path):
+ async def test_an_axios_call_survives_beside_a_confirmed_wrapper(self, tmp_path):
from repowise.core.workspace.extractors.http import HttpExtractor
source = (
@@ -599,20 +632,22 @@ class TestSupersedeCannotSubtract:
"}\n"
"export function legacy() { return axios.get('/legacy'); }\n"
)
- parsed = _parse("src/lib/mix.ts", source)
- got = HttpExtractor().extract(
- tmp_path,
- "frontend",
- None,
- files=[("src/lib/mix.ts", ".ts", source)],
- index={"src/lib/mix.ts": parsed},
- content_hashes={"src/lib/mix.ts": parsed.content_hash},
- )
+ index = await _index_for(tmp_path, "src/lib/mix.ts", source)
+ try:
+ got = HttpExtractor().extract(
+ tmp_path,
+ "frontend",
+ None,
+ files=[("src/lib/mix.ts", ".ts", source)],
+ repo_index=index,
+ )
+ finally:
+ await index.close()
ids = _ids(got)
assert "http::GET::/wrapped" in ids # from the index pass
assert "http::GET::/legacy" in ids # only the regex dialect sees this
- def test_duplicates_are_not_emitted_twice(self, tmp_path):
+ async def test_duplicates_are_not_emitted_twice(self, tmp_path):
from repowise.core.workspace.extractors.http import HttpExtractor
source = (
@@ -620,15 +655,17 @@ class TestSupersedeCannotSubtract:
" return fetch('/dup');\n"
"}\n"
)
- parsed = _parse("src/lib/dup.ts", source)
- got = HttpExtractor().extract(
- tmp_path,
- "frontend",
- None,
- files=[("src/lib/dup.ts", ".ts", source)],
- index={"src/lib/dup.ts": parsed},
- content_hashes={"src/lib/dup.ts": parsed.content_hash},
- )
+ index = await _index_for(tmp_path, "src/lib/dup.ts", source)
+ try:
+ got = HttpExtractor().extract(
+ tmp_path,
+ "frontend",
+ None,
+ files=[("src/lib/dup.ts", ".ts", source)],
+ repo_index=index,
+ )
+ finally:
+ await index.close()
dup = [c for c in got if c.contract_id == "http::GET::/dup"]
assert len(dup) == 1
diff --git a/tests/unit/workspace/test_repo_index.py b/tests/unit/workspace/test_repo_index.py
new file mode 100644
index 000000000..e22c46bcd
--- /dev/null
+++ b/tests/unit/workspace/test_repo_index.py
@@ -0,0 +1,125 @@
+"""The read-only per-repo accessor the cross-repo hooks run on.
+
+Every test here writes a real ``.repowise/wiki.db`` and reads it back, because
+the failure this accessor exists to prevent — the hooks re-deriving from text
+what the database already holds — is a plumbing failure, not a logic one.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.ingestion.models import Symbol
+from repowise.core.workspace.repo_index import open_repo_index, open_workspace_index
+
+from ._repo_index import make_repo_index
+
+
+def _symbol(name: str, start: int, end: int, *, kind="function", visibility="public"):
+ return Symbol(
+ id=f"m.py::{name}",
+ name=name,
+ qualified_name=name,
+ kind=kind,
+ signature=f"def {name}()",
+ start_line=start,
+ end_line=end,
+ docstring=None,
+ visibility=visibility,
+ )
+
+
+class TestSymbolLookups:
+ async def test_symbols_are_keyed_by_repo_relative_path(self, tmp_path: Path) -> None:
+ index = await make_repo_index(tmp_path, {"a/b.py": [_symbol("f", 1, 5)]})
+ try:
+ assert [s.name for s in index.symbols_for_file("a/b.py")] == ["f"]
+ assert index.symbols_for_file("b.py") == []
+ finally:
+ await index.close()
+
+ async def test_symbol_at_returns_the_innermost_span(self, tmp_path: Path) -> None:
+ index = await make_repo_index(
+ tmp_path,
+ {
+ "a/b.py": [
+ _symbol("Outer", 1, 20, kind="class"),
+ _symbol("inner", 5, 9, kind="method"),
+ ]
+ },
+ )
+ try:
+ assert index.symbol_at("a/b.py", 7).name == "inner"
+ assert index.symbol_at("a/b.py", 15).name == "Outer"
+ assert index.symbol_at("a/b.py", 25) is None
+ finally:
+ await index.close()
+
+ async def test_public_symbols_span_the_whole_repo(self, tmp_path: Path) -> None:
+ index = await make_repo_index(
+ tmp_path,
+ {
+ "a.py": [_symbol("pub", 1, 2)],
+ "b.py": [_symbol("_priv", 1, 2, visibility="private")],
+ },
+ )
+ try:
+ assert {s.name for s in index.public_symbols()} == {"pub"}
+ finally:
+ await index.close()
+
+
+class TestExternalImportEdges:
+ async def test_the_prefix_is_stripped_and_names_are_carried(
+ self, tmp_path: Path
+ ) -> None:
+ index = await make_repo_index(
+ tmp_path,
+ {},
+ external_edges=[
+ ("src/a.ts", "external:@repowise-dev/types", ["RepoSummary", "Health"]),
+ ("src/b.ts", "src/c.ts", ["Local"]),
+ ],
+ )
+ try:
+ edges = index.external_import_edges()
+ assert [e.external_name for e in edges] == ["@repowise-dev/types"]
+ assert edges[0].imported_names == ("RepoSummary", "Health")
+ finally:
+ await index.close()
+
+ async def test_a_malformed_names_payload_does_not_lose_the_repo(
+ self, tmp_path: Path
+ ) -> None:
+ index = await make_repo_index(
+ tmp_path, {}, external_edges=[("src/a.ts", "external:pkg", "not-a-list")]
+ )
+ try:
+ assert index.external_import_edges()[0].imported_names == ()
+ finally:
+ await index.close()
+
+
+class TestOpening:
+ async def test_a_repo_with_no_index_opens_to_none(self, tmp_path: Path) -> None:
+ assert await open_repo_index("alpha", tmp_path) is None
+
+ async def test_the_workspace_skips_repos_without_one(self, tmp_path: Path) -> None:
+ class _Entry:
+ def __init__(self, alias: str, path: str) -> None:
+ self.alias, self.path = alias, path
+
+ class _Config:
+ def __init__(self) -> None:
+ self.repos = [_Entry("alpha", "alpha"), _Entry("beta", "beta")]
+
+ (tmp_path / "beta").mkdir()
+ index = await make_repo_index(tmp_path / "alpha", {"a.py": [_symbol("f", 1, 2)]})
+ await index.close()
+
+ workspace = await open_workspace_index(_Config(), tmp_path)
+ try:
+ assert workspace.get("alpha") is not None
+ assert workspace.get("beta") is None
+ finally:
+ await workspace.close()