Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

feat(workspace): consumer hygiene and base-URL service resolution for contract matching (#568)

Raghav Chamadiyabdb3eec16f29
diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md
index 27455dc61..c43b10e11 100644
--- a/docs/WORKSPACES.md
+++ b/docs/WORKSPACES.md
@@ -225,6 +225,33 @@ Scans source files for HTTP route handlers, gRPC service definitions, and messag
| gRPC | `.proto` service definitions | gRPC client stubs |
| Topics | Kafka, RabbitMQ, Redis Pub/Sub, NATS producers | Corresponding consumers |
+HTTP routes are matched on their **full** path: a router mount prefix
+(`APIRouter(prefix=...)`, `include_router(prefix=...)`, Express `app.use('/x', router)`,
+Go route groups) is stitched onto each handler path before matching. A client call
+whose base URL is an unresolved placeholder (`fetch(\`${API_BASE}/users\`)`) matches
+on the host-relative path; the link is **exact** when exactly one workspace service
+provides that path and a lower-confidence **candidate** when the target is ambiguous.
+
+**Tuning extraction** via the `contracts:` block in `.repowise-workspace.yaml`:
+
+```yaml
+contracts:
+ # Map a consumer base token or absolute host to the repo it targets, so a
+ # call whose base is unresolved at parse time links as an exact match.
+ service_bases:
+ API_BASE: backend # ${API_BASE}/... -> the "backend" repo
+ api.example.com: backend # https://api.example.com/... -> "backend"
+ # Extra globs to skip (added to the built-in test/spec defaults).
+ exclude_globs:
+ - "generated/**"
+```
+
+Files under test trees (`tests/`, `test/`, `__tests__/`, `spec/`, `e2e/`) and test
+files (`test_*.py`, `*.test.ts`, `*.spec.ts`, ...) are excluded by default: a route
+or topic that exists only in a test is a fixture, not a service contract. Calls to a
+literal third-party host (Stripe, Formspree, ...) that is not a workspace service are
+excluded from matching and reported under the `external_host` diagnostics reason.
+
### Package Dependency Scanning
Reads package manifests (`package.json`, `pyproject.toml`, `go.mod`, `pom.xml`, etc.) to detect when one repo depends on another as a package.
@@ -265,6 +292,7 @@ The report covers:
- `no_provider` — no provider anywhere declares a matching route/service/topic.
- `internal_only` — the only matching provider is in the same repo + service, so the call is intra-service and intentionally not surfaced as a cross-repo link.
- `unlinked` — a cross-service provider with a matching id exists, but no link formed (a candidate worth inspecting).
+ - `external_host` — the call targets a literal third-party host (Stripe, Formspree, ...) that is not a workspace service, so it is intentionally excluded from matching.
- **Orphan providers** — endpoints declared but never consumed by any repo.
- **Weak links** — matched links below the confidence threshold.
diff --git a/packages/core/src/repowise/core/workspace/config.py b/packages/core/src/repowise/core/workspace/config.py
index dfbe969b0..175587511 100644
--- a/packages/core/src/repowise/core/workspace/config.py
+++ b/packages/core/src/repowise/core/workspace/config.py
@@ -104,6 +104,14 @@ class ContractConfig:
detect_grpc: bool = True
detect_topics: bool = True
manual_links: list[ManualContractLink] = field(default_factory=list)
+ # Map a consumer base token or absolute host to the repo alias it targets,
+ # so a ``fetch(`${API_BASE}/users`)`` whose base resolves to ``backend`` links
+ # to that service as an exact match. Keys are env-var identifiers
+ # (``API_BASE``, ``VITE_API_URL``) or hosts (``api.example.com``).
+ service_bases: dict[str, str] = field(default_factory=dict)
+ # Extra glob patterns (added to the built-in test/spec defaults) whose files
+ # are skipped during contract extraction.
+ exclude_globs: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
d: dict[str, Any] = {
@@ -113,6 +121,10 @@ class ContractConfig:
}
if self.manual_links:
d["manual_links"] = [ml.to_dict() for ml in self.manual_links]
+ if self.service_bases:
+ d["service_bases"] = dict(self.service_bases)
+ if self.exclude_globs:
+ d["exclude_globs"] = list(self.exclude_globs)
return d
@classmethod
@@ -123,6 +135,8 @@ class ContractConfig:
detect_grpc=bool(data.get("detect_grpc", True)),
detect_topics=bool(data.get("detect_topics", True)),
manual_links=manual,
+ service_bases={str(k): str(v) for k, v in data.get("service_bases", {}).items()},
+ exclude_globs=[str(g) for g in data.get("exclude_globs", [])],
)
@@ -209,12 +223,17 @@ class WorkspaceConfig:
}
# Only include contracts section if non-default
contracts_d = self.contracts.to_dict()
- if self.contracts.manual_links or not all(
- [
- self.contracts.detect_http,
- self.contracts.detect_grpc,
- self.contracts.detect_topics,
- ]
+ if (
+ self.contracts.manual_links
+ or self.contracts.service_bases
+ or self.contracts.exclude_globs
+ or not all(
+ [
+ self.contracts.detect_http,
+ self.contracts.detect_grpc,
+ self.contracts.detect_topics,
+ ]
+ )
):
d["contracts"] = contracts_d
# Only include conformance section when rules are declared.
diff --git a/packages/core/src/repowise/core/workspace/contracts.py b/packages/core/src/repowise/core/workspace/contracts.py
index 2375a1f62..ef6384445 100644
--- a/packages/core/src/repowise/core/workspace/contracts.py
+++ b/packages/core/src/repowise/core/workspace/contracts.py
@@ -376,19 +376,106 @@ def _build_candidate_index(
return candidate_index
+# Hosts that name the local machine, not a specific service. A consumer URL
+# pointing here carries no service identity, so it is resolved by path uniqueness
+# rather than excluded as third-party.
+_LOCALHOST_HOSTS = frozenset({"localhost", "127.0.0.1", "0.0.0.0", "::1"})
+
+# Internal-DNS suffixes whose leading label names a service (k8s / mesh / LAN).
+# Only for these do we map ``<label>.<suffix>`` to a repo alias; a public host
+# like ``backend.stripe.com`` must never be mistaken for the ``backend`` repo.
+_INTERNAL_HOST_SUFFIXES = (".local", ".internal", ".svc.cluster.local")
+
+
+def _resolve_consumer_target(
+ consumer: Contract,
+ repo_aliases: set[str],
+ service_bases: dict[str, str],
+) -> tuple[str | None, bool]:
+ """Resolve a consumer's target repo and whether it is a third-party call.
+
+ Returns ``(target_repo, is_external)``:
+
+ - ``service_bases`` (host or ``${BASE}`` token) wins first;
+ - a host equal to a workspace repo alias, or an internal-DNS host whose
+ leading label is a repo alias (``backend.svc.cluster.local``), is internal;
+ - localhost and bare unknown hostnames (e.g. a docker-compose service we
+ can't map) return ``(None, False)`` so path matching still applies;
+ - any other *public* dotted host is third-party (``is_external=True``).
+ """
+ meta = consumer.meta
+ host = meta.get("host")
+ if host:
+ if host in service_bases:
+ return service_bases[host], False
+ if host in repo_aliases:
+ return host, False
+ if host in _LOCALHOST_HOSTS:
+ return None, False
+ if host.endswith(_INTERNAL_HOST_SUFFIXES):
+ label = host.split(".")[0]
+ return (label, False) if label in repo_aliases else (None, False)
+ if "." not in host:
+ return None, False # bare unknown hostname, not necessarily third-party
+ return None, True # public dotted host, unmapped → third-party
+ token = meta.get("base_token")
+ if token and token.lower() in service_bases:
+ return service_bases[token.lower()], False
+ return None, False
+
+
+def annotate_consumer_targets(
+ contracts: list[Contract],
+ service_bases: dict[str, str] | None = None,
+) -> None:
+ """Stamp each HTTP consumer's ``meta`` with its resolved target / external bit.
+
+ Mutates the contracts in place so both :func:`match_contracts` and the
+ diagnostics builder read one resolution. ``service_bases`` maps a base token
+ or host (case-insensitive) to a repo alias.
+ """
+ repo_aliases = {c.repo for c in contracts}
+ sb = {k.lower(): v for k, v in (service_bases or {}).items()}
+ for c in contracts:
+ if c.role != "consumer" or c.contract_type != "http":
+ continue
+ target, external = _resolve_consumer_target(c, repo_aliases, sb)
+ if external:
+ c.meta["external"] = True
+ elif target:
+ c.meta["target_repo"] = target
+
+
+def _prefer_target_repo(providers: list[Contract], consumer: Contract) -> list[Contract]:
+ """Narrow *providers* to the consumer's resolved ``target_repo`` when set.
+
+ Falls back to the full list if the target declares no matching provider, so
+ a stale/typo'd ``service_bases`` entry never silently drops a real link.
+ """
+ target = consumer.meta.get("target_repo")
+ if not target:
+ return providers
+ preferred = [p for p in providers if p.repo == target]
+ return preferred or providers
+
+
def match_contracts(contracts: list[Contract]) -> list[ContractLink]:
"""Match providers to consumers across repos.
- Two passes:
+ Passes:
1. **Exact** — normalized contract IDs must be equal, with HTTP/gRPC
wildcard handling (``http::*::/path``, ``grpc::Service/*``).
- 2. **Candidate** — for HTTP consumers left unmatched (including those whose
- URL had an unresolved base prefix stripped during extraction), retry
- after collapsing known mount/version/base prefixes on both sides. These
- links are tagged ``match_type="candidate"`` with reduced confidence.
-
- Same-repo same-service calls are filtered out of both passes.
+ 1b. **Base-resolved** — a consumer whose URL had an unresolved base prefix
+ stripped is matched on its full (host-relative) path; the link is
+ ``exact`` when the target service is unambiguous (one matching service, or
+ a configured ``service_bases`` target) and ``candidate`` otherwise.
+ 2. **Candidate** — remaining unmatched HTTP consumers retry after collapsing
+ known mount/version/base prefixes on both sides, at reduced confidence.
+
+ Same-repo same-service calls, and consumers resolved to a third-party host
+ (``meta['external']``), are filtered from every pass. Target resolution is
+ read from ``meta`` (see :func:`annotate_consumer_targets`).
"""
provider_index: dict[str, list[Contract]] = defaultdict(list)
consumers: list[Contract] = []
@@ -397,35 +484,63 @@ def match_contracts(contracts: list[Contract]) -> list[ContractLink]:
if c.role == "provider":
key = normalize_contract_id(c.contract_id)
provider_index[key].append(c)
- else:
+ elif not c.meta.get("external"):
consumers.append(c)
links: list[ContractLink] = []
seen: set[tuple[str, str, str, str, str]] = set()
matched_consumers: set[int] = set()
- # --- Pass 1: exact / wildcard ---
+ # --- Pass 1: exact / wildcard (excludes base-stripped consumers) ---
for consumer in consumers:
- # Consumers with an unresolved base prefix can't be matched exactly —
- # defer them entirely to the candidate pass.
if consumer.meta.get("base_stripped"):
continue
+ matching_keys = _find_matching_keys(consumer.contract_id, provider_index)
+ providers = [p for k in matching_keys for p in provider_index[k]]
+ for provider in _prefer_target_repo(providers, consumer):
+ if _same_repo_same_service(provider, consumer):
+ continue
+ link = _make_link(
+ consumer,
+ provider,
+ "exact",
+ min(provider.confidence, consumer.confidence),
+ seen,
+ )
+ if link is not None:
+ links.append(link)
+ matched_consumers.add(id(consumer))
+ # --- Pass 1b: base-resolved exact path for base-stripped consumers ---
+ for consumer in consumers:
+ if id(consumer) in matched_consumers or not consumer.meta.get("base_stripped"):
+ continue
matching_keys = _find_matching_keys(consumer.contract_id, provider_index)
- for key in matching_keys:
- for provider in provider_index[key]:
- if _same_repo_same_service(provider, consumer):
- continue
- link = _make_link(
- consumer,
- provider,
- "exact",
- min(provider.confidence, consumer.confidence),
- seen,
- )
- if link is not None:
- links.append(link)
- matched_consumers.add(id(consumer))
+ providers = [
+ p
+ for k in matching_keys
+ for p in provider_index[k]
+ if not _same_repo_same_service(p, consumer)
+ ]
+ if not providers:
+ continue
+ # A config target only resolves the link when it actually narrows to a
+ # provider; a stale/typo'd target falls back to all providers and must
+ # not be treated as resolved (else an ambiguous link emits as exact).
+ target = consumer.meta.get("target_repo")
+ narrowed = [p for p in providers if p.repo == target] if target else []
+ if narrowed:
+ providers = narrowed
+ resolved = bool(narrowed) or len({(p.repo, p.service) for p in providers}) == 1
+ match_type = "exact" if resolved else "candidate"
+ for provider in providers:
+ confidence = min(provider.confidence, consumer.confidence)
+ if not resolved:
+ confidence = round(confidence * _CANDIDATE_CONFIDENCE_FACTOR, 3)
+ link = _make_link(consumer, provider, match_type, confidence, seen)
+ if link is not None:
+ links.append(link)
+ matched_consumers.add(id(consumer))
# --- Pass 2: candidate (mount/version/base prefix) for unmatched HTTP ---
candidate_index = _build_candidate_index(provider_index)
@@ -443,7 +558,7 @@ def match_contracts(contracts: list[Contract]) -> list[ContractLink]:
if core in ("", "/"):
continue
- for provider in candidate_index.get(core, []):
+ for provider in _prefer_target_repo(candidate_index.get(core, []), consumer):
if _same_repo_same_service(provider, consumer):
continue
psplit = _split_http_id(normalize_contract_id(provider.contract_id))
@@ -568,8 +683,10 @@ async def run_contract_extraction(
assign_service,
detect_service_boundaries,
)
+ from .extractors.base import make_exclude_predicate
contract_config = ws_config.contracts
+ exclude = make_exclude_predicate(tuple(contract_config.exclude_globs))
# Build repo_paths — only include repos that have been indexed
# (have a .repowise/ directory). Non-indexed repos must not participate
@@ -600,7 +717,7 @@ async def run_contract_extraction(
extractors.append(TopicExtractor())
for extractor in extractors:
- found = await asyncio.to_thread(extractor.extract, repo_path, alias)
+ found = await asyncio.to_thread(extractor.extract, repo_path, alias, exclude)
for c in found:
c.service = assign_service(c.file_path, boundaries)
contracts.extend(found)
@@ -614,7 +731,8 @@ async def run_contract_extraction(
for repo_contracts in results:
all_contracts.extend(repo_contracts)
- # Match contracts
+ # Resolve each consumer's target service / third-party host, then match.
+ annotate_consumer_targets(all_contracts, contract_config.service_bases)
links = match_contracts(all_contracts)
# Merge manual links
diff --git a/packages/core/src/repowise/core/workspace/diagnostics.py b/packages/core/src/repowise/core/workspace/diagnostics.py
index 0dda3d2cf..34f8126a4 100644
--- a/packages/core/src/repowise/core/workspace/diagnostics.py
+++ b/packages/core/src/repowise/core/workspace/diagnostics.py
@@ -49,6 +49,9 @@ class UnmatchedReason:
#: formed (e.g. an HTTP path that only the candidate pass could bridge and
#: did not). Rare; flags a potential matcher gap worth inspecting.
UNLINKED = "unlinked"
+ #: The call targets a literal third-party host (Stripe, Formspree, …) that is
+ #: not a workspace service, so it is intentionally excluded from matching.
+ EXTERNAL_HOST = "external_host"
# ---------------------------------------------------------------------------
@@ -177,6 +180,8 @@ def _classify_unmatched(
providers_by_norm_id: dict[str, list[Contract]],
) -> str:
"""Return the :class:`UnmatchedReason` for an unmatched *consumer*."""
+ if consumer.meta.get("external"):
+ return UnmatchedReason.EXTERNAL_HOST
candidates = providers_by_norm_id.get(normalize_contract_id(consumer.contract_id))
if not candidates:
return UnmatchedReason.NO_PROVIDER
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index c5da5fc1b..078a9f714 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -12,10 +12,62 @@ descend into sibling/vendored repos rooted under the workspace.
from __future__ import annotations
import os
-from collections.abc import Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping
from dataclasses import dataclass, field
+from fnmatch import fnmatch
from pathlib import Path
+# Path segments that mark a test/mock tree. A route handler or topic publisher
+# that exists only under one of these is a fixture, not a real service contract,
+# so it is excluded from contract extraction by default (configurable via the
+# workspace ``contracts.exclude_globs``). Kept to unambiguous test-only dir names:
+# singular ``test``/``spec``/``e2e`` are intentionally excluded because they
+# double as legitimate product directories (an OpenAPI ``spec/``, a ``test/``
+# feature). Test *files* under those dirs are still caught by the filename
+# patterns below.
+_TEST_DIR_SEGMENTS = frozenset({"tests", "__tests__", "__mocks__"})
+
+# Filename patterns that mark a test file regardless of directory.
+_TEST_FILE_PATTERNS = (
+ "test_*.py",
+ "*_test.py",
+ "*_test.go",
+ "*.test.*",
+ "*.spec.*",
+ "*.e2e.*",
+ "conftest.py",
+)
+
+
+def is_test_path(rel_path: str) -> bool:
+ """True when *rel_path* (POSIX) lives in a test tree or is a test file."""
+ parts = rel_path.split("/")
+ if any(seg in _TEST_DIR_SEGMENTS for seg in parts[:-1]):
+ return True
+ name = parts[-1]
+ return any(fnmatch(name, pat) for pat in _TEST_FILE_PATTERNS)
+
+
+def make_exclude_predicate(
+ extra_globs: tuple[str, ...] = (),
+ *,
+ exclude_tests: bool = True,
+) -> Callable[[str], bool]:
+ """Build a ``rel_path -> bool`` skip predicate for contract extraction.
+
+ Skips the default test/spec trees (unless *exclude_tests* is False) plus any
+ user-supplied ``extra_globs`` (matched against the full POSIX path and the
+ bare filename).
+ """
+
+ def skip(rel_path: str) -> bool:
+ if exclude_tests and is_test_path(rel_path):
+ return True
+ name = rel_path.rsplit("/", 1)[-1]
+ return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in extra_globs)
+
+ return skip
+
@dataclass(frozen=True)
class ScanContext:
@@ -36,7 +88,9 @@ class ScanContext:
def iter_source_files(
- repo_root: Path, extensions: frozenset[str]
+ repo_root: Path,
+ extensions: frozenset[str],
+ exclude: Callable[[str], bool] | None = None,
) -> Iterator[tuple[str, str, str]]:
"""Yield ``(rel_path, suffix, content)`` for each scannable source file.
@@ -67,6 +121,8 @@ def iter_source_files(
suffix = os.path.splitext(info.path)[1].lower()
if suffix not in extensions:
continue
+ if exclude is not None and exclude(info.path):
+ continue
try:
content = Path(info.abs_path).read_text(encoding="utf-8", errors="replace")
except OSError:
diff --git a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
index ae8e3ebc9..96b37ba73 100644
--- a/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/grpc/__init__.py
@@ -21,6 +21,7 @@ from .python import PythonGrpcDialect
from .typescript import TypeScriptGrpcDialect
if TYPE_CHECKING:
+ from collections.abc import Callable
from pathlib import Path
from repowise.core.workspace.contracts import Contract
@@ -49,11 +50,16 @@ class GrpcExtractor:
dialects: tuple[GrpcDialect, ...] = DIALECTS
- def extract(self, repo_path: Path, repo_alias: str = "") -> list[Contract]:
+ def extract(
+ self,
+ repo_path: Path,
+ repo_alias: str = "",
+ exclude: Callable[[str], bool] | None = None,
+ ) -> list[Contract]:
all_exts = _union_extensions(self.dialects)
contracts: list[Contract] = []
- for rel_path, suffix, content in iter_source_files(repo_path, all_exts):
+ for rel_path, suffix, content in iter_source_files(repo_path, all_exts, exclude):
ctx = ScanContext(repo_alias, rel_path, suffix, content)
for dialect in self.dialects:
if suffix in dialect.extensions:
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
index a92e6e627..491a3925c 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
@@ -29,6 +29,7 @@ from .rust_clients import RustClientsDialect
from .spring import SpringDialect
if TYPE_CHECKING:
+ from collections.abc import Callable
from pathlib import Path
from repowise.core.workspace.contracts import Contract
@@ -66,7 +67,12 @@ class HttpExtractor:
provider_dialects: tuple[HttpDialect, ...] = PROVIDER_DIALECTS
consumer_dialects: tuple[HttpDialect, ...] = CONSUMER_DIALECTS
- def extract(self, repo_path: Path, repo_alias: str = "") -> list[Contract]:
+ def extract(
+ self,
+ repo_path: Path,
+ repo_alias: str = "",
+ exclude: Callable[[str], bool] | None = None,
+ ) -> list[Contract]:
"""Scan all source files in *repo_path* and return Contract instances.
Files are read once into memory so a first pass can collect repo-wide
@@ -77,7 +83,7 @@ class HttpExtractor:
self.consumer_dialects
)
- files = list(iter_source_files(repo_path, all_exts))
+ files = list(iter_source_files(repo_path, all_exts, exclude))
mounts = self._collect_mounts(files)
contracts: list[Contract] = []
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py b/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
index 15f8810f4..690e3e8d9 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
@@ -70,11 +70,11 @@ class CSharpHttpDialect:
url = _to_template(prefix, text)
if "/" not in url:
return # Not a URL path — skip non-route strings.
- out.append(
- build_consumer_contract(
- ctx, method=method.upper(), url=url, client=client, confidence=0.70
- )
+ c = build_consumer_contract(
+ ctx, method=method.upper(), url=url, client=client, confidence=0.70
)
+ if c is not None:
+ out.append(c)
for m in _WRAPPER_RE.finditer(content):
emit(m.group(1), m.group(2), m.group(3), "httpclient")
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/dialect.py b/packages/core/src/repowise/core/workspace/extractors/http/dialect.py
index 36f92990a..18606dd7f 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/dialect.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/dialect.py
@@ -13,8 +13,10 @@ from typing import TYPE_CHECKING, Protocol, runtime_checkable
from ..base import ScanContext
from .paths import (
+ absolute_host,
consumer_meta,
extract_path_from_url,
+ is_unusable_consumer_path,
normalize_http_path,
strip_leading_base_expr,
)
@@ -96,13 +98,21 @@ def build_consumer_contract(
url: str,
client: str,
confidence: float = 0.75,
-) -> Contract:
- """Build a consumer contract from a raw client-call URL."""
+) -> Contract | None:
+ """Build a consumer contract from a raw client-call URL.
+
+ Returns ``None`` for URLs that can never be a meaningful match key — a
+ truncated template literal or a path with no concrete segment (see
+ :func:`is_unusable_consumer_path`).
+ """
from repowise.core.workspace.contracts import Contract
+ host = absolute_host(url)
path = extract_path_from_url(url)
- path, base_stripped = strip_leading_base_expr(path)
+ path, base_token = strip_leading_base_expr(path)
norm_path = normalize_http_path(path)
+ if is_unusable_consumer_path(norm_path):
+ return None
return Contract(
repo=ctx.repo_alias,
contract_id=f"http::{method}::{norm_path}",
@@ -112,5 +122,5 @@ def build_consumer_contract(
symbol_name=f"{client}:{method} {url}",
confidence=confidence,
service=None,
- meta=consumer_meta(method, norm_path, client, base_stripped),
+ meta=consumer_meta(method, norm_path, client, base_token, host),
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
index b2ca794e8..67f84f5a3 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
@@ -59,13 +59,16 @@ class JsClientsDialect:
content = ctx.content
out: list[Contract] = []
+ def emit(*, method: str, url: str, client: str, confidence: float = 0.75) -> None:
+ c = build_consumer_contract(
+ ctx, method=method, url=url, client=client, confidence=confidence
+ )
+ if c is not None:
+ out.append(c)
+
# fetch() with an explicit method.
for m in _FETCH_METHOD_RE.finditer(content):
- out.append(
- build_consumer_contract(
- ctx, method=m.group(2).upper(), url=m.group(1), client="fetch"
- )
- )
+ emit(method=m.group(2).upper(), url=m.group(1), client="fetch")
# fetch() without a method → GET, skipping URLs already matched above.
method_urls = {m.group(1) for m in _FETCH_METHOD_RE.finditer(content)}
@@ -73,15 +76,11 @@ class JsClientsDialect:
url = m.group(1)
if url in method_urls:
continue
- out.append(build_consumer_contract(ctx, method="GET", url=url, client="fetch"))
+ emit(method="GET", url=url, client="fetch")
# axios.<method>()
for m in _AXIOS_RE.finditer(content):
- out.append(
- build_consumer_contract(
- ctx, method=m.group(1).upper(), url=m.group(2), client="axios"
- )
- )
+ emit(method=m.group(1).upper(), url=m.group(2), client="axios")
# Wrapper calls — fetchJSON(`${BASE}/path`, { method: "POST" }) etc.
for m in _WRAPPER_CALL_RE.finditer(content):
@@ -95,10 +94,6 @@ class JsClientsDialect:
if not (_HTTP_NAME_RE.search(callee) or method_opt):
continue
method = method_opt.group(1).upper() if method_opt else "GET"
- out.append(
- build_consumer_contract(
- ctx, method=method, url=m.group(2), client="wrapper", confidence=0.65
- )
- )
+ emit(method=method, url=m.group(2), client="wrapper", confidence=0.65)
return out
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/paths.py b/packages/core/src/repowise/core/workspace/extractors/http/paths.py
index b24535b3c..4bb777423 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/paths.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/paths.py
@@ -41,51 +41,120 @@ def normalize_http_path(path: str) -> str:
return s or "/"
+def _after_authority(url: str) -> str | None:
+ """Return the portion of *url* after scheme+authority, or ``None`` if relative.
+
+ Handles both ``scheme://host/path`` and protocol-relative ``//host/path``.
+ """
+ if url.startswith("//"):
+ return url[2:]
+ if "://" in url:
+ return url.split("://", 1)[1]
+ return None
+
+
def extract_path_from_url(url: str) -> str:
"""Extract just the path portion from a URL, stripping scheme and host."""
- # If it starts with http:// or https://, strip scheme + authority
- if "://" in url:
- after_scheme = url.split("://", 1)[1]
- slash_idx = after_scheme.find("/")
- if slash_idx >= 0:
- return after_scheme[slash_idx:]
- return "/"
- return url
+ after = _after_authority(url)
+ if after is None:
+ return url
+ slash_idx = after.find("/")
+ return after[slash_idx:] if slash_idx >= 0 else "/"
+
+
+def absolute_host(url: str) -> str | None:
+ """Return the lower-cased host (sans port) of an absolute URL, else ``None``.
+
+ ``https://formspree.io/f/x`` -> ``formspree.io``; ``http://backend:8000/api``
+ -> ``backend``; protocol-relative ``//cdn.example.com/x`` -> ``cdn.example.com``.
+ A relative path (``/api/users``) or a base placeholder (``${API_BASE}/users``)
+ has no host and returns ``None``.
+ """
+ after = _after_authority(url)
+ if after is None:
+ return None
+ authority = after.split("/", 1)[0]
+ if "$" in authority or "{" in authority:
+ return None # the host is itself an unresolved expression, not a literal
+ return authority.split(":", 1)[0].lower() or None
# A leading base/host placeholder in a consumer URL, e.g. ``${API_BASE}/users``
-# or ``${apiUrl}/v1/users``. The value is unresolved at extraction time, so the
-# only thing we can match on is the concrete suffix. Anchored at the start so
-# interior expressions (real path params like ``/users/${id}``) are untouched.
-_LEADING_BASE_EXPR_RE = re.compile(r"^\s*\$\{[^}]+\}")
+# or ``${apiUrl}/v1/users``. The captured group is the inner expression, used to
+# resolve the call's target service (see :func:`base_token_identifier`). Anchored
+# at the start so interior expressions (real path params like ``/users/${id}``)
+# are untouched.
+_LEADING_BASE_EXPR_RE = re.compile(r"^\s*\$\{([^}]+)\}")
-def strip_leading_base_expr(path: str) -> tuple[str, bool]:
+def strip_leading_base_expr(path: str) -> tuple[str, str]:
"""Strip a leading base/host placeholder from a consumer URL path.
- Returns ``(path, stripped)``. ``${API_BASE}/users`` becomes ``/users`` with
- ``stripped=True``. Treating the placeholder as an ordinary ``{param}`` path
- segment (which is what :func:`normalize_http_path` would otherwise do)
- prevents the route from ever lining up with a concrete provider path, so we
- drop it and let the matcher link on the suffix as a lower-confidence
- candidate instead.
+ Returns ``(path, base_token)``. ``${API_BASE}/users`` becomes
+ ``("/users", "API_BASE")``; a path with no leading placeholder returns
+ ``(path, "")``. Treating the placeholder as an ordinary ``{param}`` segment
+ (which is what :func:`normalize_http_path` would otherwise do) prevents the
+ route from lining up with a concrete provider path, so we drop it and let the
+ matcher resolve the call's target service from the suffix and the token.
"""
- new_path = _LEADING_BASE_EXPR_RE.sub("", path, count=1)
- if new_path == path:
- return path, False
+ m = _LEADING_BASE_EXPR_RE.match(path)
+ if m is None:
+ return path, ""
+ new_path = path[m.end() :]
if not new_path.startswith("/"):
new_path = "/" + new_path
- return new_path, True
+ return new_path, m.group(1).strip()
-def consumer_meta(method: str, norm_path: str, client: str, base_stripped: bool) -> dict:
+_IDENTIFIER_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+
+def base_token_identifier(base_token: str) -> str:
+ """Reduce a base expression to its trailing identifier for config lookup.
+
+ ``import.meta.env.VITE_API_URL`` -> ``VITE_API_URL``; ``process.env.API_BASE``
+ -> ``API_BASE``; ``apiUrl`` -> ``apiUrl``. Empty when the token holds no
+ identifier (e.g. a bare string concatenation).
+ """
+ ids = _IDENTIFIER_RE.findall(base_token)
+ return ids[-1] if ids else ""
+
+
+def is_unusable_consumer_path(path: str) -> bool:
+ """True when a normalized consumer path can never be a meaningful match key.
+
+ Two cases, both pure noise rather than a real cross-repo call:
+
+ - an **unbalanced** ``${`` survived normalization (a template literal whose
+ capture was truncated by a nested quote, e.g. ``/repos/explore${qs``);
+ - the path has **no concrete segment** — every segment is ``{param}`` (or it
+ is bare ``/``), so it would match indiscriminately.
+ """
+ if "${" in path:
+ return True
+ segments = [s for s in path.split("/") if s]
+ return not any(s != "{param}" for s in segments)
+
+
+def consumer_meta(
+ method: str,
+ norm_path: str,
+ client: str,
+ base_token: str = "",
+ host: str | None = None,
+) -> dict:
"""Build a consumer contract's ``meta`` dict.
- ``base_stripped`` is recorded only when True so the matcher knows the path
- had an unresolved base prefix removed and should be linked as a candidate
- rather than an exact match.
+ ``base_stripped`` / ``base_token`` record an unresolved base placeholder so
+ the matcher resolves the call's target service from the suffix; ``host``
+ records a literal absolute host so the matcher can exclude third-party APIs.
"""
- meta = {"method": method, "path": norm_path, "client": client}
- if base_stripped:
+ meta: dict = {"method": method, "path": norm_path, "client": client}
+ if base_token:
meta["base_stripped"] = True
+ ident = base_token_identifier(base_token)
+ if ident:
+ meta["base_token"] = ident
+ if host:
+ meta["host"] = host
return meta
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
index 196c33d6f..52302617d 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
@@ -27,9 +27,9 @@ class PythonClientsDialect:
def extract(self, ctx: ScanContext) -> list[Contract]:
out: list[Contract] = []
for m in _REQUESTS_RE.finditer(ctx.content):
- out.append(
- build_consumer_contract(
- ctx, method=m.group(1).upper(), url=m.group(2), client="requests"
- )
+ c = build_consumer_contract(
+ ctx, method=m.group(1).upper(), url=m.group(2), client="requests"
)
+ if c is not None:
+ out.append(c)
return out
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
index dc6a6275a..87db3e180 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
@@ -52,11 +52,11 @@ class RustClientsDialect:
def emit(method: str, url: str) -> None:
if "/" not in url:
return # Not a URL path — skip map/collection .get("key") calls.
- out.append(
- build_consumer_contract(
- ctx, method=method.upper(), url=url, client="reqwest", confidence=0.65
- )
+ c = build_consumer_contract(
+ ctx, method=method.upper(), url=url, client="reqwest", confidence=0.65
)
+ if c is not None:
+ out.append(c)
for rx in (_METHOD_LIT_RE, _FREE_LIT_RE):
for m in rx.finditer(content):
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
index 4d6c447cc..7466036cd 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic_extractor.py
@@ -17,6 +17,8 @@ from repowise.core.ingestion.languages.registry import REGISTRY as _LANG_REGISTR
from .base import iter_source_files
if TYPE_CHECKING:
+ from collections.abc import Callable
+
from repowise.core.workspace.contracts import Contract
_log = logging.getLogger("repowise.workspace.extractors.topic")
@@ -220,7 +222,12 @@ _ALL_PATTERNS = _KAFKA_PATTERNS + _RABBITMQ_PATTERNS + _NATS_PATTERNS
class TopicExtractor:
"""Extract message topic/queue contracts from source files."""
- def extract(self, repo_path: Path, repo_alias: str = "") -> list[Contract]:
+ def extract(
+ self,
+ repo_path: Path,
+ repo_alias: str = "",
+ exclude: Callable[[str], bool] | None = None,
+ ) -> list[Contract]:
from repowise.core.workspace.contracts import Contract
contracts: list[Contract] = []
@@ -229,7 +236,7 @@ class TopicExtractor:
# File discovery is gitignore- and nested-repo-aware (see
# ``iter_source_files``) so a workspace repo whose root contains nested
# repos or large ignored trees is not scanned end-to-end.
- for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS):
+ for rel_path, _suffix, content in iter_source_files(repo_path, _EXTENSIONS, exclude):
for pdef in _ALL_PATTERNS:
for match in pdef.regex.finditer(content):
topic_name = match.group(pdef.topic_group).strip()
diff --git a/tests/unit/workspace/test_contracts.py b/tests/unit/workspace/test_contracts.py
index b38e4b094..d592c3c60 100644
--- a/tests/unit/workspace/test_contracts.py
+++ b/tests/unit/workspace/test_contracts.py
@@ -12,6 +12,7 @@ from repowise.core.workspace.contracts import (
Contract,
ContractLink,
ContractStore,
+ annotate_consumer_targets,
load_contract_store,
match_contracts,
normalize_contract_id,
@@ -450,6 +451,59 @@ class TestHttpExtractor:
contracts = HttpExtractor().extract(tmp_path, "svc")
assert contracts == []
+ def test_truncated_template_literal_dropped(self, tmp_path: Path) -> None:
+ # A nested quote inside the template expression truncates the capture,
+ # leaving an unbalanced ``${`` — this is noise and must not be emitted.
+ self._write_file(tmp_path, "src/api.ts", """
+ fetchJSON(`/repos/explore${qs ? '?' + qs : ''}`);
+ """)
+ consumers = [
+ c for c in HttpExtractor().extract(tmp_path, "frontend") if c.role == "consumer"
+ ]
+ assert consumers == []
+
+ def test_bare_param_path_dropped(self, tmp_path: Path) -> None:
+ # fetch(`${x}`) collapses to a single {param} segment — no concrete path
+ # to match on, so it is dropped rather than linked indiscriminately.
+ self._write_file(tmp_path, "src/api.ts", """
+ await fetch(`${endpoint}`);
+ """)
+ consumers = [
+ c for c in HttpExtractor().extract(tmp_path, "frontend") if c.role == "consumer"
+ ]
+ assert consumers == []
+
+ def test_absolute_host_recorded_in_meta(self, tmp_path: Path) -> None:
+ # An absolute third-party URL keeps its host so the matcher can exclude it.
+ self._write_file(tmp_path, "src/api.ts", """
+ fetch('https://formspree.io/f/mkopzvak', { method: 'POST' });
+ """)
+ consumers = [
+ c for c in HttpExtractor().extract(tmp_path, "frontend") if c.role == "consumer"
+ ]
+ assert len(consumers) == 1
+ assert consumers[0].meta.get("host") == "formspree.io"
+
+ def test_test_directory_files_excluded(self, tmp_path: Path) -> None:
+ # A route defined only under tests/ is a fixture, not a service contract.
+ self._write_file(tmp_path, "tests/test_routes.py", """
+ @app.get("/fixture-only")
+ def fixture(): ...
+ """)
+ self._write_file(tmp_path, "app/routes.py", """
+ @app.get("/real")
+ def real(): ...
+ """)
+ from repowise.core.workspace.extractors.base import make_exclude_predicate
+
+ ids = {
+ c.contract_id
+ for c in HttpExtractor().extract(tmp_path, "svc", make_exclude_predicate())
+ if c.role == "provider"
+ }
+ assert "http::GET::/real" in ids
+ assert "http::GET::/fixture-only" not in ids
+
# ---------------------------------------------------------------------------
# GrpcExtractor
@@ -917,9 +971,9 @@ class TestCandidateMatching:
assert len(links) == 1
assert links[0].match_type == "candidate"
- def test_base_stripped_consumer_is_candidate_not_exact(self) -> None:
- # Even when the suffix matches exactly, a stripped base means we can't
- # be certain of the mount point — so it is a candidate, not exact.
+ def test_base_stripped_consumer_unique_service_is_exact(self) -> None:
+ # A stripped base whose full path matches exactly one provider service is
+ # unambiguous: the base can only be that service, so the link is exact.
contracts = [
self._c(repo="backend", role="provider", contract_id="http::GET::/resource"),
self._c(repo="frontend", role="consumer", contract_id="http::GET::/resource",
@@ -927,7 +981,35 @@ class TestCandidateMatching:
]
links = match_contracts(contracts)
assert len(links) == 1
- assert links[0].match_type == "candidate"
+ assert links[0].match_type == "exact"
+
+ def test_base_stripped_consumer_ambiguous_is_candidate(self) -> None:
+ # The same stripped path provided by two distinct services is ambiguous —
+ # we can't tell which one the base points at, so both links are candidate.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="payments", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="frontend", role="consumer", contract_id="http::GET::/resource",
+ file_path="c.ts", meta={"base_stripped": True}),
+ ]
+ links = match_contracts(contracts)
+ assert len(links) == 2
+ assert all(lk.match_type == "candidate" for lk in links)
+
+ def test_base_stripped_consumer_resolved_by_service_bases(self) -> None:
+ # An ambiguous stripped path is promoted to exact (and narrowed to the
+ # configured repo) once a base token resolves the target service.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="payments", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="frontend", role="consumer", contract_id="http::GET::/resource",
+ file_path="c.ts", meta={"base_stripped": True, "base_token": "API_BASE"}),
+ ]
+ annotate_consumer_targets(contracts, {"API_BASE": "backend"})
+ links = match_contracts(contracts)
+ assert len(links) == 1
+ assert links[0].match_type == "exact"
+ assert links[0].provider_repo == "backend"
def test_exact_match_takes_precedence_over_candidate(self) -> None:
contracts = [
@@ -948,6 +1030,69 @@ class TestCandidateMatching:
links = match_contracts(contracts)
assert links == []
+ def test_external_host_consumer_excluded(self) -> None:
+ # A consumer pointing at a literal third-party host (not a workspace
+ # repo) is excluded from matching even when a path-equal provider exists.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::POST::/f/mkopzvak"),
+ self._c(repo="frontend", role="consumer", contract_id="http::POST::/f/mkopzvak",
+ file_path="c.ts", meta={"host": "formspree.io"}),
+ ]
+ annotate_consumer_targets(contracts)
+ assert any(c.meta.get("external") for c in contracts if c.role == "consumer")
+ assert match_contracts(contracts) == []
+
+ def test_absolute_host_matching_repo_alias_links(self) -> None:
+ # http://backend:8000/... resolves to the workspace 'backend' repo and
+ # links normally (target host is a service, not a third party).
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/api/data"),
+ self._c(repo="worker", role="consumer", contract_id="http::GET::/api/data",
+ file_path="c.py", meta={"host": "backend"}),
+ ]
+ annotate_consumer_targets(contracts)
+ links = match_contracts(contracts)
+ assert len(links) == 1
+ assert links[0].provider_repo == "backend"
+
+ def test_public_subdomain_not_mistaken_for_repo_alias(self) -> None:
+ # backend.stripe.com must be external even though 'backend' is a repo —
+ # only the full host or an internal-DNS leading label maps to an alias.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/v1/charges"),
+ self._c(repo="frontend", role="consumer", contract_id="http::GET::/v1/charges",
+ file_path="c.ts", meta={"host": "backend.stripe.com"}),
+ ]
+ annotate_consumer_targets(contracts)
+ assert any(c.meta.get("external") for c in contracts if c.role == "consumer")
+ assert match_contracts(contracts) == []
+
+ def test_internal_dns_host_resolves_to_alias(self) -> None:
+ # k8s service DNS backend.default.svc.cluster.local -> the 'backend' repo.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/api/data"),
+ self._c(repo="worker", role="consumer", contract_id="http::GET::/api/data",
+ file_path="c.py", meta={"host": "backend.default.svc.cluster.local"}),
+ ]
+ annotate_consumer_targets(contracts)
+ links = match_contracts(contracts)
+ assert len(links) == 1
+ assert links[0].provider_repo == "backend"
+
+ def test_base_stripped_stale_service_base_stays_candidate(self) -> None:
+ # A service_bases target that matches no provider must fall back to the
+ # ambiguity rule, not emit exact links to every service.
+ contracts = [
+ self._c(repo="backend", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="payments", role="provider", contract_id="http::GET::/resource"),
+ self._c(repo="frontend", role="consumer", contract_id="http::GET::/resource",
+ file_path="c.ts", meta={"base_stripped": True, "base_token": "API_BASE"}),
+ ]
+ annotate_consumer_targets(contracts, {"API_BASE": "typo_repo"})
+ links = match_contracts(contracts)
+ assert len(links) == 2
+ assert all(lk.match_type == "candidate" for lk in links)
+
def test_method_mismatch_no_candidate(self) -> None:
contracts = [
self._c(repo="backend", role="provider", contract_id="http::POST::/v1/resource"),