Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

fix(workspace): gate every contract dialect once per file and drop matches in comments (#4505)

Raghav Chamadiyaf1a2a712e898
diff --git a/docs/architecture/language-support.md b/docs/architecture/language-support.md
index 812387e84..735e57799 100644
--- a/docs/architecture/language-support.md
+++ b/docs/architecture/language-support.md
@@ -1290,6 +1290,18 @@ differently (HTTP mount prefixes, a queue bound to an exchange, a pattern
subscription) registers its extra passes, or its own `keys`, in
`matching.MATCHERS`.
+Every dialect declares `requires`: tokens one of which a file must hold, chosen
+so that every match needs one (a contract format such as `.proto` declares
+`EVERY_FILE`). `dialect.run_dialect` checks the gate, looking each token up
+once per file, and drops a contract whose line is only a comment. `call_sites`,
+`code_matches`, `outside_comments` and `consumer_contracts` skip a match that
+starts in a comment, so a commented-out decorator sets no state either.
+`test_dialect_registry.py` fails on an ungated entry, and
+`test_extraction_dialect_budget.py` on a dialect that costs as much as an
+ungated scan. A Python docstring example is still read, because the mask keeps
+strings, and Ruby, VB and SQL files are not masked: their comment syntax would
+mislead the masker.
+
| Contract | Providers | Consumers |
|----------|-----------|-----------|
| **HTTP** | Express, Hono, Fastify, Koa, Elysia, NestJS, Next.js, Remix, FastAPI, Flask, Django, Spring, JAX-RS, Micronaut, Laravel, Go, ASP.NET, Axum/Actix/Rocket, OpenAPI 3.x | Client calls in JS/TS, Python, PHP, Java, Kotlin, Go, C#, Ruby and Rust; JS/TS client instances carry their base URL across files |
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index 56c29e5f2..3730abb74 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -20,15 +20,17 @@ from typing import TYPE_CHECKING
from repowise.core.test_paths import is_skippable_test_path
+from .langs import APEX, CSHARP, GO, JAVA, JS_TS, KOTLIN, PHP, PROTO, RUST
+
if TYPE_CHECKING:
from repowise.core.workspace.repo_index import RepoIndex
# Repowise's own contract-extractor sources. Their docstrings and comments spell
# out the syntax each dialect matches (``@app.get("/path")``, ``topic::orders``,
# ``AuthServiceStub``), so scanning them yields contracts for endpoints and
-# queues that do not exist — on this repo, 14 of them. A regex over raw text
-# cannot tell an example in a comment from a live route; excluding the tree that
-# is nothing *but* examples is the cheap half of that fix.
+# queues that do not exist — on this repo, 14 of them. A match in a comment is
+# dropped (see :meth:`ScanContext.in_comment`), but one in a docstring is a
+# string to the mask; excluding the tree that is nothing *but* examples covers it.
_SELF_EXCLUDE_GLOBS = ("*/repowise/core/workspace/extractors/*",)
@@ -66,6 +68,43 @@ def make_exclude_predicate(
return skip
+#: Where comments are ``//`` and ``/* */`` and ``#`` is code (a C# directive, a
+#: Rust or PHP attribute). Ruby is left out: its ``/.../`` regex literals would
+#: open a comment the masker cannot close. VB and SQL are left out too. A match
+#: in an unmasked file is never treated as commented out.
+_SLASH_COMMENT_SUFFIXES = JAVA | KOTLIN | GO | CSHARP | RUST | PHP | APEX | PROTO | {".prisma"}
+
+
+def _comment_mask(suffix: str, content: str) -> str | None:
+ """*content* with its comments blanked and offsets kept; None for an unknown syntax."""
+ from repowise.core.ingestion.source_mask import mask_source
+
+ # The JS-aware masker reads regex literals and Python string prefixes.
+ if suffix == ".py":
+ return mask_source(content, suffix)
+ if suffix in JS_TS:
+ return mask_source(content, ".js")
+ if suffix not in _SLASH_COMMENT_SUFFIXES:
+ return None
+ from repowise.core.analysis.security.masking import mask_comments_and_strings, mask_dialect
+
+ return mask_comments_and_strings(
+ content, strings=False, hash_comments=False, **mask_dialect(suffix)
+ )
+
+
+class _FileMemo:
+ """What one file's dialects share: each gate token's presence, and the comment mask."""
+
+ __slots__ = ("lines", "mask", "masked", "tokens")
+
+ def __init__(self) -> None:
+ self.tokens: dict[str, bool] = {}
+ self.masked = False
+ self.mask: str | None = None
+ self.lines: tuple[list[str], list[str]] | None = None
+
+
@dataclass(frozen=True)
class ScanContext:
"""A single source file presented to a dialect.
@@ -79,6 +118,11 @@ class ScanContext:
``index`` is the repo's read-only symbol table, when the repo has one.
Every dialect may ignore it and read ``content`` exactly as before.
+
+ Gates and comments are read here once per file, however many dialects
+ ask: :meth:`has_any` memoizes each token, and the comment mask is built
+ the first time a match asks about it, so a file nothing matches in never
+ pays for it.
"""
repo_alias: str
@@ -87,6 +131,51 @@ class ScanContext:
content: str
mounts: Mapping[str, str] = field(default_factory=dict)
index: RepoIndex | None = None
+ _memo: _FileMemo = field(default_factory=_FileMemo, init=False, repr=False, compare=False)
+
+ def has_any(self, tokens: tuple[str, ...]) -> bool:
+ """Whether the file contains one of *tokens*, each searched for once per file."""
+ seen = self._memo.tokens
+ for token in tokens:
+ found = seen.get(token)
+ if found is None:
+ found = seen[token] = token in self.content
+ if found:
+ return True
+ return False
+
+ def _mask(self) -> str | None:
+ memo = self._memo
+ if not memo.masked:
+ memo.mask = _comment_mask(self.suffix, self.content)
+ memo.masked = True
+ return memo.mask
+
+ def in_comment(self, offset: int) -> bool:
+ """Whether a match at *offset* (leading whitespace skipped) sits in a comment."""
+ content = self.content
+ end = len(content)
+ while offset < end and content[offset] in " \t\n":
+ offset += 1
+ if offset >= end:
+ return False
+ mask = self._mask()
+ return mask is not None and mask[offset] != content[offset]
+
+ def comment_only_line(self, line: int | None) -> bool:
+ """Whether 1-based *line* holds a comment and nothing else."""
+ if line is None or line < 1:
+ return False
+ mask = self._mask()
+ if mask is None:
+ return False
+ memo = self._memo
+ if memo.lines is None:
+ memo.lines = (mask.split("\n"), self.content.split("\n"))
+ code, source = memo.lines
+ if line > len(code) or code[line - 1].strip():
+ return False
+ return bool(source[line - 1].strip())
# One scanned source file: ``(rel_path, suffix, content)``.
diff --git a/packages/core/src/repowise/core/workspace/extractors/calls.py b/packages/core/src/repowise/core/workspace/extractors/calls.py
index 29a46d11f..c77b0703c 100644
--- a/packages/core/src/repowise/core/workspace/extractors/calls.py
+++ b/packages/core/src/repowise/core/workspace/extractors/calls.py
@@ -14,7 +14,7 @@ from __future__ import annotations
import re
from dataclasses import dataclass
-from typing import TYPE_CHECKING, Protocol
+from typing import TYPE_CHECKING, Protocol, TypeVar
from .strings import (
call_arguments,
@@ -33,6 +33,13 @@ if TYPE_CHECKING:
from .strings import Arg, StringSyntax
+class _HasOffset(Protocol):
+ offset: int
+
+
+_Located = TypeVar("_Located", bound=_HasOffset)
+
+
class CallShape(Protocol):
"""What :func:`call_sites` reads from a table row."""
@@ -109,6 +116,24 @@ def file_strings(ctx: ScanContext) -> FileStrings | None:
return FileStrings(ctx.content, syntax) if syntax is not None else None
+def code_matches(ctx: ScanContext, pattern: re.Pattern[str]) -> Iterator[re.Match[str]]:
+ """Every match of *pattern* in *ctx*'s file that does not start in a comment.
+
+ The one match loop a recogniser should use instead of a bare ``finditer``:
+ the comment check is the file's, built once and only when something matched.
+ """
+ for m in pattern.finditer(ctx.content):
+ if not ctx.in_comment(m.start()):
+ yield m
+
+
+def outside_comments(ctx: ScanContext, found: Iterable[_Located]) -> Iterator[_Located]:
+ """The matches in *found* (a shared recogniser's, carrying ``offset``) not in a comment."""
+ for item in found:
+ if not ctx.in_comment(item.offset):
+ yield item
+
+
def call_sites(
ctx: ScanContext,
calls: Iterable[CallShape],
@@ -118,20 +143,19 @@ def call_sites(
) -> Iterator[tuple[CallShape, list[str], re.Match[str], FileStrings]]:
"""Each call in *ctx* one of *calls* recognises, with its argument list.
- A call with no arguments is skipped unless *empty* is set: for a decorator
- such as ``@Get()`` the missing argument is the answer. Pass *strings* when
- the caller already holds the file's, so its constants are read once.
+ A row is tried only on a file holding one of its ``requires`` tokens,
+ each looked up once per file across every dialect, and a call in a
+ comment is skipped. A call with no arguments is skipped unless *empty* is
+ set: for a decorator such as ``@Get()`` the missing argument is the
+ answer. Pass *strings* when the caller already holds the file's, so its
+ constants are read once.
"""
- present: dict[tuple[str, ...], bool] = {} # rows of one library share a gate
for call in calls:
if ctx.suffix not in call.extensions:
continue
- if call.requires:
- if call.requires not in present:
- present[call.requires] = any(r in ctx.content for r in call.requires)
- if not present[call.requires]:
- continue
- for m in call.head.finditer(ctx.content):
+ if call.requires and not ctx.has_any(call.requires):
+ continue
+ for m in code_matches(ctx, call.head):
args = call_arguments(ctx.content, m.end() - 1)
if args is None or not (args or empty):
continue
@@ -265,8 +289,10 @@ __all__ = [
"FileStrings",
"call_chain",
"call_sites",
+ "code_matches",
"decorated_member",
"file_strings",
+ "outside_comments",
"receiver_at",
"receiver_name",
"typed_receivers",
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/ddl.py b/packages/core/src/repowise/core/workspace/extractors/data/ddl.py
index 2a48be54d..7257c9fb0 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/ddl.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/ddl.py
@@ -39,6 +39,7 @@ class DdlDialect:
name = "ddl"
extensions = frozenset({".sql"})
+ requires = ("CREATE", "create", "Create", "ALTER", "alter", "Alter")
def extract(self, ctx: ScanContext) -> list[Contract]:
try:
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/drizzle.py b/packages/core/src/repowise/core/workspace/extractors/data/drizzle.py
index b1475ff26..6895e4d9c 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/drizzle.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/drizzle.py
@@ -40,11 +40,10 @@ def _schema_tables(names: tuple[str, ...]) -> tuple[Call, ...]:
class DrizzleDialect:
name = "drizzle"
extensions = JS_TS
+ requires = (_DRIZZLE,)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if _DRIZZLE not in content:
- return []
found: list[tuple[str, float, int]] = []
schemas: dict[str, str] = {}
for call, args, m, strings in call_sites(ctx, (_TABLE, _SCHEMA)):
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/knex.py b/packages/core/src/repowise/core/workspace/extractors/data/knex.py
index 1003a11a4..a1c042bbe 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/knex.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/knex.py
@@ -81,11 +81,9 @@ def _verb(links: list[str]) -> str:
class KnexDialect:
name = "knex"
extensions = JS_TS
+ requires = ("knex", "Knex")
def extract(self, ctx: ScanContext) -> list[Contract]:
- content = ctx.content
- if "knex" not in content and "Knex" not in content:
- return []
return self._migrations(ctx) + self._queries(ctx)
def _migrations(self, ctx: ScanContext) -> list[Contract]:
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/laravel.py b/packages/core/src/repowise/core/workspace/extractors/data/laravel.py
index 9f7c744a3..e83841f42 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/laravel.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/laravel.py
@@ -20,6 +20,7 @@ from repowise.core.ingestion.framework_routes import scan_code
from repowise.core.workspace.origins import ContractOrigin
from ..base import ScanContext, line_at
+from ..calls import code_matches
from ..langs import PHP
from .dialect import (
dedup_consumers,
@@ -41,6 +42,7 @@ class EloquentDialect:
name = "eloquent"
extensions = PHP
+ requires = ("$table", "Model", "Authenticatable")
_TABLE_RE = re.compile(r"protected\s+\$table\s*=\s*['\"](\w+)['\"]")
# Only the framework's own bases: a model extending an app base class
@@ -72,14 +74,13 @@ class LaravelMigrationDialect:
name = "laravel-migration"
extensions = PHP
+ requires = ("Schema::",) # far cheaper than the regexes, which have no literal prefix
_HEAD = r"\bSchema::(?:connection\s*\([^)]*\)\s*->\s*)?"
_CREATE_RE = re.compile(_HEAD + r"create\s*\(\s*['\"]([\w.]+)['\"]")
_ALTER_RE = re.compile(_HEAD + r"table\s*\(\s*['\"]([\w.]+)['\"]")
def extract(self, ctx: ScanContext) -> list[Contract]:
- if "Schema::" not in ctx.content:
- return [] # far cheaper than the regexes, which have no literal prefix
created = found_names(self._CREATE_RE, ctx.content, ContractOrigin.DDL)
altered = found_names(self._ALTER_RE, ctx.content, ContractOrigin.DDL)
return dedup_emit(ctx, self.name, created, schema="create") + dedup_emit(
@@ -92,6 +93,7 @@ class LaravelQueryDialect:
name = "laravel-db"
extensions = PHP
+ requires = ("DB::",)
_TABLE_RE = re.compile(
r"\bDB::(?:connection\s*\([^)]*\)\s*->\s*)?table\s*\(\s*"
@@ -126,10 +128,8 @@ class LaravelQueryDialect:
return "update" # increment / decrement
def extract(self, ctx: ScanContext) -> list[Contract]:
- if "DB::" not in ctx.content:
- return []
found = [
(m.group("table"), self._verb(ctx.content, m.end()), line_at(ctx.content, m.start()))
- for m in self._TABLE_RE.finditer(ctx.content)
+ for m in code_matches(ctx, self._TABLE_RE)
]
return dedup_consumers(ctx, self.name, found)
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/orm_models.py b/packages/core/src/repowise/core/workspace/extractors/data/orm_models.py
index 00c507172..6dd8d85e2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/orm_models.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/orm_models.py
@@ -34,6 +34,8 @@ class SqlAlchemyDjangoDialect:
name = "python-orm"
extensions = PYTHON
+ # `__tablename__`, `db_table`, `op.create_table` and `table=True` all hold it.
+ requires = ("table",)
_TABLENAME_RE = re.compile(r"^\s*__tablename__\s*[:=]\s*.*?['\"](\S+?)['\"]", re.MULTILINE)
_DB_TABLE_RE = re.compile(r"^\s*db_table\s*=\s*['\"](\S+?)['\"]", re.MULTILINE)
@@ -42,12 +44,25 @@ class SqlAlchemyDjangoDialect:
def extract(self, ctx: ScanContext) -> list[Contract]:
# A migration's `create_table` defines the schema; the rest model it.
+ # Each pattern runs only where its literal is: the `^\s*` ones have no
+ # literal prefix for the regex engine to skip ahead by.
+ content = ctx.content
seen: set[str] = set()
- created = found_names(self._CREATE_TABLE_RE, ctx.content, ContractOrigin.DDL)
+ created = (
+ found_names(self._CREATE_TABLE_RE, content, ContractOrigin.DDL)
+ if ctx.has_any(("op.create_table",))
+ else []
+ )
out = dedup_emit(ctx, self.name, created, schema="create", seen=seen)
- found = found_names(self._TABLENAME_RE, ctx.content, ContractOrigin.EXPLICIT_TABLE)
- found += found_names(self._DB_TABLE_RE, ctx.content, ContractOrigin.EXPLICIT_TABLE)
- found += found_names(self._SQLMODEL_RE, ctx.content, ContractOrigin.NAME_CONVENTION, str.lower)
+ found: list[tuple[str, ContractOrigin, int]] = []
+ if ctx.has_any(("__tablename__",)):
+ found += found_names(self._TABLENAME_RE, content, ContractOrigin.EXPLICIT_TABLE)
+ if ctx.has_any(("db_table",)):
+ found += found_names(self._DB_TABLE_RE, content, ContractOrigin.EXPLICIT_TABLE)
+ if ctx.has_any(("True",)):
+ found += found_names(
+ self._SQLMODEL_RE, content, ContractOrigin.NAME_CONVENTION, str.lower
+ )
return out + dedup_emit(ctx, self.name, found, seen=seen)
@@ -56,6 +71,7 @@ class JpaDialect:
name = "jpa"
extensions = JAVA | KOTLIN
+ requires = ("@Table", "@Entity")
_TABLE_RE = re.compile(r"@Table\s*\(\s*name\s*=\s*\"([^\"]+)\"")
# @Entity (optionally parameterized) followed by its class declaration.
@@ -79,6 +95,7 @@ class EfCoreDialect:
name = "efcore"
extensions = CSHARP
+ requires = ("[Table(", "DbSet<")
_TABLE_ATTR_RE = re.compile(r"\[Table\(\s*\"([^\"]+)\"")
_DBSET_RE = re.compile(r"\bDbSet<\s*\w+\s*>\s+(\w+)")
@@ -94,6 +111,7 @@ class ActiveRecordDialect:
name = "activerecord"
extensions = RUBY
+ requires = ("table_name", "ApplicationRecord", "ActiveRecord::Base")
_TABLE_NAME_RE = re.compile(r"self\.table_name\s*=\s*['\"](\w+)['\"]")
_MODEL_CLASS_RE = re.compile(
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/prisma.py b/packages/core/src/repowise/core/workspace/extractors/data/prisma.py
index 4bb37a9d1..b519e36e4 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/prisma.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/prisma.py
@@ -32,6 +32,7 @@ _SCHEMA_RE = re.compile(r"""^[ \t]*@@schema\(\s*"([^"]+)"\s*\)""", re.MULTILINE)
class PrismaDialect:
name = "prisma"
extensions = frozenset({".prisma"})
+ requires = ("model",)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/sequelize.py b/packages/core/src/repowise/core/workspace/extractors/data/sequelize.py
index ea87612da..af7647ea7 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/sequelize.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/sequelize.py
@@ -19,9 +19,9 @@ from typing import TYPE_CHECKING
from repowise.core.workspace.origins import ContractOrigin
from ..base import ScanContext, line_at
-from ..calls import Call, call_sites, decorated_member, file_strings, receiver_name
+from ..calls import Call, call_sites, code_matches, decorated_member, file_strings, receiver_name
from ..langs import JS_TS
-from ..strings import Arg, call_arguments, match_paren, on_comment_line, select_argument
+from ..strings import Arg, call_arguments, match_paren, select_argument
from .dialect import dedup_emit
from .names import pluralize, snake_case
@@ -68,11 +68,10 @@ def _table(model: str, options: list[str], strings: FileStrings) -> tuple[str, f
class SequelizeDialect:
name = "sequelize"
extensions = JS_TS
+ requires = _SEQUELIZE
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if _SEQUELIZE[0] not in content:
- return []
connections = {"sequelize", *(m.group("name") for m in _BOUND_RE.finditer(content))}
found: list[tuple[str, float, int]] = []
for call, args, m, strings in call_sites(ctx, (_DEFINE, _INIT)):
@@ -106,12 +105,10 @@ class SequelizeDialect:
content = ctx.content
strings = file_strings(ctx)
out: list[tuple[str, float, int]] = []
- for m in _TABLE_DECORATOR_RE.finditer(content):
+ for m in code_matches(ctx, _TABLE_DECORATOR_RE):
member = decorated_member(content, m.start())
if member is None or member.group("cls") is None or strings is None:
continue
- if on_comment_line(content, m.start()):
- continue
options: list[str] = []
if m.group("paren"):
close = match_paren(content, m.start("paren"))
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
index 8c50c5668..7eb1842ad 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/sql_strings.py
@@ -124,6 +124,12 @@ class SqlStringsDialect:
name = "sql-strings"
extensions = PYTHON | JS_TS | JAVA | KOTLIN | GO | CSHARP | RUBY | PHP
+ # A SQL literal holds one of the verbs `_SQL_VERB_RE` reads, in some case.
+ requires = tuple(
+ spelling
+ for verb in ("select", "insert", "update", "delete", "merge")
+ for spelling in (verb.upper(), verb, verb.capitalize())
+ )
def extract(self, ctx: ScanContext) -> list[Contract]:
out: list[Contract] = []
diff --git a/packages/core/src/repowise/core/workspace/extractors/data/typeorm.py b/packages/core/src/repowise/core/workspace/extractors/data/typeorm.py
index 6ab9197c0..f7cda618e 100644
--- a/packages/core/src/repowise/core/workspace/extractors/data/typeorm.py
+++ b/packages/core/src/repowise/core/workspace/extractors/data/typeorm.py
@@ -17,7 +17,7 @@ from repowise.core.workspace.origins import ContractOrigin
from ..base import ScanContext, line_at
from ..calls import Call, call_sites, decorated_member
from ..langs import JS_TS
-from ..strings import Arg, on_comment_line
+from ..strings import Arg
from .dialect import dedup_emit
from .names import snake_case
@@ -39,15 +39,14 @@ def _default_table(cls: str) -> str:
class TypeOrmDialect:
name = "typeorm"
extensions = JS_TS
+ requires = ("@Entity",)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if "@Entity" not in content:
- return []
found: list[tuple[str, float, int]] = []
for _shape, args, m, strings in call_sites(ctx, (_ENTITY,), empty=True):
member = decorated_member(content, m.start())
- if member is None or member.group("cls") is None or on_comment_line(content, m.start()):
+ if member is None or member.group("cls") is None:
continue
# `@Entity('users', { schema })` or `@Entity({ name, schema })`.
options = args[:1] if args[:1] and args[0].startswith("{") else args[1:2]
diff --git a/packages/core/src/repowise/core/workspace/extractors/dialect.py b/packages/core/src/repowise/core/workspace/extractors/dialect.py
index 50fbc1333..07f4303c1 100644
--- a/packages/core/src/repowise/core/workspace/extractors/dialect.py
+++ b/packages/core/src/repowise/core/workspace/extractors/dialect.py
@@ -2,9 +2,16 @@
A *dialect* is one framework's, library's or file format's view of a source
file: it names the extensions it reads and turns a file into contracts. Each
-contract type is a package of dialects plus a registry tuple, and
-:class:`DialectExtractor` owns the only walk-and-dispatch loop, so adding a
-library is one module and one registry entry.
+contract type is a package of dialects plus a registry tuple, and every
+dialect runs through :func:`run_dialect`, so adding a library is one module
+and one registry entry.
+
+The engine, not the dialect, owns the two checks every dialect needs. A
+dialect declares ``requires``, the tokens one of which a file must contain
+before it is read at all; the file's :class:`..base.ScanContext` looks each
+token up once, however many dialects name it. And a contract whose line is
+nothing but a comment is dropped, so a usage example in a doc comment never
+becomes an endpoint, a queue or a table.
"""
from __future__ import annotations
@@ -22,6 +29,7 @@ if TYPE_CHECKING:
from repowise.core.workspace.contracts import Contract
from .base import SourceFile
+ from .calls import CallShape
class ContractDialect(Protocol):
@@ -29,12 +37,54 @@ class ContractDialect(Protocol):
name: str
extensions: frozenset[str]
+ #: Tokens one of which a file must contain to be read (a library's import,
+ #: a decorator's name): the cheap test that keeps a dialect off the files
+ #: it can find nothing in. Every token any match needs, so the gate never
+ #: drops a contract. :data:`EVERY_FILE` for a contract file format.
+ requires: tuple[str, ...]
def extract(self, ctx: ScanContext) -> list[Contract]:
"""Return the contracts found in *ctx* (may be empty)."""
...
+#: The gate of a dialect that reads a contract format (``.proto``, ``.sql``,
+#: ``schema.prisma``), where the extension already says the file is one.
+EVERY_FILE: tuple[str, ...] = ("",)
+
+
+def rows_gate(rows: Iterable[CallShape]) -> tuple[str, ...]:
+ """A table dialect's gate: every token its rows require, or ``()`` if a row requires none.
+
+ An ungated row leaves the whole table ungated, which the registry test refuses.
+ """
+ tokens: dict[str, None] = {}
+ for row in rows:
+ if not row.requires:
+ return ()
+ tokens.update(dict.fromkeys(row.requires))
+ return tuple(tokens)
+
+
+def runs_on(dialect: ContractDialect, ctx: ScanContext) -> bool:
+ """Whether *dialect* reads *ctx*'s file: its extension, then its gate."""
+ return ctx.suffix in dialect.extensions and ctx.has_any(dialect.requires)
+
+
+def without_comments(contracts: list[Contract], ctx: ScanContext) -> list[Contract]:
+ """*contracts* less those whose line in *ctx*'s file is only a comment."""
+ if not contracts:
+ return contracts
+ return [c for c in contracts if not ctx.comment_only_line(c.line)]
+
+
+def run_dialect(dialect: ContractDialect, ctx: ScanContext) -> list[Contract]:
+ """The contracts *dialect* reads in *ctx*, gated and with commented-out matches dropped."""
+ if not runs_on(dialect, ctx):
+ return []
+ return without_comments(dialect.extract(ctx), ctx)
+
+
class RepoPass(Protocol):
"""One extraction run's state for a dialect whose contracts span files.
@@ -137,33 +187,41 @@ class DialectExtractor:
passes: dict[int, RepoPass] = {
id(d): d.repo_pass() for d in self.dialects if hasattr(d, "repo_pass")
}
+ passed: dict[str, ScanContext] = {} # files a repo pass read, for its comment check
for rel_path, suffix, content in select_files(
repo_path, self.source_extensions(), exclude, files
):
ctx = ScanContext(repo_alias, rel_path, suffix, content)
found: list[Contract] = []
for dialect in self.dialects:
- if suffix not in dialect.extensions:
- continue
repo_pass = passes.get(id(dialect))
- if repo_pass is not None:
+ if repo_pass is None:
+ found.extend(run_dialect(dialect, ctx))
+ elif runs_on(dialect, ctx):
repo_pass.scan(ctx)
- else:
- found.extend(dialect.extract(ctx))
+ passed[rel_path] = ctx
contracts.extend(found if self.identity is None else dedupe(found, self.identity))
for repo_pass in passes.values():
- found = repo_pass.finish()
+ found = [
+ c for c in repo_pass.finish()
+ if c.file_path not in passed or not passed[c.file_path].comment_only_line(c.line)
+ ]
# Every identity names the file, so a whole-repo dedupe is per file.
contracts.extend(found if self.identity is None else dedupe(found, self.identity))
return contracts
__all__ = [
+ "EVERY_FILE",
"ContractDialect",
"DialectExtractor",
"RepoPass",
"build_contract",
"dedupe",
"file_identity",
+ "rows_gate",
+ "run_dialect",
+ "runs_on",
"union_extensions",
+ "without_comments",
]
diff --git a/packages/core/src/repowise/core/workspace/extractors/grpc/languages.py b/packages/core/src/repowise/core/workspace/extractors/grpc/languages.py
index b66d205fe..483649578 100644
--- a/packages/core/src/repowise/core/workspace/extractors/grpc/languages.py
+++ b/packages/core/src/repowise/core/workspace/extractors/grpc/languages.py
@@ -19,6 +19,7 @@ from typing import TYPE_CHECKING
from repowise.core.workspace.origins import ContractOrigin
from ..base import line_at
+from ..calls import code_matches
from ..dialect import build_contract
from ..langs import CSHARP, GO, JAVA, JS_TS, PYTHON
@@ -50,6 +51,9 @@ class _Language:
name: str
extensions: frozenset[str]
rules: tuple[_Rule, ...]
+ #: Tokens one of which a file must contain before any rule runs: every
+ #: rule's pattern needs one, so the gate drops no match.
+ requires: tuple[str, ...] = ()
#: A marker the file must carry before any consumer rule runs.
consumer_gate: re.Pattern[str] | None = None
#: Stub-name prefixes that are test doubles rather than clients.
@@ -68,6 +72,7 @@ LANGUAGES: tuple[_Language, ...] = (
_Rule("consumer", re.compile(r"\.New(\w+)Client\s*\("),
"go:New{service}Client", "go_client", _CLIENT_CALL),
),
+ requires=("Register", "Client"),
),
_Language(
name="java",
@@ -80,6 +85,7 @@ LANGUAGES: tuple[_Language, ...] = (
_Rule("consumer", re.compile(r"(\w+)Grpc\.new(?:Blocking|Future)?Stub\s*\("),
"java:{service}Grpc.newStub", "java_stub", _CLIENT_CALL),
),
+ requires=("Grpc.",),
),
_Language(
name="python",
@@ -92,6 +98,7 @@ LANGUAGES: tuple[_Language, ...] = (
_Rule("consumer", re.compile(r"(\w+)Stub\s*\("),
"py:{service}Stub", "py_stub", _CLIENT_CALL),
),
+ requires=("_to_server", "Stub"),
false_prefixes=("mock", "test", "fake"),
),
_Language(
@@ -103,6 +110,7 @@ LANGUAGES: tuple[_Language, ...] = (
"ts:@GrpcMethod('{service}', '{method}')", "ts_decorator", _ROUTE_REGISTRATION,
method_group=2),
),
+ requires=("@GrpcMethod",),
),
_Language(
name="csharp",
@@ -119,6 +127,8 @@ LANGUAGES: tuple[_Language, ...] = (
_Rule("consumer", re.compile(r"\bnew\s+(\w+)Client\s*\("),
"cs:new {service}Client", "csharp_client", _NAME_HEURISTIC),
),
+ # The consumer gate's markers all hold `Grpc`, `CallInvoker` or `Base`.
+ requires=("MapGrpcService", "Base", "Grpc", "CallInvoker"),
consumer_gate=re.compile(
r"\bGrpcChannel\b|\bGrpc\.(?:Net|Core)\b|\bCallInvoker\b|\bChannelBase\b"
r"|\bAddGrpcClient\b|using\s+Grpc\b"
@@ -135,6 +145,7 @@ class LanguageGrpcDialect:
self._lang = language
self.name = language.name
self.extensions = language.extensions
+ self.requires = language.requires
def extract(self, ctx: ScanContext) -> list[Contract]:
lang = self._lang
@@ -143,7 +154,7 @@ class LanguageGrpcDialect:
for rule in lang.rules:
if rule.role == "consumer" and gated:
continue
- for m in rule.pattern.finditer(ctx.content):
+ for m in code_matches(ctx, rule.pattern):
service = m.group(1)
if rule.role == "consumer" and service.lower().startswith(lang.false_prefixes):
continue
diff --git a/packages/core/src/repowise/core/workspace/extractors/grpc/proto.py b/packages/core/src/repowise/core/workspace/extractors/grpc/proto.py
index 7bfd0d3d3..532de1ddf 100644
--- a/packages/core/src/repowise/core/workspace/extractors/grpc/proto.py
+++ b/packages/core/src/repowise/core/workspace/extractors/grpc/proto.py
@@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, NamedTuple
from repowise.core.workspace.contract_schema import ContractSchema, SchemaField
from repowise.core.workspace.origins import ContractOrigin
-from ..dialect import build_contract
+from ..dialect import EVERY_FILE, build_contract
from ..langs import PROTO
if TYPE_CHECKING:
@@ -214,6 +214,7 @@ def _parse_proto_file(
class ProtoDialect:
name = "proto"
extensions = PROTO
+ requires = EVERY_FILE
def extract(self, ctx: ScanContext) -> list[Contract]:
package, services, messages = _parse_proto_file(ctx.content)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
index 6680f43e4..d89e55884 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/__init__.py
@@ -8,7 +8,8 @@ a framework means dropping one dialect module and appending it to a registry —
no orchestrator edits. HTTP keeps its own orchestrator rather than the shared
:class:`..dialect.DialectExtractor` because it runs two passes the others do
not: router mounts collected across files, and the index-backed routes and
-client calls.
+client calls. Each dialect still runs through :func:`..dialect.run_dialect`,
+the one gate and comment check every contract type shares.
"""
from __future__ import annotations
@@ -16,7 +17,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from ..base import ScanContext, select_files
-from ..dialect import ContractDialect, union_extensions
+from ..dialect import ContractDialect, run_dialect, runs_on, union_extensions
from ..langs import PYTHON
from .apex_callouts import ApexCalloutsDialect
from .apex_rest import ApexRestDialect
@@ -189,16 +190,12 @@ class HttpExtractor:
)
for dialect in self.provider_dialects:
- if suffix not in dialect.extensions:
- continue
if from_parse and dialect.name in _INDEX_BACKED_DIALECTS:
continue # superseded by the index pass above
- contracts.extend(dialect.extract(ctx))
+ contracts.extend(run_dialect(dialect, ctx))
index_ids = {c.contract_id for c in consumers_from_parse}
for dialect in self.consumer_dialects:
- if suffix not in dialect.extensions:
- continue
- found = dialect.extract(ctx)
+ found = run_dialect(dialect, ctx)
if index_ids and dialect.name in _INDEX_BACKED_CONSUMER_DIALECTS:
# Supersede per *contract*, not per file. Dropping the whole
# dialect for a file the index read would delete anything
@@ -225,10 +222,9 @@ class HttpExtractor:
]
per_file: list[dict[str, str]] = []
for rel_path, suffix, content in files:
- ctx: ScanContext | None = None
+ ctx = ScanContext(repo_alias, rel_path, suffix, content)
for dialect in collectors:
- if suffix in dialect.extensions:
- ctx = ctx or ScanContext(repo_alias, rel_path, suffix, content)
+ if runs_on(dialect, ctx):
found = dialect.collect_mounts(ctx)
if found:
per_file.append(found)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/apex_callouts.py b/packages/core/src/repowise/core/workspace/extractors/http/apex_callouts.py
index bd9c271eb..0fbd878db 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/apex_callouts.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/apex_callouts.py
@@ -185,6 +185,8 @@ def _url(
class ApexCalloutsDialect:
name = "apex-callouts"
extensions = APEX
+ # Apex is case-insensitive; `extract` reads the exact spelling.
+ requires = ("setEndpoint", "setendpoint", "SetEndpoint", "SETENDPOINT")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/apex_rest.py b/packages/core/src/repowise/core/workspace/extractors/http/apex_rest.py
index bbefbca09..7393661d7 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/apex_rest.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/apex_rest.py
@@ -33,6 +33,8 @@ def apex_rest_path(url_mapping: str) -> str:
class ApexRestDialect:
name = "apex-rest"
extensions = APEX
+ # Apex is case-insensitive; `apex_rest_routes` reads the exact spelling.
+ requires = ("RestResource", "restresource", "Restresource", "RESTRESOURCE", "restResource")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/aspnet.py b/packages/core/src/repowise/core/workspace/extractors/http/aspnet.py
index 22b734cc1..34f8cf2df 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/aspnet.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/aspnet.py
@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.framework_routes import aspnet_groups, aspnet_routes
from ..base import line_at
+from ..calls import code_matches, outside_comments
from ..langs import CSHARP
from .dialect import build_provider_contract, nearest_prefix
from .mounts import compose_prefix, group_prefixes
@@ -52,6 +53,8 @@ _ASPNET_CLASS_ROUTE_RE = re.compile(
class AspNetDialect:
name = "aspnet"
extensions = CSHARP
+ # `[HttpGet(...)]` and `.MapGet(...)`, read without regard to case.
+ requires = ("Http", "http", "HTTP", "MapGet", "MapPost", "MapPut", "MapDelete", "MapPatch")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
@@ -62,7 +65,7 @@ class AspNetDialect:
out: list[Contract] = []
# Attribute routing — stitched onto the nearest class [Route(...)].
- for m in _ASPNET_METHOD_RE.finditer(content):
+ for m in code_matches(ctx, _ASPNET_METHOD_RE):
method = m.group(1).upper()
path_raw = m.group(2)
prefix = nearest_prefix(class_mappings, m.start())
@@ -81,7 +84,7 @@ class AspNetDialect:
# Minimal API — the prefix comes from the MapGroup chain its receiver
# is bound to, not from a class [Route(...)].
prefixes = group_prefixes(aspnet_groups(content))
- for route in aspnet_routes(content):
+ for route in outside_comments(ctx, aspnet_routes(content)):
c = build_provider_contract(
ctx,
method=route.verb,
@@ -96,7 +99,7 @@ class AspNetDialect:
# Parameterless [HttpPost] — route is whichever class [Route("...")]
# precedes it. Without a class route there is no useful path to record.
if class_mappings:
- for m in _ASPNET_BARE_METHOD_RE.finditer(content):
+ for m in code_matches(ctx, _ASPNET_BARE_METHOD_RE):
method = m.group(1).upper()
prefix = nearest_prefix(class_mappings, m.start())
if not prefix:
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/client_calls.py b/packages/core/src/repowise/core/workspace/extractors/http/client_calls.py
index 77116fb14..85fff052a 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/client_calls.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/client_calls.py
@@ -115,9 +115,14 @@ def consumer_contracts(
*rooted_only* drops a URL that is neither absolute nor rooted. A client
with a configured base composes ``"some/path"`` onto it, so the path the
request reaches has a prefix this layer cannot see.
+
+ A call in a comment (a usage example in a doc comment) is skipped here,
+ once for every recogniser.
"""
out: list[Contract] = []
for m in matches:
+ if ctx.in_comment(m.offset):
+ continue
url = resolve_string(m.url, syntax, constants)
if url is None or (path_only and "/" not in url):
continue
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py b/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
index 240156777..961a5ea40 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/csharp_http.py
@@ -93,6 +93,8 @@ def besthttp_calls(content: str) -> Iterator[ClientCallMatch]:
class CSharpHttpDialect:
name = "csharp-http"
extensions = CSHARP
+ # `GetAsync`, `GetRequest`, `UnityWebRequest`, `HTTPRequest` in any case.
+ requires = ("Async", "Request", "request", "REQUEST")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/django.py b/packages/core/src/repowise/core/workspace/extractors/http/django.py
index 7a4a65c95..22ba71eb9 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/django.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/django.py
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.framework_routes import django_includes, django_routes
from ..base import line_at
+from ..calls import outside_comments
from ..langs import PYTHON
from .dialect import build_provider_contract
from .mounts import compose_prefix
@@ -48,16 +49,13 @@ def _module_suffixes(rel_path: str) -> list[str]:
class DjangoDialect:
name = "django"
extensions = PYTHON
+ requires = _DJANGO_URLS_IMPORT
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
"""``include("api.urls")`` mounts declared in the file, keyed by module."""
- if not any(mod in ctx.content for mod in _DJANGO_URLS_IMPORT):
- return {}
return {_MOUNT_PREFIX + module: prefix for prefix, module in django_includes(ctx.content)}
def extract(self, ctx: ScanContext) -> list[Contract]:
- if not any(mod in ctx.content for mod in _DJANGO_URLS_IMPORT):
- return []
mount = next(
(
p
@@ -67,7 +65,7 @@ class DjangoDialect:
"",
)
out: list[Contract] = []
- for route in django_routes(ctx.content):
+ for route in outside_comments(ctx, django_routes(ctx.content)):
c = build_provider_contract(
ctx,
method="*",
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/express.py b/packages/core/src/repowise/core/workspace/extractors/http/express.py
index 33d7fe757..8e43da0f0 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/express.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/express.py
@@ -25,6 +25,7 @@ from repowise.core.ingestion.framework_routes import (
)
from ..base import line_at
+from ..calls import code_matches, outside_comments
from ..langs import JS_TS
from .dialect import build_provider_contract
from .mounts import compose_prefix
@@ -41,11 +42,18 @@ _APP_USE_RE = re.compile(r"""\.use\s*\(\s*['"]([^'"]+)['"]\s*,\s*([\w.]+)""")
class ExpressDialect:
name = "express"
extensions = JS_TS
+ # A route's receiver is a default name or a router a constructor binds,
+ # and a mount is a `.use(`; `app .get(` and `.use (` are allowed.
+ requires = (
+ "app.", "app ", "app\n", "app\t", "router.", "router ", "router\n", "router\t",
+ "Router", "express", "Hono", "astify", "Koa", "koa", "Elysia",
+ ".use(", ".use ", ".use\n", ".use\t",
+ ) # fmt: skip
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
"""Find ``app.use('/prefix', router)`` mounts declared in the file."""
out: dict[str, str] = {}
- for m in _APP_USE_RE.finditer(ctx.content):
+ for m in code_matches(ctx, _APP_USE_RE):
out[m.group(2).split(".")[-1]] = m.group(1)
return out
@@ -58,7 +66,7 @@ class ExpressDialect:
default = served.pop() if len(served) == 1 else "express"
out: list[Contract] = []
- for route in express_routes(ctx.content):
+ for route in outside_comments(ctx, express_routes(ctx.content)):
# The registry also yields `use`/`all`, which the graph consumer
# scans for handlers but which name no HTTP method here.
if route.verb not in HTTP_METHODS or not route.path:
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
index 6be28e8c8..3898ed5fa 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/fastapi.py
@@ -11,6 +11,7 @@ import re
from typing import TYPE_CHECKING
from ..base import line_at
+from ..calls import code_matches
from ..langs import PYTHON
from .dialect import METHODS, build_provider_contract
from .flask import flask_file
@@ -45,6 +46,9 @@ _DEFAULT_ROUTER_NAMES = frozenset({"app", "router"})
class FastApiDialect:
name = "fastapi"
extensions = PYTHON
+ # A route's receiver is `app`, `router` or a name bound to one of the two
+ # classes; a mount is an `include_router`.
+ requires = ("@app.", "@router.", "APIRouter", "FastAPI", "include_router")
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
"""Find ``include_router(var, prefix=...)`` mounts declared in the file.
@@ -55,7 +59,7 @@ class FastApiDialect:
if flask_file(ctx.content):
return {}
out: dict[str, str] = {}
- for m in _INCLUDE_ROUTER_RE.finditer(ctx.content):
+ for m in code_matches(ctx, _INCLUDE_ROUTER_RE):
args = balanced_args(ctx.content, m.end() - 1) # m.end()-1 is the '('
var_m = _FIRST_ARG_RE.match(args)
pm = _PREFIX_KW_RE.search(args)
@@ -73,7 +77,7 @@ class FastApiDialect:
known = set(prefixes) | _DEFAULT_ROUTER_NAMES
out: list[Contract] = []
- for m in _FASTAPI_RE.finditer(ctx.content):
+ for m in code_matches(ctx, _FASTAPI_RE):
var, method, path_raw = m.group(1), m.group(2), m.group(3)
if var not in known:
continue
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/flask.py b/packages/core/src/repowise/core/workspace/extractors/http/flask.py
index 205998d06..27837e7c5 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/flask.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/flask.py
@@ -79,6 +79,7 @@ def flask_file(content: str) -> bool:
class FlaskDialect:
name = "flask"
extensions = PYTHON
+ requires = ("flask",)
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
"""``register_blueprint(bp, url_prefix=...)`` mounts declared in the file.
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/go.py b/packages/core/src/repowise/core/workspace/extractors/http/go.py
index ea1f4b49d..366c8c58e 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/go.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/go.py
@@ -16,6 +16,7 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.framework_routes import HTTP_METHODS, go_groups, go_routes
from ..base import line_at
+from ..calls import outside_comments
from ..langs import GO
from .dialect import build_provider_contract
from .mounts import compose_prefix, group_prefixes
@@ -32,11 +33,17 @@ _VERBLESS = ("HANDLE", "HANDLEFUNC")
class GoDialect:
name = "go"
extensions = GO
+ # The verb opens its call directly, as gofmt writes it; a chain may break
+ # the line after the receiver's `.`.
+ requires = (
+ "GET(", "POST(", "PUT(", "DELETE(", "PATCH(", "OPTIONS(", "HEAD(", "Any(", "Handle(",
+ "HandleFunc(",
+ ) # fmt: skip
def extract(self, ctx: ScanContext) -> list[Contract]:
prefixes = group_prefixes(go_groups(ctx.content))
out: list[Contract] = []
- for route in go_routes(ctx.content):
+ for route in outside_comments(ctx, go_routes(ctx.content)):
if route.verb in _VERBLESS:
method = "*"
elif route.verb in HTTP_METHODS:
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/go_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/go_clients.py
index 13c794ce4..2faa60be4 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/go_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/go_clients.py
@@ -47,28 +47,9 @@ _NEW_REQUEST_RE = re.compile(r"\bhttp\.NewRequest(WithContext)?\s*\(")
_NEW_REQUEST_ARGS = {False: (0, 1), True: (1, 2)}
-def _commented_out(content: str, offset: int) -> bool:
- """True when a ``//`` comment opens on *offset*'s line before it.
-
- Go doc comments show usage as indented example code, so without this a
- ``// req, _ := http.NewRequest("GET", "https://host/x", nil)`` in a
- docstring becomes a contract. A ``//`` that follows a colon is a URL's
- scheme separator, not a comment.
- """
- prefix = content[content.rfind("\n", 0, offset) + 1 : offset]
- idx = prefix.find("//")
- while idx != -1:
- if idx == 0 or prefix[idx - 1] != ":":
- return True
- idx = prefix.find("//", idx + 2)
- return False
-
-
def net_http_calls(content: str) -> Iterator[ClientCallMatch]:
"""Every ``net/http`` call site whose verb and URL argument are readable."""
for m in _PACKAGE_VERB_RE.finditer(content):
- if _commented_out(content, m.start()):
- continue
paren = m.end() - 1
args = call_arguments(content, paren)
if not args:
@@ -83,8 +64,6 @@ def net_http_calls(content: str) -> Iterator[ClientCallMatch]:
)
for m in _NEW_REQUEST_RE.finditer(content):
- if _commented_out(content, m.start()):
- continue
paren = m.end() - 1
args = call_arguments(content, paren)
method_at, url_at = _NEW_REQUEST_ARGS[m.group(1) is not None]
@@ -104,6 +83,7 @@ def net_http_calls(content: str) -> Iterator[ClientCallMatch]:
class GoClientsDialect:
name = "go-clients"
extensions = GO
+ requires = ("http.",)
def extract(self, ctx: ScanContext) -> list[Contract]:
rows = list(net_http_calls(ctx.content))
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/java_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/java_clients.py
index d80a7a1a8..775776f33 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/java_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/java_clients.py
@@ -163,6 +163,7 @@ def resttemplate_calls(content: str) -> Iterator[ClientCallMatch]:
class JavaClientsDialect:
name = "java-clients"
extensions = JAVA
+ requires = ("@RequestLine", "HttpRequest", "RestTemplate")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/jaxrs.py b/packages/core/src/repowise/core/workspace/extractors/http/jaxrs.py
index 090cb7407..1a349c6f2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/jaxrs.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/jaxrs.py
@@ -19,6 +19,7 @@ from repowise.core.ingestion.framework_routes import (
)
from ..base import line_at
+from ..calls import outside_comments
from ..langs import JAVA, KOTLIN
from .dialect import build_provider_contract, nearest_prefix
from .mounts import compose_prefix
@@ -32,14 +33,13 @@ if TYPE_CHECKING:
class JaxRsDialect:
name = "jaxrs"
extensions = JAVA | KOTLIN
+ requires = ("@Path",)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if "@Path" not in content:
- return []
class_paths = jaxrs_class_paths(content)
out: list[Contract] = []
- for route in jaxrs_routes(content):
+ for route in outside_comments(ctx, jaxrs_routes(content)):
if route.verb not in HTTP_METHODS:
continue
path = compose_prefix(nearest_prefix(class_paths, route.offset), route.path or "")
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
index 20f446e1d..0b531404d 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/js_clients.py
@@ -38,10 +38,12 @@ _FETCH_METHOD_RE = re.compile(
)
# Wrapper call: IDENT("<url>" | `<url>`, ...) where the URL literal is concrete,
-# i.e. starts with `/`, a `${...}` base placeholder, or a scheme.
-_WRAPPER_CALL_RE = re.compile(
- r"""\b(\w+)\s*\(\s*['"`]((?:/|\$\{|https?:)[^'"`]*)['"`]""",
-)
+# i.e. starts with `/`, a `${...}` base placeholder, or a scheme. The scan is
+# led by the `(` and reads the identifier back from it: led by the identifier,
+# the regex was tried at every word of every file.
+_WRAPPER_ARG_RE = re.compile(r"""\(\s*['"`]((?:/|\$\{|https?:)[^'"`]*)['"`]""")
+_CALLEE_BEFORE_RE = re.compile(r"\b(\w+)\s*$")
+_CALLEE_WINDOW = 200
# The callee names that read as an HTTP wrapper rather than navigation/util.
_HTTP_NAME_RE = re.compile(r"(?i)fetch|request|http|api|ajax|rest|rpc")
@@ -89,11 +91,14 @@ def wrapper_calls(content: str, clients: frozenset[str]) -> Iterator[ClientCallM
*clients* are the client names the file calls, read with their base elsewhere.
"""
- for m in _WRAPPER_CALL_RE.finditer(content):
- callee = m.group(1)
- if callee in _WRAPPER_SKIP or _callee(content, m.start(1), m.end(1)) in clients:
+ for m in _WRAPPER_ARG_RE.finditer(content):
+ head = _CALLEE_BEFORE_RE.search(content, max(0, m.start() - _CALLEE_WINDOW), m.start())
+ if head is None:
+ continue
+ callee = head.group(1)
+ if callee in _WRAPPER_SKIP or _callee(content, head.start(1), head.end(1)) in clients:
continue
- if clients and _receiver(content, m.start(1)) in clients:
+ if clients and _receiver(content, head.start(1)) in clients:
continue # `http.post(url, { method })`, a client's own verb
nl = content.find("\n", m.end())
window = content[m.end() :] if nl == -1 else content[m.end() : nl]
@@ -105,8 +110,8 @@ def wrapper_calls(content: str, clients: frozenset[str]) -> Iterator[ClientCallM
# is, and `apiPost`/`apiDelete` carry it.
yield ClientCallMatch(
client="wrapper",
- url=literal_span(content, m, 2),
- offset=m.start(),
+ url=literal_span(content, m, 1),
+ offset=head.start(),
callee=callee,
method=method_opt.group(1).upper() if method_opt else None,
origin=ContractOrigin.NAME_HEURISTIC,
@@ -116,6 +121,11 @@ def wrapper_calls(content: str, clients: frozenset[str]) -> Iterator[ClientCallM
class JsClientsDialect:
name = "js-clients"
extensions = JS_TS
+ # Next to no JS file lacks one of these: the URL literal a fetch or a
+ # wrapper reads, the import a client library or an imported instance needs,
+ # or the export an environment module is. Recorded as the dialect's
+ # ceiling: this gate is the extension, near enough.
+ requires = ("fetch", "'/", '"/', "`/", "${", "http", "axios", "import", "require", "export")
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
return client_mounts(ctx) | environment_mounts(ctx)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/kotlin_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/kotlin_clients.py
index cfab3d1c2..bbf290aed 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/kotlin_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/kotlin_clients.py
@@ -208,11 +208,10 @@ def ktor_calls(content: str) -> Iterator[ClientCallMatch]:
class KotlinClientsDialect:
name = "kotlin-clients"
extensions = KOTLIN
+ requires = (_KTOR_PACKAGE,)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if _KTOR_PACKAGE not in content:
- return []
rows = list(ktor_calls(content))
if not rows:
return []
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/laravel.py b/packages/core/src/repowise/core/workspace/extractors/http/laravel.py
index fa0748671..a35958629 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/laravel.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/laravel.py
@@ -19,6 +19,7 @@ from repowise.core.ingestion.framework_routes import (
)
from ..base import line_at
+from ..calls import outside_comments
from ..langs import PHP
from .dialect import build_provider_contract
from .mounts import declare_mount, declared_mount
@@ -34,6 +35,7 @@ _MOUNT_KEY = "laravel-route-file:"
class LaravelDialect:
name = "laravel"
extensions = PHP
+ requires = ("Route::", "withRouting")
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
out: dict[str, str] = {}
@@ -51,7 +53,7 @@ class LaravelDialect:
if prefix is None:
return [] # served under a prefix this cannot read
out: list[Contract] = []
- for route in laravel_routes(ctx.content, prefix):
+ for route in outside_comments(ctx, laravel_routes(ctx.content, prefix)):
# `any` names no single verb, and a computed path no one route.
if route.verb not in HTTP_METHODS or not route.path:
continue
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/micronaut.py b/packages/core/src/repowise/core/workspace/extractors/http/micronaut.py
index ac9164b2f..5047483fa 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/micronaut.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/micronaut.py
@@ -24,6 +24,7 @@ from repowise.core.ingestion.framework_routes import (
)
from ..base import line_at
+from ..calls import outside_comments
from ..langs import JAVA, KOTLIN
from .dialect import build_provider_contract
from .mounts import compose_prefix
@@ -62,17 +63,16 @@ def _nearest_controller(
class MicronautDialect:
name = "micronaut"
extensions = JAVA | KOTLIN
+ requires = (_MICRONAUT_ANNOTATIONS,)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if _MICRONAUT_ANNOTATIONS not in content:
- return []
# One scan of the file's annotations, read by all three recognisers.
annotations = micronaut_annotations(content)
class_paths = micronaut_class_paths(content, annotations)
clients = micronaut_client_types(content, annotations)
out: list[Contract] = []
- for route in micronaut_routes(content, annotations):
+ for route in outside_comments(ctx, micronaut_routes(content, annotations)):
# HEAD and OPTIONS are Micronaut verbs the contract layer does not
# record; a path it could not read is refused rather than served at
# the class prefix.
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/nestjs.py b/packages/core/src/repowise/core/workspace/extractors/http/nestjs.py
index 54134b4e6..1dcb5aa75 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/nestjs.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/nestjs.py
@@ -28,7 +28,7 @@ from typing import TYPE_CHECKING
from ..base import line_at
from ..calls import Call, call_sites, decorated_member, file_strings
from ..langs import JS_TS
-from ..strings import Arg, on_comment_line, select_argument
+from ..strings import Arg, select_argument
from .dialect import build_provider_contract
from .mounts import compose_prefix, declare_mount, declared_mount, declared_names
from .paths import normalize_http_path
@@ -221,6 +221,7 @@ def _route_versions(
class NestDialect:
name = "nestjs"
extensions = JS_TS
+ requires = ("@Controller", _FACTORY, *_GLOBAL_PREFIX.requires, *_VERSIONING.requires)
def collect_mounts(self, ctx: ScanContext) -> dict[str, str]:
out: dict[str, str] = {}
@@ -250,7 +251,7 @@ class NestDialect:
decorated: dict[int, list[str] | None] = {} # member offset -> its `@Version`
for shape, args, m, _ in call_sites(ctx, (_CONTROLLER, _ROUTE, _VERSION), strings, empty=True):
member = decorated_member(content, m.start())
- if member is None or on_comment_line(content, m.start()):
+ if member is None:
continue
if shape is _ROUTE:
routes.append(
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/next_app.py b/packages/core/src/repowise/core/workspace/extractors/http/next_app.py
index 9f3c2cabe..ea4d4d39f 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/next_app.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/next_app.py
@@ -30,6 +30,7 @@ if TYPE_CHECKING:
class NextAppDialect:
name = "next-app"
extensions = JS_TS
+ requires = ("export",)
def extract(self, ctx: ScanContext) -> list[Contract]:
path = next_route_path(ctx.rel_path)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/php_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/php_clients.py
index 6103d1a04..9d69db7ab 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/php_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/php_clients.py
@@ -129,6 +129,7 @@ def laravel_http_calls(content: str) -> Iterator[ClientCallMatch]:
class PhpClientsDialect:
name = "php-clients"
extensions = PHP
+ requires = ("Client", "Http::")
def extract(self, ctx: ScanContext) -> list[Contract]:
request_rows = list(guzzle_request_calls(ctx.content))
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
index e8f76ab85..df3c64aaa 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/python_clients.py
@@ -30,6 +30,7 @@ def requests_calls(content: str) -> Iterator[ClientCallMatch]:
class PythonClientsDialect:
name = "python-clients"
extensions = PYTHON
+ requires = ("requests", "httpx", "Requests", "Httpx", "REQUESTS", "HTTPX")
def extract(self, ctx: ScanContext) -> list[Contract]:
return consumer_contracts(ctx, requests_calls(ctx.content), PYTHON_SYNTAX)
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/remix.py b/packages/core/src/repowise/core/workspace/extractors/http/remix.py
index 0d1b49f9a..eea3906b7 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/remix.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/remix.py
@@ -45,6 +45,7 @@ _HANDLER_FOR_VERB = {"GET": "loader", "*": "action"}
class RemixDialect:
name = "remix"
extensions = JS_TS
+ requires = ("@remix-run/", "react-router")
def extract(self, ctx: ScanContext) -> list[Contract]:
# The path gate is the convention test: `remix_route_path` returns None
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/ruby_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/ruby_clients.py
index 3cb27f55a..c9df01912 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/ruby_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/ruby_clients.py
@@ -142,6 +142,7 @@ def faraday_connection_calls(content: str) -> Iterator[ClientCallMatch]:
class RubyClientsDialect:
name = "ruby-clients"
extensions = RUBY
+ requires = ("HTTParty", "RestClient", "Faraday", "Net::HTTP")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/rust_axum.py b/packages/core/src/repowise/core/workspace/extractors/http/rust_axum.py
index 030fdec1d..1b2f17141 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/rust_axum.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/rust_axum.py
@@ -22,6 +22,7 @@ from typing import TYPE_CHECKING
from repowise.core.ingestion.framework_routes import axum_routes
from ..base import line_at
+from ..calls import code_matches, outside_comments
from ..langs import RUST
from .dialect import build_provider_contract
@@ -40,6 +41,7 @@ _RUST_ATTR_ROUTE_RE = re.compile(
class RustAxumDialect:
name = "rust-axum"
extensions = RUST
+ requires = ("route", "#[")
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
@@ -47,7 +49,7 @@ class RustAxumDialect:
# One call site yields a match per verb, so collapse repeats of a verb.
seen: set[tuple[int, str]] = set()
- for route in axum_routes(content):
+ for route in outside_comments(ctx, axum_routes(content)):
# `on(MethodFilter::GET, h)` names no literal verb.
if route.verb == "ON" or (route.offset, route.verb) in seen:
continue
@@ -64,7 +66,7 @@ class RustAxumDialect:
out.append(c)
# Actix-web / Rocket attribute macros.
- for m in _RUST_ATTR_ROUTE_RE.finditer(content):
+ for m in code_matches(ctx, _RUST_ATTR_ROUTE_RE):
c = build_provider_contract(
ctx,
method=m.group(1).upper(),
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py b/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
index 62e4d0230..bfcdb6d53 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/rust_clients.py
@@ -63,6 +63,7 @@ def reqwest_calls(content: str) -> Iterator[ClientCallMatch]:
class RustClientsDialect:
name = "rust-clients"
extensions = RUST
+ requires = (".get", ".post", ".put", ".delete", ".patch", ".head", "reqwest")
def extract(self, ctx: ScanContext) -> list[Contract]:
# `.get("key")` on a map has no slash; the receiver is anyone's.
diff --git a/packages/core/src/repowise/core/workspace/extractors/http/spring.py b/packages/core/src/repowise/core/workspace/extractors/http/spring.py
index 88e1b8216..32f13cad2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/http/spring.py
+++ b/packages/core/src/repowise/core/workspace/extractors/http/spring.py
@@ -19,6 +19,7 @@ from repowise.core.ingestion.framework_routes import (
)
from ..base import line_at
+from ..calls import outside_comments
from ..langs import JAVA, KOTLIN
from .dialect import build_provider_contract
from .mounts import compose_prefix
@@ -32,15 +33,14 @@ if TYPE_CHECKING:
class SpringDialect:
name = "spring"
extensions = JAVA | KOTLIN
+ requires = ("Mapping",)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if "Mapping" not in content:
- return []
annotations = micronaut_annotations(content)
class_paths = spring_class_paths(content, annotations)
out: list[Contract] = []
- for route in spring_routes(content, annotations):
+ for route in outside_comments(ctx, spring_routes(content, annotations)):
# An unrestricted `@RequestMapping` serves every verb, which the
# contract layer has no method for; an unreadable path is refused.
if route.verb not in HTTP_METHODS or route.path is None:
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket/broadcasting.py b/packages/core/src/repowise/core/workspace/extractors/socket/broadcasting.py
index c126ddca3..58667993e 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket/broadcasting.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket/broadcasting.py
@@ -28,6 +28,7 @@ from repowise.core.ingestion.languages.php_same_namespace import (
)
from ..calls import call_chain, call_sites, file_strings
+from ..dialect import rows_gate
from ..langs import JS_TS, PHP
from ..strings import Arg, call_arguments, match_paren, unescape_backslashes
from .dialect import SocketCall, channel_scope, event_contracts
@@ -153,6 +154,7 @@ class LaravelBroadcastDialect:
name = "laravel-broadcast"
extensions = PHP
+ requires = ("broadcastOn", *_PUSHER_TRIGGER.requires, *_ANONYMOUS.requires)
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
@@ -202,6 +204,7 @@ class EchoDialect:
name = "echo"
extensions = JS_TS
+ requires = rows_gate((_ECHO_LISTEN, _USE_ECHO, _PUSHER_SUBSCRIBE))
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket/dialect.py b/packages/core/src/repowise/core/workspace/extractors/socket/dialect.py
index 0f17da410..9a641944e 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket/dialect.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket/dialect.py
@@ -25,7 +25,7 @@ from repowise.core.workspace.origins import ContractOrigin
from ..base import line_at
from ..calls import call_sites
-from ..dialect import build_contract
+from ..dialect import build_contract, rows_gate
from ..http.paths import (
extract_path_from_url,
is_unusable_consumer_path,
@@ -159,6 +159,7 @@ class SocketDialect:
self.name = name
self.calls = calls
self.extensions = frozenset().union(*(c.extensions for c in calls))
+ self.requires = rows_gate(calls)
def extract(self, ctx: ScanContext) -> list[Contract]:
out: list[Contract] = []
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket/dotnet.py b/packages/core/src/repowise/core/workspace/extractors/socket/dotnet.py
index 8c9ec825b..1fc0d610d 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket/dotnet.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket/dotnet.py
@@ -26,6 +26,7 @@ DOTNET = SocketDialect(
label="ClientWebSocket.ConnectAsync",
extensions=CSHARP,
name=_URL,
+ requires=("ConnectAsync", "connectAsync"),
),
SocketCall(
head=re.compile(r"\bWithUrl\s*\(", re.IGNORECASE),
@@ -62,6 +63,7 @@ DOTNET = SocketDialect(
extensions=CSHARP,
name=_URL,
origin=ContractOrigin.ROUTE_REGISTRATION,
+ requires=("MapHub", "mapHub"),
),
),
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket/python.py b/packages/core/src/repowise/core/workspace/extractors/socket/python.py
index 1a59f2327..d794ec52f 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket/python.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket/python.py
@@ -18,6 +18,7 @@ PYTHON_SOCKETS = SocketDialect(
label="@app.websocket",
extensions=PYTHON,
name=Arg(keys=("path",), pos=0),
+ requires=("websocket", "Websocket", "WebSocket"),
),
),
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/socket/socketio.py b/packages/core/src/repowise/core/workspace/extractors/socket/socketio.py
index dcd268804..977d521c1 100644
--- a/packages/core/src/repowise/core/workspace/extractors/socket/socketio.py
+++ b/packages/core/src/repowise/core/workspace/extractors/socket/socketio.py
@@ -148,11 +148,10 @@ _DEFAULT_NAMESPACE_RE = re.compile(r"(?<![\w$.])io\s*\.\s*(?:on|emit|to|in)\s*\(
class SocketIoDialect:
name = "socket.io"
extensions = JS_TS
+ requires = _SOCKET_IO
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if not any(r in content for r in _SOCKET_IO):
- return []
strings = file_strings(ctx)
if strings is None:
return []
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/dialect.py b/packages/core/src/repowise/core/workspace/extractors/topic/dialect.py
index ea9101b59..8b3c02762 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/dialect.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/dialect.py
@@ -27,7 +27,7 @@ from repowise.core.workspace.origins import ContractOrigin
from ..base import line_at
from ..calls import call_sites
-from ..dialect import build_contract
+from ..dialect import build_contract, rows_gate
if TYPE_CHECKING:
import re
@@ -161,6 +161,7 @@ class TopicDialect:
self.name = name
self.calls = calls
self.extensions = frozenset().union(*(c.extensions for c in calls))
+ self.requires = rows_gate(calls)
def extract(self, ctx: ScanContext) -> list[Contract]:
out: list[Contract] = []
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/kafka.py b/packages/core/src/repowise/core/workspace/extractors/topic/kafka.py
index 49d5d0f2e..b5f60350f 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/kafka.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/kafka.py
@@ -23,6 +23,7 @@ KAFKA = TopicDialect(
label="@KafkaListener",
extensions=JAVA,
name=_TOPIC,
+ requires=("@KafkaListener",),
),
TopicCall(
head=re.compile(r"@KafkaListener\s*\("),
@@ -32,6 +33,7 @@ KAFKA = TopicDialect(
name=Arg(keys=("topicPattern",)),
pattern=TOPIC_PATTERN_REGEX,
normalize=unescape_backslashes,
+ requires=("@KafkaListener",),
),
TopicCall(
head=re.compile(r"kafkaTemplate\.send\s*\("),
@@ -39,6 +41,7 @@ KAFKA = TopicDialect(
label="kafkaTemplate.send",
extensions=JAVA,
name=_FIRST,
+ requires=("kafkaTemplate",),
),
TopicCall(
head=re.compile(r"producer\.send\s*\("),
@@ -46,6 +49,7 @@ KAFKA = TopicDialect(
label="producer.send({topic})",
extensions=JS_TS,
name=_TOPIC,
+ requires=("producer",),
),
TopicCall(
head=re.compile(r"consumer\.subscribe\s*\("),
@@ -53,6 +57,7 @@ KAFKA = TopicDialect(
label="consumer.subscribe({topic})",
extensions=JS_TS,
name=_TOPIC,
+ requires=("consumer",),
),
TopicCall(
head=re.compile(r"KafkaConsumer\s*\("),
@@ -60,6 +65,7 @@ KAFKA = TopicDialect(
label="KafkaConsumer",
extensions=PYTHON,
name=_FIRST,
+ requires=("KafkaConsumer",),
),
TopicCall(
head=re.compile(r"producer\.produce\s*\("),
@@ -68,6 +74,7 @@ KAFKA = TopicDialect(
extensions=PYTHON | JS_TS,
name=Arg(keys=("topic",), pos=0),
origin=ContractOrigin.NAME_HEURISTIC, # read off a variable named `producer`
+ requires=("producer",),
),
TopicCall(
head=re.compile(r"ConsumePartition\s*\("),
@@ -75,6 +82,7 @@ KAFKA = TopicDialect(
label="ConsumePartition",
extensions=GO,
name=_FIRST,
+ requires=("ConsumePartition",),
),
),
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/laravel.py b/packages/core/src/repowise/core/workspace/extractors/topic/laravel.py
index 54549efa7..8e1004f6c 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/laravel.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/laravel.py
@@ -52,7 +52,7 @@ _NAME = PHP_CLASS_NAME
# Every app has this queue: two apps naming it share nothing.
_DEFAULT_QUEUE = "default"
-# A file must contain one of these for any pattern below to run.
+# The dialect's gate: a file must contain one of these for any pattern below to run.
_HINTS = ("dispatch", "Queue::", "ShouldQueue", "job")
# Literal-led heads; the text before a `dispatch(` says which form it is.
@@ -238,8 +238,6 @@ class _LaravelQueuePass:
def scan(self, ctx: ScanContext) -> None:
content = ctx.content
- if not any(h in content for h in _HINTS):
- return
reader = _FileReader(ctx)
if "ShouldQueue" in content and (job := reader.queued_class()) is not None:
self.jobs.setdefault(job.name, job)
@@ -292,6 +290,7 @@ class _LaravelQueuePass:
class LaravelQueueDialect:
name = _BROKER
extensions = PHP
+ requires = _HINTS
def repo_pass(self) -> _LaravelQueuePass:
return _LaravelQueuePass()
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/nats.py b/packages/core/src/repowise/core/workspace/extractors/topic/nats.py
index 7fe0e5cab..6e60ca228 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/nats.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/nats.py
@@ -29,6 +29,7 @@ NATS = TopicDialect(
name=Arg(pos=0),
kind=TOPIC_KIND_SUBJECT,
pattern=TOPIC_PATTERN_NATS,
+ requires=("Subscribe", "subscribe"),
),
TopicCall(
head=re.compile(_RECEIVER + r"(?:Publish|publish)\s*\("),
@@ -37,6 +38,7 @@ NATS = TopicDialect(
extensions=_LANGUAGES,
name=Arg(pos=0),
kind=TOPIC_KIND_SUBJECT,
+ requires=("Publish", "publish"),
),
),
)
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/nestjs.py b/packages/core/src/repowise/core/workspace/extractors/topic/nestjs.py
index 99c40f77b..dc66c1e69 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/nestjs.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/nestjs.py
@@ -94,11 +94,10 @@ def _pattern(raw: str, strings: FileStrings) -> str | None:
class NestMicroservicesDialect:
name = "nestjs"
extensions = JS_TS
+ requires = _MICROSERVICES # handlers and ClientProxy both come from this package
def extract(self, ctx: ScanContext) -> list[Contract]:
content = ctx.content
- if not any(r in content for r in _MICROSERVICES):
- return [] # handlers and ClientProxy both come from this package
calls = list(_HANDLERS)
names = set(typed_receivers(content, _PROXY_TYPE_RE))
if names:
diff --git a/packages/core/src/repowise/core/workspace/extractors/topic/rabbitmq.py b/packages/core/src/repowise/core/workspace/extractors/topic/rabbitmq.py
index b5366f283..3c8428a69 100644
--- a/packages/core/src/repowise/core/workspace/extractors/topic/rabbitmq.py
+++ b/packages/core/src/repowise/core/workspace/extractors/topic/rabbitmq.py
@@ -38,6 +38,7 @@ RABBITMQ = TopicDialect(
extensions=JAVA,
name=Arg(keys=("queues", "queue")),
kind=TOPIC_KIND_QUEUE,
+ requires=("@RabbitListener",),
),
TopicCall(
head=re.compile(r"rabbitTemplate\.convertAndSend\s*\("),
@@ -49,6 +50,7 @@ RABBITMQ = TopicDialect(
# `(exchange, routingKey, message)`; in the two-argument form the
# second is the message, which does not resolve, so it routes nowhere.
routing_key=Arg(pos=1),
+ requires=("rabbitTemplate",),
),
TopicCall(
head=re.compile(r"\.\s*consume\s*\("),
@@ -98,6 +100,7 @@ RABBITMQ = TopicDialect(
name=Arg(keys=("queue",), pos=0),
kind=TOPIC_KIND_QUEUE,
origin=ContractOrigin.NAME_HEURISTIC, # a variable named `channel`
+ requires=("basic_consume",),
),
TopicCall(
head=re.compile(r"channel\.basic_publish\s*\("),
@@ -109,6 +112,7 @@ RABBITMQ = TopicDialect(
origin=ContractOrigin.NAME_HEURISTIC, # a variable named `channel`
routing_key=Arg(keys=("routing_key",)),
default_exchange=True,
+ requires=("basic_publish",),
),
TopicCall(
head=re.compile(r"channel\.queue_bind\s*\("),
@@ -120,6 +124,7 @@ RABBITMQ = TopicDialect(
origin=ContractOrigin.NAME_HEURISTIC, # a variable named `channel`
routing_key=Arg(keys=("routing_key",), pos=2),
queue=Arg(keys=("queue",), pos=0),
+ requires=("queue_bind",),
),
# php-amqplib: `basic_publish($msg, $exchange = '', $routing_key = '')`.
TopicCall(
diff --git a/tests/unit/workspace/test_dialect_comments.py b/tests/unit/workspace/test_dialect_comments.py
new file mode 100644
index 000000000..116ec2be2
--- /dev/null
+++ b/tests/unit/workspace/test_dialect_comments.py
@@ -0,0 +1,147 @@
+"""A match in a comment is not a contract, for every dialect, checked in one place.
+
+Doc comments show usage as example code. The engine checks a match against the
+file's comment mask (built once, and only when something matched): a contract
+whose line is nothing but a comment is dropped, and a table row's call in a
+comment is never read, so a commented-out decorator cannot set state either.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from repowise.core.workspace.extractors import (
+ GrpcExtractor,
+ HttpExtractor,
+ TopicExtractor,
+)
+from repowise.core.workspace.extractors.base import ScanContext
+from repowise.core.workspace.extractors.dialect import run_dialect
+from repowise.core.workspace.extractors.http.nestjs import NestDialect
+
+
+def _http(rel_path: str, content: str) -> list[tuple[str, str, int | None]]:
+ suffix = Path(rel_path).suffix
+ found = HttpExtractor().extract(Path("."), "svc", files=[(rel_path, suffix, content)])
+ return sorted((c.role, c.contract_id, c.line) for c in found)
+
+
+# The Sock Shop front-end's `helpers/index.js`: a JSDoc usage example that was
+# read as a provider and as a consumer of an external host.
+SOCK_SHOP_HELPER = """\
+var request = require("request");
+var helpers = {};
+
+ /* Public: performs an HTTP GET request to the given URL
+ *
+ * Examples:
+ *
+ * app.get("/users", function(req, res) {
+ * helpers.simpleHttpRequest("http://api.example.org/users", res, function(err) {
+ * res.send({ error: err });
+ * });
+ * });
+ */
+helpers.simpleHttpRequest = function(url, res, next) {};
+module.exports = helpers;
+"""
+
+
+def test_a_jsdoc_example_is_neither_a_provider_nor_a_consumer() -> None:
+ assert _http("helpers/index.js", SOCK_SHOP_HELPER) == []
+
+
+def test_a_line_comment_route_is_dropped_and_the_live_one_kept() -> None:
+ content = (
+ "const app = express();\n"
+ "// app.get('/old', legacy);\n"
+ "app.get('/new', handler); // was app.get('/older', h)\n"
+ )
+ assert _http("server.js", content) == [("provider", "http::GET::/new", 3)]
+
+
+def test_a_hash_comment_route_in_python_is_dropped() -> None:
+ content = (
+ "from fastapi import APIRouter\n"
+ "router = APIRouter()\n"
+ "# @router.post('/disabled')\n"
+ "@router.get('/live')\n"
+ "def live(): ...\n"
+ )
+ assert _http("api.py", content) == [("provider", "http::GET::/live", 4)]
+
+
+def test_a_go_doc_comment_request_is_dropped() -> None:
+ content = (
+ 'import "net/http"\n'
+ "// Fetch calls the API, for example:\n"
+ '// req, _ := http.NewRequest("GET", "https://host/v1/items", nil)\n'
+ "func Fetch() {\n"
+ '\thttp.Get("https://host/v1/orders")\n'
+ "}\n"
+ )
+ assert _http("client.go", content) == [("consumer", "http::GET::/v1/orders", 5)]
+
+
+def test_a_url_with_a_scheme_is_not_read_as_a_comment() -> None:
+ content = 'import "net/http"\nfunc F() { http.Get("http://host/v1/x") }\n'
+ assert _http("client.go", content) == [("consumer", "http::GET::/v1/x", 2)]
+
+
+def test_a_commented_out_controller_does_not_set_the_prefix() -> None:
+ # The decorator is skipped where it is matched, so the class takes the
+ # live `@Controller`, not the one above it in a comment.
+ content = (
+ "import { Controller, Get } from '@nestjs/common';\n"
+ "// @Controller('legacy')\n"
+ "@Controller('orders')\n"
+ "export class OrdersController {\n"
+ " @Get(':id')\n"
+ " find() {}\n"
+ "}\n"
+ )
+ ctx = ScanContext("svc", "src/orders.controller.ts", ".ts", content, {"nest-app:": ""})
+ found = run_dialect(NestDialect(), ctx)
+ assert [c.contract_id for c in found] == ["http::GET::/orders/{param}"]
+
+
+def test_a_commented_topic_call_is_not_a_contract() -> None:
+ content = (
+ "@Service\n"
+ "class Orders {\n"
+ " /* kafkaTemplate.send(\"orders.legacy\", order); */\n"
+ " void place() { kafkaTemplate.send(\"orders.placed\", order); }\n"
+ "}\n"
+ )
+ found = TopicExtractor().extract(
+ Path("."), "svc", files=[("src/Orders.java", ".java", content)]
+ )
+ assert [c.contract_id for c in found] == ["topic::orders.placed"]
+
+
+def test_a_commented_grpc_stub_is_not_a_consumer() -> None:
+ content = (
+ "import grpc\n"
+ "# stub = LegacyServiceStub(channel)\n"
+ "stub = OrdersServiceStub(channel)\n"
+ )
+ found = GrpcExtractor().extract(Path("."), "svc", files=[("client.py", ".py", content)])
+ assert [c.contract_id for c in found] == ["grpc::OrdersService/*"]
+
+
+def test_a_rust_attribute_route_is_code_not_a_hash_comment() -> None:
+ content = '#[get("/health")]\nasync fn health() -> &\'static str { "ok" }\n'
+ assert _http("src/main.rs", content) == [("provider", "http::GET::/health", 1)]
+
+
+def test_a_python_docstring_example_is_still_read() -> None:
+ # Ceiling: masking keeps strings, so an example inside a docstring is code
+ # as far as the mask can tell.
+ content = (
+ "from fastapi import FastAPI\n"
+ "app = FastAPI()\n"
+ '"""\n'
+ "@app.get('/from-docstring')\n"
+ '"""\n'
+ )
+ assert _http("main.py", content) == [("provider", "http::GET::/from-docstring", 4)]
diff --git a/tests/unit/workspace/test_dialect_registry.py b/tests/unit/workspace/test_dialect_registry.py
new file mode 100644
index 000000000..ab0c9e6d9
--- /dev/null
+++ b/tests/unit/workspace/test_dialect_registry.py
@@ -0,0 +1,113 @@
+"""Every registered dialect declares what the shared engine needs.
+
+A dialect is one module plus one registry line, and the engine
+(:func:`run_dialect`) does the gating and the comment check. These tests fail
+when a registered entry leaves out what that engine reads, so a new dialect
+cannot skip it: a gate (tokens one of which a file must hold) on every entry,
+and on every row of a table dialect, which also names its origin from the
+closed vocabulary in :mod:`repowise.core.workspace.origins`.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from repowise.core.workspace.extractors import data, grpc, http, socket, topic
+from repowise.core.workspace.extractors.base import ScanContext
+from repowise.core.workspace.extractors.dialect import EVERY_FILE, run_dialect
+from repowise.core.workspace.origins import ContractOrigin
+
+REGISTERED = (
+ *http.PROVIDER_DIALECTS,
+ *http.CONSUMER_DIALECTS,
+ *topic.DIALECTS,
+ *socket.DIALECTS,
+ *grpc.DIALECTS,
+ *data.PROVIDER_DIALECTS,
+ *data.CONSUMER_DIALECTS,
+)
+
+#: Contract file formats, where the extension alone says the file is one.
+FORMAT_EXTENSIONS = frozenset({".proto", ".sql", ".prisma"})
+
+
+def _ids(dialect: object) -> str:
+ return f"{type(dialect).__name__}:{dialect.name}"
+
+
[email protected]("dialect", REGISTERED, ids=_ids)
+def test_every_dialect_declares_a_gate(dialect) -> None:
+ gate = getattr(dialect, "requires", None)
+ assert isinstance(gate, tuple) and gate, f"{dialect.name} declares no `requires` gate"
+ assert all(isinstance(t, str) for t in gate)
+ if gate == EVERY_FILE:
+ assert dialect.extensions <= FORMAT_EXTENSIONS, (
+ f"{dialect.name} reads source code: name the tokens a match needs"
+ )
+ else:
+ assert all(gate), f"{dialect.name} gates on an empty token, which every file holds"
+
+
[email protected]("dialect", REGISTERED, ids=_ids)
+def test_every_table_row_declares_a_gate_and_an_origin(dialect) -> None:
+ rows = getattr(dialect, "calls", ())
+ for row in rows:
+ assert row.requires, f"{dialect.name}: a row with head {row.head.pattern!r} has no gate"
+ assert isinstance(row.origin, ContractOrigin), f"{dialect.name}: {row.head.pattern!r}"
+ language = getattr(dialect, "_lang", None) # a gRPC stub table
+ for rule in getattr(language, "rules", ()):
+ assert isinstance(rule.origin, ContractOrigin), f"{dialect.name}: {rule.pattern.pattern!r}"
+
+
+def test_dialect_names_are_unique_per_contract_type() -> None:
+ for registry in (
+ http.PROVIDER_DIALECTS + http.CONSUMER_DIALECTS,
+ topic.DIALECTS,
+ socket.DIALECTS,
+ grpc.DIALECTS,
+ data.PROVIDER_DIALECTS + data.CONSUMER_DIALECTS,
+ ):
+ names = [d.name for d in registry]
+ assert len(names) == len(set(names))
+
+
+class _CountingText(str):
+ """File text that counts each substring test made on it."""
+
+ def __new__(cls, text: str, counts: dict[str, int]):
+ self = super().__new__(cls, text)
+ self.counts = counts
+ return self
+
+ def __contains__(self, token: object) -> bool:
+ self.counts[token] = self.counts.get(token, 0) + 1
+ return super().__contains__(token)
+
+
+def test_a_token_is_searched_for_once_per_file_however_many_dialects_name_it() -> None:
+ # `producer` gates the kafka dialect and two of its rows; `publish` gates
+ # NATS and one of its rows. Each is looked for once.
+ counts: dict[str, int] = {}
+ text = _CountingText("producer.send({ topic: 'orders' })\nnc.publish('audit', x)\n", counts)
+ ctx = ScanContext("svc", "src/a.ts", ".ts", text)
+ for dialect in topic.DIALECTS:
+ run_dialect(dialect, ctx)
+ assert counts.get("producer") == 1
+ assert counts.get("publish") == 1
+
+
+def test_a_dialect_is_not_run_on_a_file_without_its_tokens() -> None:
+ class Probe:
+ name = "probe"
+ extensions = frozenset({".py"})
+ requires = ("needle",)
+ calls = 0
+
+ def extract(self, ctx):
+ Probe.calls += 1
+ return []
+
+ run_dialect(Probe(), ScanContext("svc", "a.py", ".py", "haystack only\n"))
+ assert Probe.calls == 0
+ run_dialect(Probe(), ScanContext("svc", "a.py", ".py", "a needle here\n"))
+ assert Probe.calls == 1
diff --git a/tests/unit/workspace/test_extraction_dialect_budget.py b/tests/unit/workspace/test_extraction_dialect_budget.py
new file mode 100644
index 000000000..de8a07665
--- /dev/null
+++ b/tests/unit/workspace/test_extraction_dialect_budget.py
@@ -0,0 +1,112 @@
+"""Time budget: no dialect costs what an ungated scan of every file costs.
+
+Every dialect runs over every file of its extensions, so one that skips its
+gate, or gates on a token every file holds while scanning with an
+identifier-led regex, makes extraction grow with each dialect added. Measured
+on an 8,141-file corpus, the five heaviest dialects before their gates cost
+650 to 1,200 microseconds per file; gated, every dialect costs under a quarter
+of the reference below.
+
+Wall time is machine-dependent, so the budget is relative: each dialect is
+timed against one identifier-led regex pass over the same text in the same
+process (the shape of an ungated dialect), and may cost at most 40% of it.
+The text is this repository's own source, given each dialect's extension, so
+a Java or Go dialect is timed on code-shaped text too.
+"""
+
+from __future__ import annotations
+
+import re
+import time
+from pathlib import Path
+
+import pytest
+
+from repowise.core.workspace.extractors import data, grpc, http, socket, topic
+from repowise.core.workspace.extractors.base import ScanContext
+from repowise.core.workspace.extractors.dialect import run_dialect
+
+_ROOT = Path(__file__).resolve().parents[3] / "packages"
+_SAMPLES = (("core/src/repowise/core", "*.py", ".py"), ("ui/src", "*.tsx", ".tsx"))
+_PER_SAMPLE = 40
+_ROUNDS = 5
+#: A dialect may cost at most this share of the reference scan.
+_BUDGET = 0.4
+
+# An ungated dialect's shape: tried at every identifier of every file.
+_REFERENCE_RE = re.compile(r"(\w+)\s*\.\s*(\w+)\s*\(")
+
+#: Contract file formats are read whole by a parser: every such file is one.
+_FORMATS = frozenset({".proto", ".sql", ".prisma"})
+
+SOURCE_DIALECTS = tuple(
+ d
+ for d in (
+ *http.PROVIDER_DIALECTS,
+ *http.CONSUMER_DIALECTS,
+ *topic.DIALECTS,
+ *socket.DIALECTS,
+ *grpc.DIALECTS,
+ *data.PROVIDER_DIALECTS,
+ *data.CONSUMER_DIALECTS,
+ )
+ if not d.extensions <= _FORMATS
+)
+
+
+def _texts() -> dict[str, list[str]]:
+ out: dict[str, list[str]] = {}
+ for folder, pattern, kind in _SAMPLES:
+ paths = sorted((_ROOT / folder).rglob(pattern))[:_PER_SAMPLE]
+ out[kind] = [p.read_text(encoding="utf-8", errors="replace") for p in paths]
+ return out
+
+
+def _timed(run) -> float:
+ start = time.perf_counter()
+ run()
+ return time.perf_counter() - start
+
+
[email protected](scope="module")
+def texts() -> dict[str, list[str]]:
+ found = _texts()
+ if not all(found.values()):
+ pytest.skip("the repository's own sources are not in this checkout")
+ return found
+
+
+def _kinds(dialect) -> list[tuple[str, str]]:
+ """``(text kind, suffix)`` pairs to time *dialect* on: Python text for a
+ Python extension, C-shaped text for every other one it reads."""
+ exts = sorted(dialect.extensions)
+ py = [e for e in exts if e in (".py", ".pyi")]
+ other = [e for e in exts if e not in (".py", ".pyi")]
+ return [(".py", e) for e in py[:1]] + [(".tsx", e) for e in other[:1]]
+
+
[email protected]("dialect", SOURCE_DIALECTS, ids=lambda d: f"{type(d).__name__}:{d.name}")
+def test_a_dialect_costs_less_than_an_ungated_scan(dialect, texts) -> None:
+ for kind, suffix in _kinds(dialect):
+ files = texts[kind]
+
+ def reference(files=files) -> None:
+ for text in files:
+ for _ in _REFERENCE_RE.finditer(text):
+ pass
+
+ def run(files=files, suffix=suffix) -> None:
+ # A fresh context per round: the per-file gate memo must not carry over.
+ for i, text in enumerate(files):
+ run_dialect(dialect, ScanContext("svc", f"src/f{i}{suffix}", suffix, text))
+
+ # Interleaved, so a stall on a busy worker hits both sides of one round,
+ # and the best round of each is compared.
+ spent = ref = float("inf")
+ for _ in range(_ROUNDS):
+ ref = min(ref, _timed(reference))
+ spent = min(spent, _timed(run))
+ assert spent <= _BUDGET * ref, (
+ f"{dialect.name} took {spent * 1e3:.1f} ms on {suffix} where an ungated scan of the "
+ f"same {len(files)} files took {ref * 1e3:.1f} ms: check its `requires` gate"
+ )
diff --git a/tests/unit/workspace/test_kotlin_clients.py b/tests/unit/workspace/test_kotlin_clients.py
index ed061c403..cb44e7083 100644
--- a/tests/unit/workspace/test_kotlin_clients.py
+++ b/tests/unit/workspace/test_kotlin_clients.py
@@ -13,6 +13,7 @@ from __future__ import annotations
from pathlib import Path
from repowise.core.workspace.extractors.base import ScanContext
+from repowise.core.workspace.extractors.dialect import run_dialect
from repowise.core.workspace.extractors.http import HttpExtractor
from repowise.core.workspace.extractors.http.fastapi import FastApiDialect
from repowise.core.workspace.extractors.http.kotlin_clients import (
@@ -132,7 +133,7 @@ def create_account():
def contracts(source: str, rel_path: str = "src/Sync.kt") -> list:
ctx = ScanContext("portal", rel_path, ".kt", source, {}, None)
- return KotlinClientsDialect().extract(ctx)
+ return run_dialect(KotlinClientsDialect(), ctx)
def ids(source: str, rel_path: str = "src/Sync.kt") -> list[str]: