Skip to content

Codebase intelligence for AI and humans: code health scores, auto-generated docs, git analytics, dead code detection, and architectural decisions via MCP.

refactor(test-paths): one classifier for contract extraction, keeping possible contracts (#3290)

Raghav Chamadiyaf6dd9155e276
diff --git a/docs/scale/WORKSPACES.md b/docs/scale/WORKSPACES.md
index af0b97922..81bb52e6d 100644
--- a/docs/scale/WORKSPACES.md
+++ b/docs/scale/WORKSPACES.md
@@ -197,12 +197,12 @@ contracts:
- "generated/**"
```
-Directories named `tests/`, `__tests__/`, and `__mocks__/` are excluded by name;
-`test/`, `spec/`, and `e2e/` are deliberately *not* excluded by directory, since those
-names double as legitimate product directories in some codebases. Regardless of
-directory, filenames matching `test_*.py`, `*_test.py`, `*_test.go`, `*.test.*`,
-`*.spec.*`, `*.e2e.*`, or `conftest.py` are always excluded: a route or topic that
-exists only in a test is a fixture, not a service contract. Calls to a literal
+Test material is excluded using the index's test classifier, and a file is never
+dropped on a guess that could lose a real contract. Test trees and test-named source
+files are skipped; spec files such as `openapi.yaml` and `.proto` are always scanned.
+
+A route or topic that exists only in a test is a fixture, not a service contract.
+Calls to a literal
third-party host (Stripe, Formspree, ...) that is not a workspace service are
excluded from matching and reported under the `external_host` diagnostics reason.
diff --git a/packages/core/src/repowise/core/analysis/execution_graph.py b/packages/core/src/repowise/core/analysis/execution_graph.py
index 2bedd3907..5bdf69f84 100644
--- a/packages/core/src/repowise/core/analysis/execution_graph.py
+++ b/packages/core/src/repowise/core/analysis/execution_graph.py
@@ -25,7 +25,9 @@ from typing import Any, Literal, TypeVar
from repowise.core.ingestion.models import EXECUTION_EDGE_TYPES
UNRELIABLE_EXECUTION_ORIGINS = frozenset({"global_unique"})
-_EXCLUDED_EXECUTION_PATHS = re.compile(
+# Not a test classifier: flows also leave out demos, examples, scripts and
+# benchmarks, which are no more product code than tests are.
+_NON_PRODUCT_FLOW_PATH_RE = re.compile(
r"(test[s_/]|_test\.|\.test\.|\.spec\.|__tests__|conftest|"
r"fixture[s]?[/.]|mock[s]?[/.]|stub[s]?[/.]|fake[s]?[/.]|"
r"demo[_/.]|example[s]?[/.]|sample[s]?[/.]|benchmark[s]?[/.]|"
@@ -74,7 +76,7 @@ def is_walkable_execution_edge(
def is_excluded_execution_path(path: str) -> bool:
"""Whether a flow target is test/demo/fixture/tooling material."""
- return bool(_EXCLUDED_EXECUTION_PATHS.search(path))
+ return bool(_NON_PRODUCT_FLOW_PATH_RE.search(path))
def _append_unique(
diff --git a/packages/core/src/repowise/core/ingestion/languages/specs/javascript.py b/packages/core/src/repowise/core/ingestion/languages/specs/javascript.py
index 9a3e119a0..8629de318 100644
--- a/packages/core/src/repowise/core/ingestion/languages/specs/javascript.py
+++ b/packages/core/src/repowise/core/ingestion/languages/specs/javascript.py
@@ -7,7 +7,7 @@ SPEC = LanguageSpec(
tag="javascript",
display_name="JavaScript",
import_support="full",
- test_infixes=(".test.", ".spec."),
+ test_infixes=(".test.", ".spec.", ".e2e."),
extensions=frozenset({".js", ".jsx", ".mjs", ".cjs"}),
grammar_package="tree_sitter_javascript",
scm_file="javascript.scm",
diff --git a/packages/core/src/repowise/core/ingestion/languages/specs/typescript.py b/packages/core/src/repowise/core/ingestion/languages/specs/typescript.py
index d921e75c6..7ff2af94a 100644
--- a/packages/core/src/repowise/core/ingestion/languages/specs/typescript.py
+++ b/packages/core/src/repowise/core/ingestion/languages/specs/typescript.py
@@ -61,7 +61,8 @@ SPEC = LanguageSpec(
# ``.suite.`` holds ``describe`` blocks a ``.test.ts`` file registers.
# Infixes apply to every source extension, so a benchmark.js
# ``array.suite.js`` reads as test too: accepted, none seen in the corpora.
- test_infixes=(".test.", ".spec.", ".test-d.", ".suite."),
+ # ``.e2e.`` names end-to-end suites and their helpers (``gateway.e2e.ts``).
+ test_infixes=(".test.", ".spec.", ".test-d.", ".suite.", ".e2e."),
extensions=frozenset({".ts", ".tsx", ".mts", ".cts"}),
grammar_package="tree_sitter_typescript",
grammar_loader="language_typescript",
diff --git a/packages/core/src/repowise/core/test_paths.py b/packages/core/src/repowise/core/test_paths.py
index 05c133b98..344549f61 100644
--- a/packages/core/src/repowise/core/test_paths.py
+++ b/packages/core/src/repowise/core/test_paths.py
@@ -69,9 +69,12 @@ if TYPE_CHECKING:
# Directory segments that mark every file beneath them as test material,
# whatever the filename. ``__test__`` is the Jest variant of ``__tests__``;
-# ``integration_test`` is the directory Flutter's integration tests must live in.
+# ``integration_test`` is the directory Flutter's integration tests must live in;
+# ``testsuite`` is the one-word suite directory (``gcc/testsuite/``, ``flask/testsuite/``).
+# Ceiling: a product package named ``testsuite`` (a library for running suites)
+# reads as test material too; none seen so far.
_TEST_DIR_TOKENS: frozenset[str] = frozenset(
- {"test", "tests", "__tests__", "__test__", "e2e", "integration_test"}
+ {"test", "tests", "__tests__", "__test__", "e2e", "integration_test", "testsuite"}
)
# A compound segment (``e2e-tests``, ``pkg_tests``, ``client-e2e``) is a test
@@ -188,6 +191,7 @@ class _Conventions:
dir_paths: tuple[tuple[str, ...], ...]
dir_wildcards: tuple[tuple[str, str], ...]
dir_suffixes: tuple[str, ...]
+ unambiguous_dir_suffixes: tuple[str, ...]
lang_dir_tokens: dict[str, frozenset[str]]
@@ -224,6 +228,12 @@ def _conventions() -> _Conventions:
),
# Case-sensitive .NET sibling project dirs (Foo.Tests/, Foo.Specs/).
dir_suffixes=REGISTRY.test_dir_suffixes(),
+ # Without the ``spec`` forms, which also name specification projects.
+ unambiguous_dir_suffixes=tuple(
+ s
+ for s in REGISTRY.test_dir_suffixes()
+ if s.lstrip(".").lower() not in _AMBIGUOUS_TEST_DIR_TOKENS
+ ),
lang_dir_tokens=REGISTRY.test_dir_tokens_by_language(),
)
@@ -294,7 +304,7 @@ def _is_test_dir(
) -> bool:
"""Whether any directory segment marks this path as sitting in a test tree."""
return _has_test_segment(segments, original, filename, language) or _has_test_layout(
- segments, original
+ segments, original, _conventions().dir_suffixes
)
@@ -319,16 +329,24 @@ def _has_test_segment(
def _is_test_segment(seg: str, orig: str, lang_tokens: frozenset[str], corroborated: bool) -> bool:
+ if _is_named_test_dir(seg):
+ return True
words = _words(seg)
head = words[-1] if words else ""
- if seg in _TEST_DIR_TOKENS or head in _TEST_DIR_HEAD_WORDS:
- return True
if head in _AMBIGUOUS_TEST_DIR_TOKENS and (head in lang_tokens or corroborated):
return True
return _CAMEL_TESTS_HEAD_RE.search(orig) is not None
-def _has_test_layout(segments: list[str], original: list[str]) -> bool:
+def _is_named_test_dir(seg: str) -> bool:
+ """Whether a directory's name alone says it holds tests (``tests``, ``e2e-tests``)."""
+ words = _words(seg)
+ return seg in _TEST_DIR_TOKENS or (bool(words) and words[-1] in _TEST_DIR_HEAD_WORDS)
+
+
+def _has_test_layout(
+ segments: list[str], original: list[str], dir_suffixes: tuple[str, ...]
+) -> bool:
"""Whether the directories form a known test layout (``src/test/java``)."""
rules = _conventions()
return (
@@ -339,7 +357,7 @@ def _has_test_layout(segments: list[str], original: list[str]) -> bool:
_has_wildcard_pair(segments, original, prefix_seg, camel_suffix)
for prefix_seg, camel_suffix in rules.dir_wildcards
)
- or any(seg.endswith(rules.dir_suffixes) for seg in original)
+ or any(seg.endswith(dir_suffixes) for seg in original)
)
@@ -487,6 +505,39 @@ def is_unambiguous_test_path(path: str, language: str | None = None) -> bool:
return "src" not in lowered[:-1]
+def is_skippable_test_path(path: str) -> bool:
+ """Whether a scan that drops test material should drop *path*.
+
+ For callers that skip files outright, such as contract extraction, where a
+ wrong yes silently loses a real contract. Stricter than
+ :func:`is_test_related_path` in three ways:
+
+ - A non-source file (``openapi.yaml``, ``api.proto``) counts only inside a
+ named test tree (``tests/``, ``e2e/``, ``src/test/java``, ``Foo.Tests/``):
+ not under ``spec/``, ``testdata/``, ``Foo.Specs/`` or ``LoadTests/``.
+ - A source file needs a named test tree, a test or support filename, or a
+ ``__mocks__``-style dir; a PascalCase ``LoadTests/`` folder is not enough.
+ - Under ``src``, a Python test-shaped name alone does not count
+ (``src/api/test_routes.py``), as in :func:`is_unambiguous_test_path`;
+ other toolchains' names (``client.test.ts``, ``FooTest.kt``) still do.
+ """
+ if not is_test_related_path(path):
+ return False
+ original, lowered = _parts(path)
+ filename, segments = original[-1], lowered[:-1]
+ rules = _conventions()
+ if any(_is_named_test_dir(seg) for seg in segments) or _has_test_layout(
+ segments, original[:-1], rules.unambiguous_dir_suffixes
+ ):
+ return True
+ ext = PurePosixPath(filename.lower()).suffix
+ if ext not in rules.source_exts:
+ return False
+ if _is_support_name(filename) or any(_is_support_anywhere_dir(seg) for seg in segments):
+ return True
+ return _is_test_name(filename) and (ext != ".py" or "src" not in segments)
+
+
def is_test_to_production_pair(
code_path: str, partner_path: str, *, code_language: str | None = None
) -> bool:
diff --git a/packages/core/src/repowise/core/workspace/extractors/base.py b/packages/core/src/repowise/core/workspace/extractors/base.py
index 9685bbcfc..56c29e5f2 100644
--- a/packages/core/src/repowise/core/workspace/extractors/base.py
+++ b/packages/core/src/repowise/core/workspace/extractors/base.py
@@ -18,41 +18,11 @@ from fnmatch import fnmatch
from pathlib import Path
from typing import TYPE_CHECKING
-from repowise.core.ingestion.languages.registry import REGISTRY
+from repowise.core.test_paths import is_skippable_test_path
if TYPE_CHECKING:
from repowise.core.workspace.repo_index import RepoIndex
-# Path segments that mark a test/mock tree. A route handler or topic publisher
-# that exists only under one of these is a fixture, not a real service contract,
-# so it is excluded from contract extraction by default (configurable via the
-# workspace ``contracts.exclude_globs``). Kept to unambiguous test-only dir names:
-# singular ``test``/``spec``/``e2e`` are intentionally excluded because they
-# double as legitimate product directories (an OpenAPI ``spec/``, a ``test/``
-# feature). Test *files* under those dirs are still caught by the filename
-# patterns below.
-_TEST_DIR_SEGMENTS = frozenset({"tests", "__tests__", "__mocks__"})
-
-# .NET test projects are sibling directories named after the project under test
-# (``Foo.Tests/``, case-sensitive), not a ``tests/`` tree. Read from the language
-# registry so a suffix added there is excluded here too. ``.Specs`` is left out
-# for the reason ``spec/`` is above: it can hold real OpenAPI or proto contracts.
-_TEST_PROJECT_DIR_SUFFIXES = tuple(
- suffix for suffix in REGISTRY.test_dir_suffixes() if suffix != ".Specs"
-)
-
-# Filename patterns that mark a test file regardless of directory.
-_TEST_FILE_PATTERNS = (
- "test_*.py",
- "*_test.py",
- "*_test.go",
- "*.test.*",
- "*.spec.*",
- "*.e2e.*",
- "conftest.py",
-)
-
-
# Repowise's own contract-extractor sources. Their docstrings and comments spell
# out the syntax each dialect matches (``@app.get("/path")``, ``topic::orders``,
# ``AuthServiceStub``), so scanning them yields contracts for endpoints and
@@ -71,17 +41,6 @@ def line_at(content: str, offset: int) -> int:
return content.count("\n", 0, offset) + 1
-def is_test_path(rel_path: str) -> bool:
- """True when *rel_path* (POSIX) lives in a test tree or is a test file."""
- parts = rel_path.split("/")
- if any(
- seg in _TEST_DIR_SEGMENTS or seg.endswith(_TEST_PROJECT_DIR_SUFFIXES) for seg in parts[:-1]
- ):
- return True
- name = parts[-1]
- return any(fnmatch(name, pat) for pat in _TEST_FILE_PATTERNS)
-
-
def make_exclude_predicate(
extra_globs: tuple[str, ...] = (),
*,
@@ -89,7 +48,9 @@ def make_exclude_predicate(
) -> Callable[[str], bool]:
"""Build a ``rel_path -> bool`` skip predicate for contract extraction.
- Skips the default test/spec trees (unless *exclude_tests* is False), the
+ Skips test material per :func:`is_skippable_test_path` (unless
+ *exclude_tests* is False): a route handler or topic publisher that exists
+ only there is a fixture, not a real service contract. Also skips the
extractors' own source tree (see :data:`_SELF_EXCLUDE_GLOBS`), plus any
user-supplied ``extra_globs`` (matched against the full POSIX path and the
bare filename).
@@ -97,7 +58,7 @@ def make_exclude_predicate(
globs = _SELF_EXCLUDE_GLOBS + tuple(extra_globs)
def skip(rel_path: str) -> bool:
- if exclude_tests and is_test_path(rel_path):
+ if exclude_tests and is_skippable_test_path(rel_path):
return True
name = rel_path.rsplit("/", 1)[-1]
return any(fnmatch(rel_path, g) or fnmatch(name, g) for g in globs)
diff --git a/tests/unit/ingestion/test_language_capabilities.py b/tests/unit/ingestion/test_language_capabilities.py
index 26ad0b3ad..e33506d78 100644
--- a/tests/unit/ingestion/test_language_capabilities.py
+++ b/tests/unit/ingestion/test_language_capabilities.py
@@ -67,7 +67,7 @@ class TestParityGoldens:
def test_test_infixes_match_historical_set(self) -> None:
# `.test-d.` is the TypeScript type-test infix (`*.test-d.ts`), `.suite.`
# a vitest suite module (`*.suite.ts`).
- assert set(REGISTRY.test_infixes()) == {".test.", ".spec.", ".test-d.", ".suite."}
+ assert set(REGISTRY.test_infixes()) == {".test.", ".spec.", ".test-d.", ".suite.", ".e2e."}
def test_test_fixture_stems_match_historical_set(self) -> None:
assert frozenset(
diff --git a/tests/unit/test_no_test_path_copies.py b/tests/unit/test_no_test_path_copies.py
index 966362e0d..76ef72b7d 100644
--- a/tests/unit/test_no_test_path_copies.py
+++ b/tests/unit/test_no_test_path_copies.py
@@ -84,10 +84,6 @@ _KNOWN: frozenset[str] = frozenset(
# config-shaped names, because fixing `vitest.config.ts` is a toolchain
# fix rather than a defect.
"packages/core/src/repowise/core/ingestion/git_indexer/fix_shape.py",
- # Excludes singular `test/`, `spec/` and `e2e/` on purpose: a route
- # handler under an OpenAPI `spec/` is a real service contract, and
- # over-excluding drops it from the workspace contract map entirely.
- "packages/core/src/repowise/core/workspace/extractors/base.py",
# pytest's own `python_files` rule, not ours: a fixture is injected
# only into a file pytest collects, so this asks "does pytest run
# this?" rather than "is this test-related code". The shared rules also
diff --git a/tests/unit/test_test_paths.py b/tests/unit/test_test_paths.py
index 482b7846c..8d0592691 100644
--- a/tests/unit/test_test_paths.py
+++ b/tests/unit/test_test_paths.py
@@ -12,6 +12,7 @@ import pytest
from repowise.core.pytest_roots import read_pytest_roots
from repowise.core.test_paths import (
+ is_skippable_test_path,
is_test_path,
is_test_related_path,
is_test_support_path,
@@ -105,6 +106,14 @@ _CORPUS: tuple[tuple[str, str | None, str], ...] = (
("contest/x.py", None, ""),
# e2e suites
("e2e/login.ts", None, "test"),
+ # ...and the `.e2e.` infix beside `.test.`/`.spec.`, suites and their helpers
+ ("src/gateway/gateway.e2e.ts", None, "test"),
+ ("src/gateway/test-helpers.e2e.ts", None, "test"),
+ ("apps/desktop/tsconfig.e2e.json", None, ""),
+ ("src/harness/e2e_utils.py", None, ""),
+ # one-word suite directories (gcc, flask before 0.11)
+ ("flask/testsuite/basic.py", None, "test"),
+ ("gcc/testsuite/gcc.dg/pr123.c", None, "test"),
# production code that merely contains the word: the unanchored
# `test[s_/]` substring rule classified the first three as tests
("src/latest/api.py", None, ""), # #1103 finding 1
@@ -340,6 +349,28 @@ def test_unambiguous_test_paths_need_more_than_a_test_shaped_name_under_src(
assert is_unambiguous_test_path(path) is expected
+ ("path", "expected"),
+ [
+ ("tests/api/openapi.yaml", True),
+ ("Foo.Tests/Clients/Orders.cs", True),
+ ("src/api/client.test.ts", True),
+ ("src/__mocks__/api.ts", True),
+ ("pkg/server/handler_test.go", True),
+ # non-source files need a named test tree
+ ("Foo.Specs/openapi.yaml", False),
+ ("spec/support/openapi.yaml", False),
+ ("testdata/openapi.yaml", False),
+ # a PascalCase suite folder alone, or a Python test name under src
+ ("LoadTests/Runner.cs", False),
+ ("src/api/test_routes.py", False),
+ ("src/app/main.ts", False),
+ ],
+)
+def test_skippable_test_paths_never_drop_a_possible_contract(path: str, expected: bool) -> None:
+ assert is_skippable_test_path(path) is expected
+
+
_ROOTS = read_pytest_roots(
[("pyproject.toml", '[tool.pytest.ini_options]\ntestpaths = ["tests"]\n')]
)
diff --git a/tests/unit/workspace/test_dialect_regressions.py b/tests/unit/workspace/test_dialect_regressions.py
index 947dd17dd..1397aa03f 100644
--- a/tests/unit/workspace/test_dialect_regressions.py
+++ b/tests/unit/workspace/test_dialect_regressions.py
@@ -22,6 +22,8 @@ from __future__ import annotations
from pathlib import Path
+import pytest
+
from repowise.core.workspace.extractors.base import (
ScanContext,
make_exclude_predicate,
@@ -133,16 +135,42 @@ class TestExtractorSelfExclusion:
assert skip("Foo.Worker.Tests/OrderPlacedTests.cs")
assert skip("src/Billing.Tests/Consumers/InvoiceConsumerTests.vb")
- def test_a_dotnet_specs_project_is_still_scanned(self) -> None:
- # Left in for the reason a `spec/` directory is: it can hold real
- # OpenAPI or proto contracts.
- skip = make_exclude_predicate()
- assert not skip("Foo.Specs/openapi.yaml")
-
- def test_a_name_that_only_ends_like_a_test_project_is_still_scanned(self) -> None:
- skip = make_exclude_predicate()
- assert not skip("LoadTests/Runner.cs")
- assert not skip("src/Foo.Testing/Clients/OrdersClient.cs")
+ @pytest.mark.parametrize(
+ "path",
+ [
+ "src/gateway/test-helpers.e2e.ts",
+ "src/api/client.test.ts", # colocated tests stay skipped
+ "src/__mocks__/api.ts",
+ "test/scripts/probe.ts",
+ "scripts/e2e/lib/client.mjs",
+ "tests/contracts/openapi.yaml", # a named test tree takes any file
+ "apps/android/app/src/testThirdParty/java/x/SmsManagerTest.kt",
+ "conftest.py",
+ ],
+ )
+ def test_test_material_is_skipped(self, path: str) -> None:
+ assert make_exclude_predicate()(path)
+
+ @pytest.mark.parametrize(
+ "path",
+ [
+ # Contract files outside a named test tree can be the real thing.
+ "spec/openapi.yaml",
+ "spec/support/openapi.yaml",
+ "Foo.Specs/openapi.yaml",
+ "testdata/openapi.yaml",
+ "api/users.spec.yaml",
+ # A PascalCase suite folder or a shipped `.Testing` library is not
+ # enough for a source file without a test name.
+ "LoadTests/Runner.cs",
+ "src/Foo.Testing/Clients/OrdersClient.cs",
+ # A Python test-shaped name under `src` is a module named for what
+ # it does unless pytest collects it.
+ "src/api/test_routes.py",
+ ],
+ )
+ def test_real_contracts_are_still_scanned(self, path: str) -> None:
+ assert not make_exclude_predicate()(path)
def test_dotnet_test_projects_are_scanned_when_tests_are_included(self) -> None:
skip = make_exclude_predicate(exclude_tests=False)