:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(devex): run warehouse source vendors in their own ci lane (#115221)
Julian Bezc985c0dd7a7e
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.ymlindex 4d3aef94a369..2adf96552ed5 100644--- a/.depot/workflows/ci-backend.yml+++ b/.depot/workflows/ci-backend.yml@@ -306,6 +306,10 @@ jobs:# products/desktop is the imported desktop app; it has its own# desktop-* CI and must not drag this suite onto its PRs- '!products/desktop/**'+ # Warehouse source vendors have their own job in ci-python.yml. They+ # match 'products/**/*.py' above, so a vendor-only diff would start this+ # suite too. The product code the vendors build on stays under backend/.+ - '!products/warehouse_sources/sources/**'# Each product's package.json declares the per-product# backend:test script that turbo-discover reads to# compose the Product matrix. Without this entry,diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.jsindex 67887b8fbbb9..057a4c672c3c 100644--- a/.github/scripts/trunk-impacted-targets.js+++ b/.github/scripts/trunk-impacted-targets.js@@ -642,13 +642,16 @@ const TRIPWIRE_RULES = [// suite's tests run on a PR, so an under-selection can only mask conflicts// that suite's lanes already serialize: the snob shadow, its verdict, and// the testmon fanout list select Django tests, the dagster selector its- // own suite, and the playwright pair the E2E specs.+ // own suite, the warehouse sources selector the vendor tests, and the+ // playwright pair the E2E specs.['tools/snob_backend_test_selection_shadow.py', PYTHON],['tools/test_snob_backend_test_selection_shadow.py', PYTHON],['tools/test_selection_verdict.py', PYTHON],['tools/test_test_selection_verdict.py', PYTHON],['tools/dagster_test_selection.py', PYTHON],['tools/test_dagster_test_selection.py', PYTHON],+ ['tools/warehouse_sources_vendor_selection.py', PYTHON],+ ['tools/test_warehouse_sources_vendor_selection.py', PYTHON],['tools/testmon_high_fanout_files.txt', PYTHON],['tools/playwright_spec_selection.py', FULLSTACK],['tools/test_playwright_spec_selection.py', FULLSTACK],diff --git a/.github/scripts/trunk-impacted-targets.test.js b/.github/scripts/trunk-impacted-targets.test.jsindex 8db12f6ad096..add66d087bd4 100644--- a/.github/scripts/trunk-impacted-targets.test.js+++ b/.github/scripts/trunk-impacted-targets.test.js@@ -2229,6 +2229,7 @@ test('test-selection scripts claim the lanes of the suite they select for', () ='tools/snob_backend_test_selection_shadow.py','tools/test_selection_verdict.py','tools/dagster_test_selection.py',+ 'tools/warehouse_sources_vendor_selection.py','tools/testmon_high_fanout_files.txt',]) {assert.deepEqual(computeTargets([file], CONTEXT), computeTargets(['mypy.ini'], CONTEXT), file)diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.ymlindex 289b4386987f..da5b5ca51424 100644--- a/.github/workflows/ci-python.yml+++ b/.github/workflows/ci-python.yml@@ -70,6 +70,7 @@ jobs:api_ratchet: ${{ steps.filter.outputs.api_ratchet || 'true' }}mypy_cache_key: ${{ steps.filter.outputs.mypy_cache_key || 'false' }}schema_python: ${{ steps.filter.outputs.schema_python || 'true' }}+ warehouse_sources: ${{ steps.filter.outputs.warehouse_sources || 'true' }}steps:# For pull requests it's not necessary to checkout the code, but we# also want this to run on master so we need to checkout@@ -267,6 +268,21 @@ jobs:- '.semgrep/rules/devex/prefer-codegen-api-namespaced.yaml'- 'hogli.yaml'- .github/workflows/ci-python.yml+ # The warehouse source vendors, their SDK and the product code the+ # SDK re-exports.+ warehouse_sources:+ - 'products/warehouse_sources/sources/**'+ - 'products/warehouse_sources/backend/**'+ - 'pyproject.toml'+ - 'uv.lock'+ - 'pytest.ini'+ - 'conftest.py'+ - 'tach.toml'+ - .github/workflows/ci-python.yml+ - .github/actions/setup-python-cached/**+ - .github/actions/setup-uv/**+ - tools/warehouse_sources_vendor_selection.py+ - tools/test_warehouse_sources_vendor_selection.pycode-quality:needs: changes@@ -620,11 +636,151 @@ jobs:fi./bin/hogli lint:api-ratchet+ # The warehouse source vendors under products/warehouse_sources/sources/ need no service:+ # no Postgres, ClickHouse, Redis or Kafka. So a pull request that changes only vendors+ # runs this job, and Backend CI does not start (its `backend` filter excludes that+ # directory). code-quality already runs ruff, mypy and ty on the vendors.+ warehouse-sources:+ needs: changes+ # Skipped on master push: the merge queue already ran this for every landed commit,+ # and the hourly scheduled run tests every vendor. The scheduled run also catches a+ # posthog/ change that breaks a vendor through the SDK, which no filter above sees.+ if: needs.changes.outputs.warehouse_sources == 'true' && github.event_name != 'push'+ name: Warehouse sources checks and tests+ runs-on: depot-ubuntu-24.04+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2+ with:+ # The pull request merge commit and its first parent, the base, so the+ # vendor selection below can diff the two without the GitHub API.+ fetch-depth: 2+ persist-credentials: false++ - name: Set up Python+ uses: ./.github/actions/setup-python-cached+ with:+ python-version: 3.14.7++ - name: Install uv+ id: setup-uv+ uses: ./.github/actions/setup-uv++ - name: Install SAML dependencies+ if: steps.setup-uv.outputs.cache-hit != 'true'+ run: sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl++ - name: Install Python dependencies+ run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev++ # A broken selector must fail here, before it narrows a run.+ - name: Test the vendor selector+ run: python tools/test_warehouse_sources_vendor_selection.py++ # A pull request diff that stays inside vendor directories runs only those vendors.+ # A vendor cannot import another vendor (tach and import-linter enforce that), so the+ # other vendors cannot change. Any other file (the SDK, the conftest.py and other+ # files at the top of the plugin directory, the product, a manifest, this workflow)+ # selects the full run. The merge queue, the+ # hourly run and a manual run have no pull request diff to narrow by, so they also+ # take the full run.+ - name: Select vendors+ id: select+ env:+ EVENT_NAME: ${{ github.event_name }}+ HEAD_REF: ${{ github.head_ref }}+ run: |+ if [ "$EVENT_NAME" != "pull_request" ] || [[ "$HEAD_REF" == trunk-merge/* ]]; then+ python tools/warehouse_sources_vendor_selection.py --full+ else+ git diff --name-only HEAD^1 HEAD | python tools/warehouse_sources_vendor_selection.py+ fi | tee -a "$GITHUB_OUTPUT"++ - name: Check module boundaries (tach)+ run: ./bin/hogli lint:tach++ # tach leaves test files out of its dependency pass. The import-linter contracts+ # for the vendors also check the vendor tests, and one of them fails a vendor that+ # imports a sibling vendor.+ - name: Check vendor imports (import-linter)+ run: lint-imports++ # The Django matrix runs every repo invariant, but it does not start for a vendor-only+ # diff. So this job runs the ones that walk the vendor tree. The full run takes all+ # of them. A vendors run takes the ones that check code a vendor writes, and scopes+ # their walk to the changed vendor directories. The others look for calls into+ # posthog/, products/ or DRF, which a vendor cannot import, so a vendor diff cannot+ # break them. The lists are explicit: a new invariant that walks products/ has to be+ # added here by hand.+ - name: Run repo invariants that scan the vendors+ env:+ MODE: ${{ steps.select.outputs.mode }}+ SCAN_SCOPE: ${{ steps.select.outputs.scan_scope }}+ run: |+ vendor_invariants=(+ test_warehouse_source_static_contract.py+ test_warehouse_source_test_shapes.py+ test_resume_state_staged_before_yield.py+ test_resume_state_saved_by_the_source_generator.py+ test_dataclass_defaults.py+ test_dataclass_secret_fields.py+ test_database_free_test_classes.py+ )+ if [ "$MODE" = "vendors" ]; then+ REPO_INVARIANTS_SCAN_SCOPE="$SCAN_SCOPE" pytest -p no:cacheprovider "${vendor_invariants[@]/#/posthog/test/repo_invariants/}"+ exit 0+ fi+ full_invariants=(+ "${vendor_invariants[@]}"+ test_authentication_credential_types.py+ test_feature_flag_gated_writes.py+ test_feature_flag_raw_reads.py+ test_local_kafka_consumers.py+ test_rbac_shadow_resolver_callers.py+ )+ pytest -p no:cacheprovider -n auto "${full_invariants[@]/#/posthog/test/repo_invariants/}"++ # These tests load the full vendor catalog, so a change to one vendor can fail them.+ # A vendors run keeps the tests of the whole catalog and runs the tests that check one+ # source at a time only for the selected vendors.+ - name: Run vendor catalog tests+ env:+ MODE: ${{ steps.select.outputs.mode }}+ VENDORS: ${{ steps.select.outputs.vendors }}+ run: |+ narrow=()+ if [ "$MODE" = "vendors" ]; then+ narrow=("--source-vendors=${VENDORS// /,}")+ fi+ pytest -p no:cacheprovider products/warehouse_sources/backend/temporal/data_imports/sources/tests --tb=short "${narrow[@]}"++ # The committed credential kind and auth type maps hold one line per vendor. A vendors+ # run skips Backend CI, so these suites run here in both modes, or a vendor change that+ # makes the maps stale would merge. They need no database.+ - name: Run credential map tests+ run: |+ credential_tests=products/warehouse_sources/backend/temporal/data_imports/sources/common/test+ pytest -p no:cacheprovider "$credential_tests/test_credential_kind.py" "$credential_tests/test_credential_auth_type.py" --tb=short++ - name: Run vendor tests+ if: steps.select.outputs.mode == 'full' || steps.select.outputs.vendors != ''+ env:+ MODE: ${{ steps.select.outputs.mode }}+ VENDORS: ${{ steps.select.outputs.vendors }}+ run: |+ plugin_dir=products/warehouse_sources/sources+ targets=("$plugin_dir/")+ if [ "$MODE" = "vendors" ]; then+ read -ra vendors <<< "$VENDORS"+ targets=("${vendors[@]/#/$plugin_dir/}")+ fi+ pytest -p no:cacheprovider "${targets[@]}" -m 'not async_migrations' --reruns=2 --reruns-delay=1 --tb=short+# Collate job - single required status check for branch protection.# The code-quality job skips when no Python changes, but this job always# runs and reports success, allowing PRs to merge.python_tests:- needs: [changes, code-quality, api-ratchet]+ needs: [changes, code-quality, api-ratchet, warehouse-sources]name: Python code quality checksruns-on: ubuntu-24.04timeout-minutes: 5@@ -645,6 +801,13 @@ jobs:exit 1fi+ # Checked before the no-Python-changes exit below for the same reason: the job+ # has its own filter, and a vendor change with no Python file still runs it.+ if [[ "${{ needs.warehouse-sources.result }}" != "success" && "${{ needs.warehouse-sources.result }}" != "skipped" ]]; then+ echo "Warehouse sources checks and tests did not succeed (result: ${{ needs.warehouse-sources.result }})."+ exit 1+ fi+# Pass if no Python changes detected (jobs were skipped)if [[ "${{ needs.changes.outputs.python }}" != "true" ]]; thenecho "Python checks were skipped (no relevant changes)"diff --git a/posthog/test/repo_invariants/repo_invariant_scan_scope.py b/posthog/test/repo_invariants/repo_invariant_scan_scope.pynew file mode 100644index 000000000000..fd28ff7a4ca1--- /dev/null+++ b/posthog/test/repo_invariants/repo_invariant_scan_scope.py@@ -0,0 +1,54 @@+"""Narrow the file walk of a repo invariant to a few directories.++CI sets REPO_INVARIANTS_SCAN_SCOPE on a pull request that changes only warehouse source vendors,+to the vendor directories it changed. A vendor cannot import another vendor, so such a diff+cannot change what a walk finds outside those directories. Each walk then reads only those+directories and compares only the baseline lines inside them. Without the variable, every walk+reads all of its roots.+"""++import os+from collections.abc import Iterable+from pathlib import Path, PurePosixPath++SCAN_SCOPE_VARIABLE = "REPO_INVARIANTS_SCAN_SCOPE"+++def _inside(path: str, directory: str) -> bool:+ return PurePosixPath(path).is_relative_to(PurePosixPath(directory))+++class ScanScope:+ def __init__(self, directories: tuple[str, ...] | None) -> None:+ # None means the whole repo. An empty tuple means no directory at all.+ self.directories = directories++ @classmethod+ def from_environment(cls) -> "ScanScope":+ value = os.environ.get(SCAN_SCOPE_VARIABLE)+ return cls(None if value is None else tuple(value.split()))++ def roots(self, roots: Iterable[str]) -> list[str]:+ """The repo-relative directories to walk: the roots, or the scope directories inside them."""+ roots = list(roots)+ if self.directories is None:+ return roots+ return [directory for directory in self.directories if any(_inside(directory, root) for root in roots)]++ def covers(self, path: str) -> bool:+ """Whether a repo-relative path sits in the scope."""+ if self.directories is None:+ return True+ return any(_inside(path, directory) for directory in self.directories)++ def walk(self, repo_root: Path, roots: Iterable[Path], pattern: str) -> list[tuple[Path, Path]]:+ """Each file that matches the pattern in the scope, with the root it sits under."""+ files: list[tuple[Path, Path]] = []+ for root in roots:+ for directory in self.roots([root.relative_to(repo_root).as_posix()]):+ files.extend((root, path) for path in (repo_root / directory).rglob(pattern))+ return files++ def covers_root_relative(self, repo_root: Path, roots: Iterable[Path], path: str) -> bool:+ """Whether a path written relative to one of the roots sits in the scope."""+ return any(self.covers((root / path).relative_to(repo_root).as_posix()) for root in roots)diff --git a/posthog/test/repo_invariants/test_database_free_test_classes.py b/posthog/test/repo_invariants/test_database_free_test_classes.pyindex b12803e88a22..c980b7db5729 100644--- a/posthog/test/repo_invariants/test_database_free_test_classes.py+++ b/posthog/test/repo_invariants/test_database_free_test_classes.py@@ -14,6 +14,7 @@ from pathlib import Pathfrom parameterized import parameterizedfrom posthog.test.repo_invariants.ratchet import enforce_baseline+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScopeREPO_ROOT = Path(__file__).parents[3]BASELINE_PATH = Path(__file__).parent / "database_free_test_classes_baseline.txt"@@ -145,7 +146,9 @@ def collect_candidates() -> list[str]:# so limiting it to test paths knows a fraction of the names and reports those bases.inherited: set[str] = set()found: list[tuple[str, str]] = []- for root in SCANNED_ROOTS:+ # A scoped walk gathers the inherited names only inside the scope. That can only report more+ # classes, never fewer, because no code outside a vendor imports its test classes.+ for root in ScanScope.from_environment().roots(SCANNED_ROOTS):for path in (REPO_ROOT / root).rglob("*.py"):if SKIPPED_DIRS.intersection(path.parts):continue@@ -288,7 +291,8 @@ def write_baseline(candidates: list[str]) -> None:def test_database_free_test_classes_match_the_baseline() -> None:scanned = collect_candidates()- recorded = read_baseline()+ scope = ScanScope.from_environment()+ recorded = [line for line in read_baseline() if scope.covers(line.split("::")[0])]if scanned == recorded:returndiff --git a/posthog/test/repo_invariants/test_dataclass_defaults.py b/posthog/test/repo_invariants/test_dataclass_defaults.pyindex 64253f340ea6..610de1daa805 100644--- a/posthog/test/repo_invariants/test_dataclass_defaults.py+++ b/posthog/test/repo_invariants/test_dataclass_defaults.py@@ -12,6 +12,8 @@ Regenerate the baseline (only to ratchet DOWN or after moving code):import astfrom pathlib import Path+from repo_invariant_scan_scope import ScanScope+REPO_ROOT = Path(__file__).parents[3]BASELINE_PATH = Path(__file__).parent / "dataclass_frozen_baseline.txt"SCANNED_ROOTS = ("posthog", "ee", "products", "common")@@ -42,7 +44,7 @@ def _is_undeclared_dataclass_decorator(decorator: ast.expr) -> bool:def collect_offenders() -> dict[str, list[int]]:offenders: dict[str, list[int]] = {}- for root in SCANNED_ROOTS:+ for root in ScanScope.from_environment().roots(SCANNED_ROOTS):for path in (REPO_ROOT / root).rglob("*.py"):if SKIPPED_DIRS.intersection(path.parts):continuediff --git a/posthog/test/repo_invariants/test_dataclass_secret_fields.py b/posthog/test/repo_invariants/test_dataclass_secret_fields.pyindex d6d98ddb2ac5..0d2f9dae5a2f 100644--- a/posthog/test/repo_invariants/test_dataclass_secret_fields.py+++ b/posthog/test/repo_invariants/test_dataclass_secret_fields.py@@ -14,6 +14,8 @@ import astimport subprocessfrom pathlib import Path+from repo_invariant_scan_scope import ScanScope+REPO_ROOT = Path(__file__).parents[3]EXEMPTIONS_PATH = Path(__file__).parent / "dataclass_secret_field_exemptions.txt"SCANNED_ROOTS = ("posthog", "ee", "products", "dags", "common")@@ -114,7 +116,17 @@ def _is_test_file(path: Path) -> bool:def _candidate_paths() -> list[Path]:"""Tracked Python files with a dataclass decorator, found by git grep so most files are never read."""- pathspecs = [f"{root}/**/*.py" for root in SCANNED_ROOTS]+ scope = ScanScope.from_environment()+ if scope.directories is None:+ pathspecs = [f"{root}/**/*.py" for root in SCANNED_ROOTS]+ else:+ # A scope directory sits inside a root, so `<root>/**/*.py` takes every Python file in it.+ # `<directory>/**/*.py` would skip the files at its top. A `*` in a git pathspec also+ # matches `/`, so `<directory>/*.py` takes the same files that the full walk takes.+ pathspecs = [f"{directory}/*.py" for directory in scope.roots(SCANNED_ROOTS)]+ # git grep with no pathspec searches the whole repo, so an empty scope must stop here.+ if not pathspecs:+ return []result = subprocess.run(["git", "grep", "-l", "-E", "-e", "@([A-Za-z_]+\\.)?(dataclass|frozen)", "--", *pathspecs],cwd=REPO_ROOT,diff --git a/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py b/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.pyindex 8e4eba0b3f5b..b46b2d18dac9 100644--- a/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py+++ b/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py@@ -12,6 +12,7 @@ import warningsfrom pathlib import Pathfrom parameterized import parameterized+from repo_invariant_scan_scope import ScanScopeREPO_ROOT = Path(__file__).parents[3]# Vendor directories live in the product and in the `products/warehouse_sources/sources/` tree.@@ -35,7 +36,7 @@ OFF_THREAD_CALLS = {def _source_files(pattern: str) -> list[tuple[Path, Path]]:- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)def _own_nodes(function: ast.AST) -> list[ast.AST]:diff --git a/posthog/test/repo_invariants/test_warehouse_source_static_contract.py b/posthog/test/repo_invariants/test_warehouse_source_static_contract.pyindex 9ae655e92193..54a574ec7ff1 100644--- a/posthog/test/repo_invariants/test_warehouse_source_static_contract.py+++ b/posthog/test/repo_invariants/test_warehouse_source_static_contract.py@@ -24,6 +24,7 @@ from pathlib import Pathimport pytestfrom posthog.test.repo_invariants.ratchet import enforce_baseline+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScopeREPO_ROOT = Path(__file__).parents[3]BASELINE_DIR = Path(__file__).parent@@ -45,7 +46,7 @@ Check = Callable[[ast.Module], list[str]]def _source_files(pattern: str) -> list[tuple[Path, Path]]:- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)def _own_nodes(node: ast.AST) -> list[ast.AST]:@@ -318,7 +319,12 @@ def read_baseline(file_name: str) -> list[str]:def test_source_functions_match_the_baseline(check: str) -> None:collector, words, file_name, explanation = CHECKS[check]scanned = collect(collector, words)- recorded = read_baseline(file_name)+ scope = ScanScope.from_environment()+ recorded = [+ line+ for line in read_baseline(file_name)+ if scope.covers_root_relative(REPO_ROOT, SOURCE_ROOTS, line.split("::")[0])+ ]if scanned == recorded:returndiff --git a/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py b/posthog/test/repo_invariants/test_warehouse_source_test_shapes.pyindex f5d7dc4abd0d..6d71c5301e6c 100644--- a/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py+++ b/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py@@ -22,6 +22,7 @@ from pathlib import Pathimport pytestfrom posthog.test.repo_invariants.ratchet import enforce_baseline+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScopeREPO_ROOT = Path(__file__).parents[3]BASELINE_DIR = Path(__file__).parent@@ -48,7 +49,7 @@ Check = Callable[[ast.Module], bool]def _source_files(pattern: str) -> list[tuple[Path, Path]]:- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)def _terminal_name(node: ast.AST) -> str | None:@@ -246,7 +247,8 @@ def read_baseline(file_name: str) -> list[str]:def test_source_test_files_match_the_baseline(check: str) -> None:collector, words, file_name, explanation = CHECKS[check]scanned = collect(collector, words)- recorded = read_baseline(file_name)+ scope = ScanScope.from_environment()+ recorded = [line for line in read_baseline(file_name) if scope.covers(line)]if scanned == recorded:returndiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.pynew file mode 100644index 000000000000..cafc5b747706--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.py@@ -0,0 +1,62 @@+import pytest++from products.warehouse_sources.backend.temporal.data_imports.sources import (+ PLUGIN_SOURCES_PACKAGE,+ SourceRegistry,+ source_module_path,+)++# The parameters through which a test runs once per source. Their values are source keys.+PER_SOURCE_PARAMETERS = ("source_type", "name")+++def pytest_addoption(parser: pytest.Parser) -> None:+ parser.addoption(+ "--source-vendors",+ default=None,+ help=(+ "Comma-separated vendor directory names under products/warehouse_sources/sources/. The tests that run once per source "+ "then run only for these vendors. The tests of the whole catalog still run. CI sets it on a "+ "pull request that changes only these vendors."+ ),+ )+++def _plugin_vendor(module_path: str | None) -> str | None:+ """The vendor directory under the plugin sources of a source module, or None for a vendor in the product."""+ prefix = f"{PLUGIN_SOURCES_PACKAGE}."+ if module_path is None or not module_path.startswith(prefix):+ return None+ parts = module_path.removeprefix(prefix).split(".")+ if len(parts) == 2 and parts[1] == "source":+ return parts[0]+ return None+++def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:+ option = config.getoption("--source-vendors")+ if option is None:+ return+ vendors = {vendor for vendor in option.split(",") if vendor}+ vendor_by_key = {+ str(source_type): _plugin_vendor(source_module_path(source_type))+ for source_type in SourceRegistry.get_all_sources()+ }+ missing = sorted(vendors - set(vendor_by_key.values()))+ if missing:+ # A narrowed run that matches no source would run none of its tests and still pass.+ raise pytest.UsageError(f"--source-vendors names vendors that register no source: {', '.join(missing)}")++ kept: list[pytest.Item] = []+ deselected: list[pytest.Item] = []+ for item in items:+ callspec = getattr(item, "callspec", None)+ params = callspec.params if callspec is not None else {}+ keys = {str(params[name]) for name in PER_SOURCE_PARAMETERS if name in params} & vendor_by_key.keys()+ if keys and not any(vendor_by_key[key] in vendors for key in keys):+ deselected.append(item)+ else:+ kept.append(item)+ if deselected:+ config.hook.pytest_deselected(items=deselected)+ items[:] = keptdiff --git a/products/warehouse_sources/package.json b/products/warehouse_sources/package.jsonindex 2be5fe280f77..6372e8ca2935 100644--- a/products/warehouse_sources/package.json+++ b/products/warehouse_sources/package.json@@ -1,7 +1,7 @@{"name": "@posthog/products-warehouse-sources","scripts": {- "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ sources/ -m 'not async_migrations' --reruns=2 --reruns-delay=1 -v --tb=short",+ "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ -m 'not async_migrations' --reruns=2 --reruns-delay=1 -v --tb=short","backend:contract-check": "echo 'Contract files unchanged'"},"dependencies": {diff --git a/products/warehouse_sources/sources/AGENTS.md b/products/warehouse_sources/sources/AGENTS.mdindex 342e333bbc13..67da5c3d6978 100644--- a/products/warehouse_sources/sources/AGENTS.md+++ b/products/warehouse_sources/sources/AGENTS.md@@ -24,7 +24,7 @@ A new vendor still needs its `ExternalDataSourceType` enum member.Import shared code only from `products.warehouse_sources.sources.sdk`. Tests can also import from `products.warehouse_sources.sources.sdk.testing`.Import your own directory as `products.warehouse_sources.sources.<vendor>`. Do not import another vendor.-tach (`products.warehouse_sources.sources.*` in `tach.toml`) checks vendor code, and import-linter (`pyproject.toml`) also checks vendor tests.+tach (`products.warehouse_sources.sources.*` in `tach.toml`) checks vendor code, and import-linter (`pyproject.toml`) also checks vendor tests, including that a vendor does not import a sibling vendor.If you need a shared name that the SDK does not have, add it to `products/warehouse_sources/sources/sdk/__init__.py` (or `testing.py`) from its origin module.`products/warehouse_sources/sources/sdk/internals.py` holds names from product modules that tach does not expose. Do not add to it if a module that the product exposes has the name.A `mock.patch` target names the origin module, not `products.warehouse_sources.sources.sdk`: a patch on a re-export changes nothing.@@ -40,4 +40,8 @@ hogli test products/warehouse_sources/sources/<vendor>/````products/warehouse_sources/sources/conftest.py` isolates the job context of each test and caps `time.sleep`, so retry backoff does not run at real duration.-CI runs these tests in the `warehouse_sources` product job (`products/warehouse_sources/package.json`, `backend:test`).+CI runs these tests in the `Warehouse sources checks and tests` job of `.github/workflows/ci-python.yml`, not in Backend CI or the product shard (`backend:test` collects `backend/` only).+That job has no Postgres, ClickHouse, Redis or Kafka, so a vendor test must not need one.+A pull request that changes only vendor directories runs the tests of those vendors, the catalog tests of those vendors, and the repo invariants scoped to those directories.+A change to `products/warehouse_sources/sources/sdk/`, a file at the top of `products/warehouse_sources/sources/` (for example `conftest.py`), `products/warehouse_sources/backend/` or any other file runs every vendor.+`tools/warehouse_sources_vendor_selection.py` makes that choice.diff --git a/products/warehouse_sources/turbo.json b/products/warehouse_sources/turbo.jsonindex b131ee44765a..875d96621911 100644--- a/products/warehouse_sources/turbo.json+++ b/products/warehouse_sources/turbo.json@@ -1,9 +1,6 @@{"extends": ["//"],"tasks": {- "backend:test": {- "inputs": ["$TURBO_EXTENDS$", "sources/**/*.py"]- },"backend:contract-check": {"inputs": ["backend/facade/**",diff --git a/pyproject.toml b/pyproject.tomlindex 8a8307672246..d7791055d750 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -631,6 +631,19 @@ ignore_imports = ["products.warehouse_sources.sources.conftest -> products.**",]+# Vendors do not import each other, in code or in tests. tach covers vendor code only, and the+# contract above ignores imports inside the shared package. Independence expands the wildcard to+# one module per vendor, so a vendor that imports a sibling fails here. Imports of the SDK are+# allowed, and so are imports inside one vendor.+[[tool.importlinter.contracts]]+name = "sources vendors do not import each other"+type = "independence"+modules = ["products.warehouse_sources.sources.*"]+ignore_imports = [+ "products.warehouse_sources.sources.** -> products.warehouse_sources.sources.sdk",+ "products.warehouse_sources.sources.** -> products.warehouse_sources.sources.sdk.**",+]+# Presentation must only reach product internals through the facade.# Forbid all backend imports, then allowlist facade + self-imports.# This catches any new internal modules (cache.py, helpers.py, etc.)diff --git a/tools/hogli-commands/hogli_commands/product/checks.py b/tools/hogli-commands/hogli_commands/product/checks.pyindex d9b7224aad0a..3f6373de8b5c 100644--- a/tools/hogli-commands/hogli_commands/product/checks.py+++ b/tools/hogli-commands/hogli_commands/product/checks.py@@ -1285,7 +1285,9 @@ class OrphanedTestFilesCheck(ProductCheck):# .depot/workflows/ci-backend.yml "Run Temporal tests" step pytest paths:"batch_exports": ("backend/tests/temporal/",),"tasks": ("backend/temporal/",),- "warehouse_sources": ("backend/temporal/",),+ # backend/temporal/ is the Temporal segment above. sources/ holds the vendor plugins, which+ # the "Warehouse sources checks and tests" job in ci-python.yml runs.+ "warehouse_sources": ("backend/temporal/", "sources/"),"signals": ("backend/emission/",),# Covered by the "Run pr-approval-agent (stamphog) tests" step in ci-python.yml. The review# engine is a flat script bundle with bare sibling imports, so it runs as its own pytestdiff --git a/tools/test_warehouse_sources_vendor_selection.py b/tools/test_warehouse_sources_vendor_selection.pynew file mode 100644index 000000000000..0435502f8e41--- /dev/null+++ b/tools/test_warehouse_sources_vendor_selection.py@@ -0,0 +1,155 @@+from __future__ import annotations++import sys+import tempfile+import importlib.util+from pathlib import Path+from types import ModuleType++import unittest++from parameterized import parameterized++SCRIPT_PATH = Path(__file__).with_name("warehouse_sources_vendor_selection.py")+++def _load_module() -> ModuleType:+ spec = importlib.util.spec_from_file_location("warehouse_sources_vendor_selection", SCRIPT_PATH)+ assert spec is not None+ assert spec.loader is not None+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++selection = _load_module()+PLUGIN_DIR = selection.PLUGIN_DIR++SELECTION_CASES = [+ (+ "one_vendor",+ [+ "products/warehouse_sources/sources/demo_acme/source.py",+ "products/warehouse_sources/sources/demo_acme/tests/test_demo_acme.py",+ ],+ "vendors",+ ("demo_acme",),+ (),+ ),+ (+ "two_vendors",+ [+ "products/warehouse_sources/sources/stripe/source.py",+ "products/warehouse_sources/sources/demo_acme/source.py",+ ],+ "vendors",+ ("demo_acme", "stripe"),+ (),+ ),+ (+ "deleted_vendor_is_dropped",+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/gone/source.py"],+ "vendors",+ ("demo_acme",),+ ("gone",),+ ),+ (+ "sdk",+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/sdk/testing.py"],+ "full",+ (),+ (),+ ),+ (+ "plugin_root_conftest",+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/conftest.py"],+ "full",+ (),+ (),+ ),+ ("plugin_root_agents_file", ["products/warehouse_sources/sources/AGENTS.md"], "full", (), ()),+ (+ "product_backend",+ [+ "products/warehouse_sources/sources/demo_acme/source.py",+ "products/warehouse_sources/backend/temporal/data_imports/pipelines/x.py",+ ],+ "full",+ (),+ (),+ ),+ ("empty_diff", [], "full", (), ()),+ (+ "vendor_with_registry_files",+ [+ "products/warehouse_sources/sources/demo_acme/source.py",+ "products/warehouse_sources/backend/facade/credential_kinds.py",+ "products/warehouse_sources/backend/facade/credential_auth_types.py",+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",+ ],+ "vendors",+ ("demo_acme",),+ (),+ ),+ (+ "vendor_with_product_docs",+ [+ "products/warehouse_sources/sources/demo_acme/source.py",+ "products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md",+ "products/warehouse_sources/sources/AGENTS.md",+ ],+ "vendors",+ ("demo_acme",),+ (),+ ),+ (+ "registry_files_only",+ [+ "products/warehouse_sources/backend/facade/credential_kinds.py",+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",+ ],+ "full",+ (),+ (),+ ),+]+++class TestVendorSelection(unittest.TestCase):+ def setUp(self) -> None:+ self._tmp = tempfile.TemporaryDirectory()+ self.addCleanup(self._tmp.cleanup)+ self.root = Path(self._tmp.name)+ for vendor in ("sdk", "demo_acme", "stripe"):+ (self.root / PLUGIN_DIR / vendor).mkdir(parents=True)++ @parameterized.expand(SELECTION_CASES)+ def test_select(+ self,+ _name: str,+ changed_files: list[str],+ mode: str,+ vendors: tuple[str, ...],+ deleted_vendors: tuple[str, ...],+ ) -> None:+ result = selection.VendorSelector(self.root).select(changed_files)++ self.assertEqual(+ (result.mode, result.vendors, result.deleted_vendors),+ (mode, vendors, deleted_vendors),+ )++ def test_scan_scope_covers_deleted_vendors_too(self) -> None:+ result = selection.VendorSelector(self.root).select(+ ["products/warehouse_sources/sources/gone/source.py", "products/warehouse_sources/sources/stripe/source.py"]+ )++ self.assertEqual(+ result.github_output(),+ "mode=vendors\nvendors=stripe\nscan_scope=products/warehouse_sources/sources/gone products/warehouse_sources/sources/stripe",+ )+++if __name__ == "__main__":+ unittest.main()diff --git a/tools/warehouse_sources_vendor_selection.py b/tools/warehouse_sources_vendor_selection.pynew file mode 100644index 000000000000..27f6631b57ef--- /dev/null+++ b/tools/warehouse_sources_vendor_selection.py@@ -0,0 +1,125 @@+"""Select the warehouse source vendors a pull request can affect.++Each directory under `products/warehouse_sources/sources/` is one vendor, except `sdk`. A vendor+imports shared code only from the SDK, and tach, import-linter and a repo invariant enforce that,+so no vendor can change what another vendor does. A diff that stays inside vendor directories can+therefore only break those vendors. Any other changed file (the SDK, a file at the top of the+plugin directory, the product, a manifest, the workflow) can break every vendor, so it selects+the full run.++Three generated registry files are the exception: a new vendor adds one generated line to each,+so a diff that also touches a vendor directory stays in vendors mode. Backend CI still runs on+them, because they sit in the product backend. A diff that changes only these files has no+vendor to narrow to, so it selects the full run.++Markdown files in the product are skipped the same way. A doc cannot change a test result, and+vendor PRs often add their own section to a shared doc such as the coverage appendix.++Reads the changed files, one per line, on stdin and prints GitHub step outputs:++ git diff --name-only HEAD^1 HEAD | python3 tools/warehouse_sources_vendor_selection.py++`--full` skips the diff and selects the full run, for runs that have no pull request diff.+"""++from __future__ import annotations++import sys+import argparse+from collections.abc import Iterable+from dataclasses import dataclass+from pathlib import Path, PurePosixPath+from typing import Literal++REPO_ROOT = Path(__file__).parent.parent.resolve()+PRODUCT_DIR = "products/warehouse_sources"+PLUGIN_DIR = "products/warehouse_sources/sources"+SDK_DIR = "sdk"+VENDOR_REGISTRY_FILES = frozenset(+ {+ "products/warehouse_sources/backend/facade/credential_kinds.py",+ "products/warehouse_sources/backend/facade/credential_auth_types.py",+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",+ }+)+++@dataclass(frozen=True, kw_only=True)+class VendorSelection:+ mode: Literal["full", "vendors"]+ # Vendor directory names that exist in the checkout, so their tests can run.+ vendors: tuple[str, ...] = ()+ # Vendor directories the diff deleted. They have no tests left, but a baseline line+ # under them must still go, so the repo invariants scan them.+ deleted_vendors: tuple[str, ...] = ()++ def scan_scope(self) -> tuple[str, ...]:+ """The repo-relative directories a vendors-mode run scans with the repo invariants."""+ return tuple(f"{PLUGIN_DIR}/{name}" for name in sorted(self.vendors + self.deleted_vendors))++ def github_output(self) -> str:+ return "\n".join(+ [+ f"mode={self.mode}",+ f"vendors={' '.join(self.vendors)}",+ f"scan_scope={' '.join(self.scan_scope())}",+ ]+ )+++class VendorSelector:+ def __init__(self, repo_root: Path) -> None:+ self.plugin_root = repo_root / PLUGIN_DIR++ def _vendor_of(self, changed_file: str) -> str | None:+ """The vendor directory name a changed file sits in, or None when it sits in no vendor."""+ path = PurePosixPath(changed_file)+ if not path.is_relative_to(PLUGIN_DIR):+ return None+ parts = path.relative_to(PLUGIN_DIR).parts+ # A file directly under the plugin directory (conftest.py, __init__.py, AGENTS.md) has+ # one part. The SDK is shared code, not a vendor.+ if len(parts) < 2 or parts[0] == SDK_DIR:+ return None+ return parts[0]++ def _is_product_doc(self, changed_file: str) -> bool:+ path = PurePosixPath(changed_file)+ return path.suffix == ".md" and path.is_relative_to(PRODUCT_DIR)++ def select(self, changed_files: Iterable[str]) -> VendorSelection:+ changed_files = list(changed_files)+ # The job runs only when its paths filter matched a file, so an empty diff means the+ # diff missed that file. A narrowed run would then test nothing.+ if not changed_files:+ return VendorSelection(mode="full")+ vendors: set[str] = set()+ for changed_file in changed_files:+ if changed_file in VENDOR_REGISTRY_FILES or self._is_product_doc(changed_file):+ continue+ vendor = self._vendor_of(changed_file)+ if vendor is None:+ return VendorSelection(mode="full")+ vendors.add(vendor)+ if not vendors:+ return VendorSelection(mode="full")+ present = tuple(sorted(name for name in vendors if (self.plugin_root / name).is_dir()))+ deleted = tuple(sorted(vendors - set(present)))+ return VendorSelection(mode="vendors", vendors=present, deleted_vendors=deleted)+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--full", action="store_true", help="select the full run without reading a diff")+ args = parser.parse_args()++ if args.full:+ selection = VendorSelection(mode="full")+ else:+ changed_files = [line.strip() for line in sys.stdin if line.strip()]+ selection = VendorSelector(REPO_ROOT).select(changed_files)+ sys.stdout.write(selection.github_output() + "\n")+++if __name__ == "__main__":+ main()