Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(devex): run warehouse source vendors in their own ci lane (#115221)

Julian Bezc985c0dd7a7e
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 4d3aef94a369..2adf96552ed5 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -306,6 +306,10 @@ jobs:
# products/desktop is the imported desktop app; it has its own
# desktop-* CI and must not drag this suite onto its PRs
- '!products/desktop/**'
+ # Warehouse source vendors have their own job in ci-python.yml. They
+ # match 'products/**/*.py' above, so a vendor-only diff would start this
+ # suite too. The product code the vendors build on stays under backend/.
+ - '!products/warehouse_sources/sources/**'
# Each product's package.json declares the per-product
# backend:test script that turbo-discover reads to
# compose the Product matrix. Without this entry,
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.js
index 67887b8fbbb9..057a4c672c3c 100644
--- a/.github/scripts/trunk-impacted-targets.js
+++ b/.github/scripts/trunk-impacted-targets.js
@@ -642,13 +642,16 @@ const TRIPWIRE_RULES = [
// suite's tests run on a PR, so an under-selection can only mask conflicts
// that suite's lanes already serialize: the snob shadow, its verdict, and
// the testmon fanout list select Django tests, the dagster selector its
- // own suite, and the playwright pair the E2E specs.
+ // own suite, the warehouse sources selector the vendor tests, and the
+ // playwright pair the E2E specs.
['tools/snob_backend_test_selection_shadow.py', PYTHON],
['tools/test_snob_backend_test_selection_shadow.py', PYTHON],
['tools/test_selection_verdict.py', PYTHON],
['tools/test_test_selection_verdict.py', PYTHON],
['tools/dagster_test_selection.py', PYTHON],
['tools/test_dagster_test_selection.py', PYTHON],
+ ['tools/warehouse_sources_vendor_selection.py', PYTHON],
+ ['tools/test_warehouse_sources_vendor_selection.py', PYTHON],
['tools/testmon_high_fanout_files.txt', PYTHON],
['tools/playwright_spec_selection.py', FULLSTACK],
['tools/test_playwright_spec_selection.py', FULLSTACK],
diff --git a/.github/scripts/trunk-impacted-targets.test.js b/.github/scripts/trunk-impacted-targets.test.js
index 8db12f6ad096..add66d087bd4 100644
--- a/.github/scripts/trunk-impacted-targets.test.js
+++ b/.github/scripts/trunk-impacted-targets.test.js
@@ -2229,6 +2229,7 @@ test('test-selection scripts claim the lanes of the suite they select for', () =
'tools/snob_backend_test_selection_shadow.py',
'tools/test_selection_verdict.py',
'tools/dagster_test_selection.py',
+ 'tools/warehouse_sources_vendor_selection.py',
'tools/testmon_high_fanout_files.txt',
]) {
assert.deepEqual(computeTargets([file], CONTEXT), computeTargets(['mypy.ini'], CONTEXT), file)
diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml
index 289b4386987f..da5b5ca51424 100644
--- a/.github/workflows/ci-python.yml
+++ b/.github/workflows/ci-python.yml
@@ -70,6 +70,7 @@ jobs:
api_ratchet: ${{ steps.filter.outputs.api_ratchet || 'true' }}
mypy_cache_key: ${{ steps.filter.outputs.mypy_cache_key || 'false' }}
schema_python: ${{ steps.filter.outputs.schema_python || 'true' }}
+ warehouse_sources: ${{ steps.filter.outputs.warehouse_sources || 'true' }}
steps:
# For pull requests it's not necessary to checkout the code, but we
# also want this to run on master so we need to checkout
@@ -267,6 +268,21 @@ jobs:
- '.semgrep/rules/devex/prefer-codegen-api-namespaced.yaml'
- 'hogli.yaml'
- .github/workflows/ci-python.yml
+ # The warehouse source vendors, their SDK and the product code the
+ # SDK re-exports.
+ warehouse_sources:
+ - 'products/warehouse_sources/sources/**'
+ - 'products/warehouse_sources/backend/**'
+ - 'pyproject.toml'
+ - 'uv.lock'
+ - 'pytest.ini'
+ - 'conftest.py'
+ - 'tach.toml'
+ - .github/workflows/ci-python.yml
+ - .github/actions/setup-python-cached/**
+ - .github/actions/setup-uv/**
+ - tools/warehouse_sources_vendor_selection.py
+ - tools/test_warehouse_sources_vendor_selection.py
code-quality:
needs: changes
@@ -620,11 +636,151 @@ jobs:
fi
./bin/hogli lint:api-ratchet
+ # The warehouse source vendors under products/warehouse_sources/sources/ need no service:
+ # no Postgres, ClickHouse, Redis or Kafka. So a pull request that changes only vendors
+ # runs this job, and Backend CI does not start (its `backend` filter excludes that
+ # directory). code-quality already runs ruff, mypy and ty on the vendors.
+ warehouse-sources:
+ needs: changes
+ # Skipped on master push: the merge queue already ran this for every landed commit,
+ # and the hourly scheduled run tests every vendor. The scheduled run also catches a
+ # posthog/ change that breaks a vendor through the SDK, which no filter above sees.
+ if: needs.changes.outputs.warehouse_sources == 'true' && github.event_name != 'push'
+ name: Warehouse sources checks and tests
+ runs-on: depot-ubuntu-24.04
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+ with:
+ # The pull request merge commit and its first parent, the base, so the
+ # vendor selection below can diff the two without the GitHub API.
+ fetch-depth: 2
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: ./.github/actions/setup-python-cached
+ with:
+ python-version: 3.14.7
+
+ - name: Install uv
+ id: setup-uv
+ uses: ./.github/actions/setup-uv
+
+ - name: Install SAML dependencies
+ if: steps.setup-uv.outputs.cache-hit != 'true'
+ run: sudo rm -f /etc/apt/sources.list.d/*twingate* && sudo apt-get update && sudo apt-get install -y libxml2-dev libxmlsec1-dev libxmlsec1-openssl
+
+ - name: Install Python dependencies
+ run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
+
+ # A broken selector must fail here, before it narrows a run.
+ - name: Test the vendor selector
+ run: python tools/test_warehouse_sources_vendor_selection.py
+
+ # A pull request diff that stays inside vendor directories runs only those vendors.
+ # A vendor cannot import another vendor (tach and import-linter enforce that), so the
+ # other vendors cannot change. Any other file (the SDK, the conftest.py and other
+ # files at the top of the plugin directory, the product, a manifest, this workflow)
+ # selects the full run. The merge queue, the
+ # hourly run and a manual run have no pull request diff to narrow by, so they also
+ # take the full run.
+ - name: Select vendors
+ id: select
+ env:
+ EVENT_NAME: ${{ github.event_name }}
+ HEAD_REF: ${{ github.head_ref }}
+ run: |
+ if [ "$EVENT_NAME" != "pull_request" ] || [[ "$HEAD_REF" == trunk-merge/* ]]; then
+ python tools/warehouse_sources_vendor_selection.py --full
+ else
+ git diff --name-only HEAD^1 HEAD | python tools/warehouse_sources_vendor_selection.py
+ fi | tee -a "$GITHUB_OUTPUT"
+
+ - name: Check module boundaries (tach)
+ run: ./bin/hogli lint:tach
+
+ # tach leaves test files out of its dependency pass. The import-linter contracts
+ # for the vendors also check the vendor tests, and one of them fails a vendor that
+ # imports a sibling vendor.
+ - name: Check vendor imports (import-linter)
+ run: lint-imports
+
+ # The Django matrix runs every repo invariant, but it does not start for a vendor-only
+ # diff. So this job runs the ones that walk the vendor tree. The full run takes all
+ # of them. A vendors run takes the ones that check code a vendor writes, and scopes
+ # their walk to the changed vendor directories. The others look for calls into
+ # posthog/, products/ or DRF, which a vendor cannot import, so a vendor diff cannot
+ # break them. The lists are explicit: a new invariant that walks products/ has to be
+ # added here by hand.
+ - name: Run repo invariants that scan the vendors
+ env:
+ MODE: ${{ steps.select.outputs.mode }}
+ SCAN_SCOPE: ${{ steps.select.outputs.scan_scope }}
+ run: |
+ vendor_invariants=(
+ test_warehouse_source_static_contract.py
+ test_warehouse_source_test_shapes.py
+ test_resume_state_staged_before_yield.py
+ test_resume_state_saved_by_the_source_generator.py
+ test_dataclass_defaults.py
+ test_dataclass_secret_fields.py
+ test_database_free_test_classes.py
+ )
+ if [ "$MODE" = "vendors" ]; then
+ REPO_INVARIANTS_SCAN_SCOPE="$SCAN_SCOPE" pytest -p no:cacheprovider "${vendor_invariants[@]/#/posthog/test/repo_invariants/}"
+ exit 0
+ fi
+ full_invariants=(
+ "${vendor_invariants[@]}"
+ test_authentication_credential_types.py
+ test_feature_flag_gated_writes.py
+ test_feature_flag_raw_reads.py
+ test_local_kafka_consumers.py
+ test_rbac_shadow_resolver_callers.py
+ )
+ pytest -p no:cacheprovider -n auto "${full_invariants[@]/#/posthog/test/repo_invariants/}"
+
+ # These tests load the full vendor catalog, so a change to one vendor can fail them.
+ # A vendors run keeps the tests of the whole catalog and runs the tests that check one
+ # source at a time only for the selected vendors.
+ - name: Run vendor catalog tests
+ env:
+ MODE: ${{ steps.select.outputs.mode }}
+ VENDORS: ${{ steps.select.outputs.vendors }}
+ run: |
+ narrow=()
+ if [ "$MODE" = "vendors" ]; then
+ narrow=("--source-vendors=${VENDORS// /,}")
+ fi
+ pytest -p no:cacheprovider products/warehouse_sources/backend/temporal/data_imports/sources/tests --tb=short "${narrow[@]}"
+
+ # The committed credential kind and auth type maps hold one line per vendor. A vendors
+ # run skips Backend CI, so these suites run here in both modes, or a vendor change that
+ # makes the maps stale would merge. They need no database.
+ - name: Run credential map tests
+ run: |
+ credential_tests=products/warehouse_sources/backend/temporal/data_imports/sources/common/test
+ pytest -p no:cacheprovider "$credential_tests/test_credential_kind.py" "$credential_tests/test_credential_auth_type.py" --tb=short
+
+ - name: Run vendor tests
+ if: steps.select.outputs.mode == 'full' || steps.select.outputs.vendors != ''
+ env:
+ MODE: ${{ steps.select.outputs.mode }}
+ VENDORS: ${{ steps.select.outputs.vendors }}
+ run: |
+ plugin_dir=products/warehouse_sources/sources
+ targets=("$plugin_dir/")
+ if [ "$MODE" = "vendors" ]; then
+ read -ra vendors <<< "$VENDORS"
+ targets=("${vendors[@]/#/$plugin_dir/}")
+ fi
+ pytest -p no:cacheprovider "${targets[@]}" -m 'not async_migrations' --reruns=2 --reruns-delay=1 --tb=short
+
# Collate job - single required status check for branch protection.
# The code-quality job skips when no Python changes, but this job always
# runs and reports success, allowing PRs to merge.
python_tests:
- needs: [changes, code-quality, api-ratchet]
+ needs: [changes, code-quality, api-ratchet, warehouse-sources]
name: Python code quality checks
runs-on: ubuntu-24.04
timeout-minutes: 5
@@ -645,6 +801,13 @@ jobs:
exit 1
fi
+ # Checked before the no-Python-changes exit below for the same reason: the job
+ # has its own filter, and a vendor change with no Python file still runs it.
+ if [[ "${{ needs.warehouse-sources.result }}" != "success" && "${{ needs.warehouse-sources.result }}" != "skipped" ]]; then
+ echo "Warehouse sources checks and tests did not succeed (result: ${{ needs.warehouse-sources.result }})."
+ exit 1
+ fi
+
# Pass if no Python changes detected (jobs were skipped)
if [[ "${{ needs.changes.outputs.python }}" != "true" ]]; then
echo "Python checks were skipped (no relevant changes)"
diff --git a/posthog/test/repo_invariants/repo_invariant_scan_scope.py b/posthog/test/repo_invariants/repo_invariant_scan_scope.py
new file mode 100644
index 000000000000..fd28ff7a4ca1
--- /dev/null
+++ b/posthog/test/repo_invariants/repo_invariant_scan_scope.py
@@ -0,0 +1,54 @@
+"""Narrow the file walk of a repo invariant to a few directories.
+
+CI sets REPO_INVARIANTS_SCAN_SCOPE on a pull request that changes only warehouse source vendors,
+to the vendor directories it changed. A vendor cannot import another vendor, so such a diff
+cannot change what a walk finds outside those directories. Each walk then reads only those
+directories and compares only the baseline lines inside them. Without the variable, every walk
+reads all of its roots.
+"""
+
+import os
+from collections.abc import Iterable
+from pathlib import Path, PurePosixPath
+
+SCAN_SCOPE_VARIABLE = "REPO_INVARIANTS_SCAN_SCOPE"
+
+
+def _inside(path: str, directory: str) -> bool:
+ return PurePosixPath(path).is_relative_to(PurePosixPath(directory))
+
+
+class ScanScope:
+ def __init__(self, directories: tuple[str, ...] | None) -> None:
+ # None means the whole repo. An empty tuple means no directory at all.
+ self.directories = directories
+
+ @classmethod
+ def from_environment(cls) -> "ScanScope":
+ value = os.environ.get(SCAN_SCOPE_VARIABLE)
+ return cls(None if value is None else tuple(value.split()))
+
+ def roots(self, roots: Iterable[str]) -> list[str]:
+ """The repo-relative directories to walk: the roots, or the scope directories inside them."""
+ roots = list(roots)
+ if self.directories is None:
+ return roots
+ return [directory for directory in self.directories if any(_inside(directory, root) for root in roots)]
+
+ def covers(self, path: str) -> bool:
+ """Whether a repo-relative path sits in the scope."""
+ if self.directories is None:
+ return True
+ return any(_inside(path, directory) for directory in self.directories)
+
+ def walk(self, repo_root: Path, roots: Iterable[Path], pattern: str) -> list[tuple[Path, Path]]:
+ """Each file that matches the pattern in the scope, with the root it sits under."""
+ files: list[tuple[Path, Path]] = []
+ for root in roots:
+ for directory in self.roots([root.relative_to(repo_root).as_posix()]):
+ files.extend((root, path) for path in (repo_root / directory).rglob(pattern))
+ return files
+
+ def covers_root_relative(self, repo_root: Path, roots: Iterable[Path], path: str) -> bool:
+ """Whether a path written relative to one of the roots sits in the scope."""
+ return any(self.covers((root / path).relative_to(repo_root).as_posix()) for root in roots)
diff --git a/posthog/test/repo_invariants/test_database_free_test_classes.py b/posthog/test/repo_invariants/test_database_free_test_classes.py
index b12803e88a22..c980b7db5729 100644
--- a/posthog/test/repo_invariants/test_database_free_test_classes.py
+++ b/posthog/test/repo_invariants/test_database_free_test_classes.py
@@ -14,6 +14,7 @@ from pathlib import Path
from parameterized import parameterized
from posthog.test.repo_invariants.ratchet import enforce_baseline
+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScope
REPO_ROOT = Path(__file__).parents[3]
BASELINE_PATH = Path(__file__).parent / "database_free_test_classes_baseline.txt"
@@ -145,7 +146,9 @@ def collect_candidates() -> list[str]:
# so limiting it to test paths knows a fraction of the names and reports those bases.
inherited: set[str] = set()
found: list[tuple[str, str]] = []
- for root in SCANNED_ROOTS:
+ # A scoped walk gathers the inherited names only inside the scope. That can only report more
+ # classes, never fewer, because no code outside a vendor imports its test classes.
+ for root in ScanScope.from_environment().roots(SCANNED_ROOTS):
for path in (REPO_ROOT / root).rglob("*.py"):
if SKIPPED_DIRS.intersection(path.parts):
continue
@@ -288,7 +291,8 @@ def write_baseline(candidates: list[str]) -> None:
def test_database_free_test_classes_match_the_baseline() -> None:
scanned = collect_candidates()
- recorded = read_baseline()
+ scope = ScanScope.from_environment()
+ recorded = [line for line in read_baseline() if scope.covers(line.split("::")[0])]
if scanned == recorded:
return
diff --git a/posthog/test/repo_invariants/test_dataclass_defaults.py b/posthog/test/repo_invariants/test_dataclass_defaults.py
index 64253f340ea6..610de1daa805 100644
--- a/posthog/test/repo_invariants/test_dataclass_defaults.py
+++ b/posthog/test/repo_invariants/test_dataclass_defaults.py
@@ -12,6 +12,8 @@ Regenerate the baseline (only to ratchet DOWN or after moving code):
import ast
from pathlib import Path
+from repo_invariant_scan_scope import ScanScope
+
REPO_ROOT = Path(__file__).parents[3]
BASELINE_PATH = Path(__file__).parent / "dataclass_frozen_baseline.txt"
SCANNED_ROOTS = ("posthog", "ee", "products", "common")
@@ -42,7 +44,7 @@ def _is_undeclared_dataclass_decorator(decorator: ast.expr) -> bool:
def collect_offenders() -> dict[str, list[int]]:
offenders: dict[str, list[int]] = {}
- for root in SCANNED_ROOTS:
+ for root in ScanScope.from_environment().roots(SCANNED_ROOTS):
for path in (REPO_ROOT / root).rglob("*.py"):
if SKIPPED_DIRS.intersection(path.parts):
continue
diff --git a/posthog/test/repo_invariants/test_dataclass_secret_fields.py b/posthog/test/repo_invariants/test_dataclass_secret_fields.py
index d6d98ddb2ac5..0d2f9dae5a2f 100644
--- a/posthog/test/repo_invariants/test_dataclass_secret_fields.py
+++ b/posthog/test/repo_invariants/test_dataclass_secret_fields.py
@@ -14,6 +14,8 @@ import ast
import subprocess
from pathlib import Path
+from repo_invariant_scan_scope import ScanScope
+
REPO_ROOT = Path(__file__).parents[3]
EXEMPTIONS_PATH = Path(__file__).parent / "dataclass_secret_field_exemptions.txt"
SCANNED_ROOTS = ("posthog", "ee", "products", "dags", "common")
@@ -114,7 +116,17 @@ def _is_test_file(path: Path) -> bool:
def _candidate_paths() -> list[Path]:
"""Tracked Python files with a dataclass decorator, found by git grep so most files are never read."""
- pathspecs = [f"{root}/**/*.py" for root in SCANNED_ROOTS]
+ scope = ScanScope.from_environment()
+ if scope.directories is None:
+ pathspecs = [f"{root}/**/*.py" for root in SCANNED_ROOTS]
+ else:
+ # A scope directory sits inside a root, so `<root>/**/*.py` takes every Python file in it.
+ # `<directory>/**/*.py` would skip the files at its top. A `*` in a git pathspec also
+ # matches `/`, so `<directory>/*.py` takes the same files that the full walk takes.
+ pathspecs = [f"{directory}/*.py" for directory in scope.roots(SCANNED_ROOTS)]
+ # git grep with no pathspec searches the whole repo, so an empty scope must stop here.
+ if not pathspecs:
+ return []
result = subprocess.run(
["git", "grep", "-l", "-E", "-e", "@([A-Za-z_]+\\.)?(dataclass|frozen)", "--", *pathspecs],
cwd=REPO_ROOT,
diff --git a/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py b/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py
index 8e4eba0b3f5b..b46b2d18dac9 100644
--- a/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py
+++ b/posthog/test/repo_invariants/test_resume_state_saved_by_the_source_generator.py
@@ -12,6 +12,7 @@ import warnings
from pathlib import Path
from parameterized import parameterized
+from repo_invariant_scan_scope import ScanScope
REPO_ROOT = Path(__file__).parents[3]
# Vendor directories live in the product and in the `products/warehouse_sources/sources/` tree.
@@ -35,7 +36,7 @@ OFF_THREAD_CALLS = {
def _source_files(pattern: str) -> list[tuple[Path, Path]]:
- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]
+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)
def _own_nodes(function: ast.AST) -> list[ast.AST]:
diff --git a/posthog/test/repo_invariants/test_warehouse_source_static_contract.py b/posthog/test/repo_invariants/test_warehouse_source_static_contract.py
index 9ae655e92193..54a574ec7ff1 100644
--- a/posthog/test/repo_invariants/test_warehouse_source_static_contract.py
+++ b/posthog/test/repo_invariants/test_warehouse_source_static_contract.py
@@ -24,6 +24,7 @@ from pathlib import Path
import pytest
from posthog.test.repo_invariants.ratchet import enforce_baseline
+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScope
REPO_ROOT = Path(__file__).parents[3]
BASELINE_DIR = Path(__file__).parent
@@ -45,7 +46,7 @@ Check = Callable[[ast.Module], list[str]]
def _source_files(pattern: str) -> list[tuple[Path, Path]]:
- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]
+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)
def _own_nodes(node: ast.AST) -> list[ast.AST]:
@@ -318,7 +319,12 @@ def read_baseline(file_name: str) -> list[str]:
def test_source_functions_match_the_baseline(check: str) -> None:
collector, words, file_name, explanation = CHECKS[check]
scanned = collect(collector, words)
- recorded = read_baseline(file_name)
+ scope = ScanScope.from_environment()
+ recorded = [
+ line
+ for line in read_baseline(file_name)
+ if scope.covers_root_relative(REPO_ROOT, SOURCE_ROOTS, line.split("::")[0])
+ ]
if scanned == recorded:
return
diff --git a/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py b/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py
index f5d7dc4abd0d..6d71c5301e6c 100644
--- a/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py
+++ b/posthog/test/repo_invariants/test_warehouse_source_test_shapes.py
@@ -22,6 +22,7 @@ from pathlib import Path
import pytest
from posthog.test.repo_invariants.ratchet import enforce_baseline
+from posthog.test.repo_invariants.repo_invariant_scan_scope import ScanScope
REPO_ROOT = Path(__file__).parents[3]
BASELINE_DIR = Path(__file__).parent
@@ -48,7 +49,7 @@ Check = Callable[[ast.Module], bool]
def _source_files(pattern: str) -> list[tuple[Path, Path]]:
- return [(root, path) for root in SOURCE_ROOTS for path in root.rglob(pattern)]
+ return ScanScope.from_environment().walk(REPO_ROOT, SOURCE_ROOTS, pattern)
def _terminal_name(node: ast.AST) -> str | None:
@@ -246,7 +247,8 @@ def read_baseline(file_name: str) -> list[str]:
def test_source_test_files_match_the_baseline(check: str) -> None:
collector, words, file_name, explanation = CHECKS[check]
scanned = collect(collector, words)
- recorded = read_baseline(file_name)
+ scope = ScanScope.from_environment()
+ recorded = [line for line in read_baseline(file_name) if scope.covers(line)]
if scanned == recorded:
return
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.py
new file mode 100644
index 000000000000..cafc5b747706
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/conftest.py
@@ -0,0 +1,62 @@
+import pytest
+
+from products.warehouse_sources.backend.temporal.data_imports.sources import (
+ PLUGIN_SOURCES_PACKAGE,
+ SourceRegistry,
+ source_module_path,
+)
+
+# The parameters through which a test runs once per source. Their values are source keys.
+PER_SOURCE_PARAMETERS = ("source_type", "name")
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+ parser.addoption(
+ "--source-vendors",
+ default=None,
+ help=(
+ "Comma-separated vendor directory names under products/warehouse_sources/sources/. The tests that run once per source "
+ "then run only for these vendors. The tests of the whole catalog still run. CI sets it on a "
+ "pull request that changes only these vendors."
+ ),
+ )
+
+
+def _plugin_vendor(module_path: str | None) -> str | None:
+ """The vendor directory under the plugin sources of a source module, or None for a vendor in the product."""
+ prefix = f"{PLUGIN_SOURCES_PACKAGE}."
+ if module_path is None or not module_path.startswith(prefix):
+ return None
+ parts = module_path.removeprefix(prefix).split(".")
+ if len(parts) == 2 and parts[1] == "source":
+ return parts[0]
+ return None
+
+
+def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
+ option = config.getoption("--source-vendors")
+ if option is None:
+ return
+ vendors = {vendor for vendor in option.split(",") if vendor}
+ vendor_by_key = {
+ str(source_type): _plugin_vendor(source_module_path(source_type))
+ for source_type in SourceRegistry.get_all_sources()
+ }
+ missing = sorted(vendors - set(vendor_by_key.values()))
+ if missing:
+ # A narrowed run that matches no source would run none of its tests and still pass.
+ raise pytest.UsageError(f"--source-vendors names vendors that register no source: {', '.join(missing)}")
+
+ kept: list[pytest.Item] = []
+ deselected: list[pytest.Item] = []
+ for item in items:
+ callspec = getattr(item, "callspec", None)
+ params = callspec.params if callspec is not None else {}
+ keys = {str(params[name]) for name in PER_SOURCE_PARAMETERS if name in params} & vendor_by_key.keys()
+ if keys and not any(vendor_by_key[key] in vendors for key in keys):
+ deselected.append(item)
+ else:
+ kept.append(item)
+ if deselected:
+ config.hook.pytest_deselected(items=deselected)
+ items[:] = kept
diff --git a/products/warehouse_sources/package.json b/products/warehouse_sources/package.json
index 2be5fe280f77..6372e8ca2935 100644
--- a/products/warehouse_sources/package.json
+++ b/products/warehouse_sources/package.json
@@ -1,7 +1,7 @@
{
"name": "@posthog/products-warehouse-sources",
"scripts": {
- "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ sources/ -m 'not async_migrations' --reruns=2 --reruns-delay=1 -v --tb=short",
+ "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/ -m 'not async_migrations' --reruns=2 --reruns-delay=1 -v --tb=short",
"backend:contract-check": "echo 'Contract files unchanged'"
},
"dependencies": {
diff --git a/products/warehouse_sources/sources/AGENTS.md b/products/warehouse_sources/sources/AGENTS.md
index 342e333bbc13..67da5c3d6978 100644
--- a/products/warehouse_sources/sources/AGENTS.md
+++ b/products/warehouse_sources/sources/AGENTS.md
@@ -24,7 +24,7 @@ A new vendor still needs its `ExternalDataSourceType` enum member.
Import shared code only from `products.warehouse_sources.sources.sdk`. Tests can also import from `products.warehouse_sources.sources.sdk.testing`.
Import your own directory as `products.warehouse_sources.sources.<vendor>`. Do not import another vendor.
-tach (`products.warehouse_sources.sources.*` in `tach.toml`) checks vendor code, and import-linter (`pyproject.toml`) also checks vendor tests.
+tach (`products.warehouse_sources.sources.*` in `tach.toml`) checks vendor code, and import-linter (`pyproject.toml`) also checks vendor tests, including that a vendor does not import a sibling vendor.
If you need a shared name that the SDK does not have, add it to `products/warehouse_sources/sources/sdk/__init__.py` (or `testing.py`) from its origin module.
`products/warehouse_sources/sources/sdk/internals.py` holds names from product modules that tach does not expose. Do not add to it if a module that the product exposes has the name.
A `mock.patch` target names the origin module, not `products.warehouse_sources.sources.sdk`: a patch on a re-export changes nothing.
@@ -40,4 +40,8 @@ hogli test products/warehouse_sources/sources/<vendor>/
```
`products/warehouse_sources/sources/conftest.py` isolates the job context of each test and caps `time.sleep`, so retry backoff does not run at real duration.
-CI runs these tests in the `warehouse_sources` product job (`products/warehouse_sources/package.json`, `backend:test`).
+CI runs these tests in the `Warehouse sources checks and tests` job of `.github/workflows/ci-python.yml`, not in Backend CI or the product shard (`backend:test` collects `backend/` only).
+That job has no Postgres, ClickHouse, Redis or Kafka, so a vendor test must not need one.
+A pull request that changes only vendor directories runs the tests of those vendors, the catalog tests of those vendors, and the repo invariants scoped to those directories.
+A change to `products/warehouse_sources/sources/sdk/`, a file at the top of `products/warehouse_sources/sources/` (for example `conftest.py`), `products/warehouse_sources/backend/` or any other file runs every vendor.
+`tools/warehouse_sources_vendor_selection.py` makes that choice.
diff --git a/products/warehouse_sources/turbo.json b/products/warehouse_sources/turbo.json
index b131ee44765a..875d96621911 100644
--- a/products/warehouse_sources/turbo.json
+++ b/products/warehouse_sources/turbo.json
@@ -1,9 +1,6 @@
{
"extends": ["//"],
"tasks": {
- "backend:test": {
- "inputs": ["$TURBO_EXTENDS$", "sources/**/*.py"]
- },
"backend:contract-check": {
"inputs": [
"backend/facade/**",
diff --git a/pyproject.toml b/pyproject.toml
index 8a8307672246..d7791055d750 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -631,6 +631,19 @@ ignore_imports = [
"products.warehouse_sources.sources.conftest -> products.**",
]
+# Vendors do not import each other, in code or in tests. tach covers vendor code only, and the
+# contract above ignores imports inside the shared package. Independence expands the wildcard to
+# one module per vendor, so a vendor that imports a sibling fails here. Imports of the SDK are
+# allowed, and so are imports inside one vendor.
+[[tool.importlinter.contracts]]
+name = "sources vendors do not import each other"
+type = "independence"
+modules = ["products.warehouse_sources.sources.*"]
+ignore_imports = [
+ "products.warehouse_sources.sources.** -> products.warehouse_sources.sources.sdk",
+ "products.warehouse_sources.sources.** -> products.warehouse_sources.sources.sdk.**",
+]
+
# Presentation must only reach product internals through the facade.
# Forbid all backend imports, then allowlist facade + self-imports.
# This catches any new internal modules (cache.py, helpers.py, etc.)
diff --git a/tools/hogli-commands/hogli_commands/product/checks.py b/tools/hogli-commands/hogli_commands/product/checks.py
index d9b7224aad0a..3f6373de8b5c 100644
--- a/tools/hogli-commands/hogli_commands/product/checks.py
+++ b/tools/hogli-commands/hogli_commands/product/checks.py
@@ -1285,7 +1285,9 @@ class OrphanedTestFilesCheck(ProductCheck):
# .depot/workflows/ci-backend.yml "Run Temporal tests" step pytest paths:
"batch_exports": ("backend/tests/temporal/",),
"tasks": ("backend/temporal/",),
- "warehouse_sources": ("backend/temporal/",),
+ # backend/temporal/ is the Temporal segment above. sources/ holds the vendor plugins, which
+ # the "Warehouse sources checks and tests" job in ci-python.yml runs.
+ "warehouse_sources": ("backend/temporal/", "sources/"),
"signals": ("backend/emission/",),
# Covered by the "Run pr-approval-agent (stamphog) tests" step in ci-python.yml. The review
# engine is a flat script bundle with bare sibling imports, so it runs as its own pytest
diff --git a/tools/test_warehouse_sources_vendor_selection.py b/tools/test_warehouse_sources_vendor_selection.py
new file mode 100644
index 000000000000..0435502f8e41
--- /dev/null
+++ b/tools/test_warehouse_sources_vendor_selection.py
@@ -0,0 +1,155 @@
+from __future__ import annotations
+
+import sys
+import tempfile
+import importlib.util
+from pathlib import Path
+from types import ModuleType
+
+import unittest
+
+from parameterized import parameterized
+
+SCRIPT_PATH = Path(__file__).with_name("warehouse_sources_vendor_selection.py")
+
+
+def _load_module() -> ModuleType:
+ spec = importlib.util.spec_from_file_location("warehouse_sources_vendor_selection", SCRIPT_PATH)
+ assert spec is not None
+ assert spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[spec.name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+selection = _load_module()
+PLUGIN_DIR = selection.PLUGIN_DIR
+
+SELECTION_CASES = [
+ (
+ "one_vendor",
+ [
+ "products/warehouse_sources/sources/demo_acme/source.py",
+ "products/warehouse_sources/sources/demo_acme/tests/test_demo_acme.py",
+ ],
+ "vendors",
+ ("demo_acme",),
+ (),
+ ),
+ (
+ "two_vendors",
+ [
+ "products/warehouse_sources/sources/stripe/source.py",
+ "products/warehouse_sources/sources/demo_acme/source.py",
+ ],
+ "vendors",
+ ("demo_acme", "stripe"),
+ (),
+ ),
+ (
+ "deleted_vendor_is_dropped",
+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/gone/source.py"],
+ "vendors",
+ ("demo_acme",),
+ ("gone",),
+ ),
+ (
+ "sdk",
+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/sdk/testing.py"],
+ "full",
+ (),
+ (),
+ ),
+ (
+ "plugin_root_conftest",
+ ["products/warehouse_sources/sources/demo_acme/source.py", "products/warehouse_sources/sources/conftest.py"],
+ "full",
+ (),
+ (),
+ ),
+ ("plugin_root_agents_file", ["products/warehouse_sources/sources/AGENTS.md"], "full", (), ()),
+ (
+ "product_backend",
+ [
+ "products/warehouse_sources/sources/demo_acme/source.py",
+ "products/warehouse_sources/backend/temporal/data_imports/pipelines/x.py",
+ ],
+ "full",
+ (),
+ (),
+ ),
+ ("empty_diff", [], "full", (), ()),
+ (
+ "vendor_with_registry_files",
+ [
+ "products/warehouse_sources/sources/demo_acme/source.py",
+ "products/warehouse_sources/backend/facade/credential_kinds.py",
+ "products/warehouse_sources/backend/facade/credential_auth_types.py",
+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",
+ ],
+ "vendors",
+ ("demo_acme",),
+ (),
+ ),
+ (
+ "vendor_with_product_docs",
+ [
+ "products/warehouse_sources/sources/demo_acme/source.py",
+ "products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md",
+ "products/warehouse_sources/sources/AGENTS.md",
+ ],
+ "vendors",
+ ("demo_acme",),
+ (),
+ ),
+ (
+ "registry_files_only",
+ [
+ "products/warehouse_sources/backend/facade/credential_kinds.py",
+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",
+ ],
+ "full",
+ (),
+ (),
+ ),
+]
+
+
+class TestVendorSelection(unittest.TestCase):
+ def setUp(self) -> None:
+ self._tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self._tmp.cleanup)
+ self.root = Path(self._tmp.name)
+ for vendor in ("sdk", "demo_acme", "stripe"):
+ (self.root / PLUGIN_DIR / vendor).mkdir(parents=True)
+
+ @parameterized.expand(SELECTION_CASES)
+ def test_select(
+ self,
+ _name: str,
+ changed_files: list[str],
+ mode: str,
+ vendors: tuple[str, ...],
+ deleted_vendors: tuple[str, ...],
+ ) -> None:
+ result = selection.VendorSelector(self.root).select(changed_files)
+
+ self.assertEqual(
+ (result.mode, result.vendors, result.deleted_vendors),
+ (mode, vendors, deleted_vendors),
+ )
+
+ def test_scan_scope_covers_deleted_vendors_too(self) -> None:
+ result = selection.VendorSelector(self.root).select(
+ ["products/warehouse_sources/sources/gone/source.py", "products/warehouse_sources/sources/stripe/source.py"]
+ )
+
+ self.assertEqual(
+ result.github_output(),
+ "mode=vendors\nvendors=stripe\nscan_scope=products/warehouse_sources/sources/gone products/warehouse_sources/sources/stripe",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/warehouse_sources_vendor_selection.py b/tools/warehouse_sources_vendor_selection.py
new file mode 100644
index 000000000000..27f6631b57ef
--- /dev/null
+++ b/tools/warehouse_sources_vendor_selection.py
@@ -0,0 +1,125 @@
+"""Select the warehouse source vendors a pull request can affect.
+
+Each directory under `products/warehouse_sources/sources/` is one vendor, except `sdk`. A vendor
+imports shared code only from the SDK, and tach, import-linter and a repo invariant enforce that,
+so no vendor can change what another vendor does. A diff that stays inside vendor directories can
+therefore only break those vendors. Any other changed file (the SDK, a file at the top of the
+plugin directory, the product, a manifest, the workflow) can break every vendor, so it selects
+the full run.
+
+Three generated registry files are the exception: a new vendor adds one generated line to each,
+so a diff that also touches a vendor directory stays in vendors mode. Backend CI still runs on
+them, because they sit in the product backend. A diff that changes only these files has no
+vendor to narrow to, so it selects the full run.
+
+Markdown files in the product are skipped the same way. A doc cannot change a test result, and
+vendor PRs often add their own section to a shared doc such as the coverage appendix.
+
+Reads the changed files, one per line, on stdin and prints GitHub step outputs:
+
+ git diff --name-only HEAD^1 HEAD | python3 tools/warehouse_sources_vendor_selection.py
+
+`--full` skips the diff and selects the full run, for runs that have no pull request diff.
+"""
+
+from __future__ import annotations
+
+import sys
+import argparse
+from collections.abc import Iterable
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath
+from typing import Literal
+
+REPO_ROOT = Path(__file__).parent.parent.resolve()
+PRODUCT_DIR = "products/warehouse_sources"
+PLUGIN_DIR = "products/warehouse_sources/sources"
+SDK_DIR = "sdk"
+VENDOR_REGISTRY_FILES = frozenset(
+ {
+ "products/warehouse_sources/backend/facade/credential_kinds.py",
+ "products/warehouse_sources/backend/facade/credential_auth_types.py",
+ "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",
+ }
+)
+
+
+@dataclass(frozen=True, kw_only=True)
+class VendorSelection:
+ mode: Literal["full", "vendors"]
+ # Vendor directory names that exist in the checkout, so their tests can run.
+ vendors: tuple[str, ...] = ()
+ # Vendor directories the diff deleted. They have no tests left, but a baseline line
+ # under them must still go, so the repo invariants scan them.
+ deleted_vendors: tuple[str, ...] = ()
+
+ def scan_scope(self) -> tuple[str, ...]:
+ """The repo-relative directories a vendors-mode run scans with the repo invariants."""
+ return tuple(f"{PLUGIN_DIR}/{name}" for name in sorted(self.vendors + self.deleted_vendors))
+
+ def github_output(self) -> str:
+ return "\n".join(
+ [
+ f"mode={self.mode}",
+ f"vendors={' '.join(self.vendors)}",
+ f"scan_scope={' '.join(self.scan_scope())}",
+ ]
+ )
+
+
+class VendorSelector:
+ def __init__(self, repo_root: Path) -> None:
+ self.plugin_root = repo_root / PLUGIN_DIR
+
+ def _vendor_of(self, changed_file: str) -> str | None:
+ """The vendor directory name a changed file sits in, or None when it sits in no vendor."""
+ path = PurePosixPath(changed_file)
+ if not path.is_relative_to(PLUGIN_DIR):
+ return None
+ parts = path.relative_to(PLUGIN_DIR).parts
+ # A file directly under the plugin directory (conftest.py, __init__.py, AGENTS.md) has
+ # one part. The SDK is shared code, not a vendor.
+ if len(parts) < 2 or parts[0] == SDK_DIR:
+ return None
+ return parts[0]
+
+ def _is_product_doc(self, changed_file: str) -> bool:
+ path = PurePosixPath(changed_file)
+ return path.suffix == ".md" and path.is_relative_to(PRODUCT_DIR)
+
+ def select(self, changed_files: Iterable[str]) -> VendorSelection:
+ changed_files = list(changed_files)
+ # The job runs only when its paths filter matched a file, so an empty diff means the
+ # diff missed that file. A narrowed run would then test nothing.
+ if not changed_files:
+ return VendorSelection(mode="full")
+ vendors: set[str] = set()
+ for changed_file in changed_files:
+ if changed_file in VENDOR_REGISTRY_FILES or self._is_product_doc(changed_file):
+ continue
+ vendor = self._vendor_of(changed_file)
+ if vendor is None:
+ return VendorSelection(mode="full")
+ vendors.add(vendor)
+ if not vendors:
+ return VendorSelection(mode="full")
+ present = tuple(sorted(name for name in vendors if (self.plugin_root / name).is_dir()))
+ deleted = tuple(sorted(vendors - set(present)))
+ return VendorSelection(mode="vendors", vendors=present, deleted_vendors=deleted)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ parser.add_argument("--full", action="store_true", help="select the full run without reading a diff")
+ args = parser.parse_args()
+
+ if args.full:
+ selection = VendorSelection(mode="full")
+ else:
+ changed_files = [line.strip() for line in sys.stdin if line.strip()]
+ selection = VendorSelector(REPO_ROOT).select(changed_files)
+ sys.stdout.write(selection.github_output() + "\n")
+
+
+if __name__ == "__main__":
+ main()