Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.3 KB
Raw Blame
"""Select the warehouse source vendors a pull request can affect.

Each directory under `products/warehouse_sources/sources/` is one vendor, except `sdk`. A vendor
imports shared code only from the SDK, and tach, import-linter and a repo invariant enforce that,
so no vendor can change what another vendor does. A diff that stays inside vendor directories can
therefore only break those vendors. Any other changed file (the SDK, a file at the top of the
plugin directory, the product, a manifest, the workflow) can break every vendor, so it selects
the full run.

Three generated registry files are the exception: a new vendor adds one generated line to each,
so a diff that also touches a vendor directory stays in vendors mode. Backend CI still runs on
them, because they sit in the product backend. A diff that changes only these files has no
vendor to narrow to, so it selects the full run.

Markdown files in the product are skipped the same way. A doc cannot change a test result, and
vendor PRs often add their own section to a shared doc such as the coverage appendix.

Reads the changed files, one per line, on stdin and prints GitHub step outputs:

    git diff --name-only HEAD^1 HEAD | python3 tools/warehouse_sources_vendor_selection.py

`--full` skips the diff and selects the full run, for runs that have no pull request diff.
"""

from __future__ import annotations

import sys
import argparse
from collections.abc import Iterable
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Literal

REPO_ROOT = Path(__file__).parent.parent.resolve()
PRODUCT_DIR = "products/warehouse_sources"
PLUGIN_DIR = "products/warehouse_sources/sources"
SDK_DIR = "sdk"
VENDOR_REGISTRY_FILES = frozenset(
    {
        "products/warehouse_sources/backend/facade/credential_kinds.py",
        "products/warehouse_sources/backend/facade/credential_auth_types.py",
        "products/warehouse_sources/backend/temporal/data_imports/sources/field_classification.csv",
    }
)


@dataclass(frozen=True, kw_only=True)
class VendorSelection:
    mode: Literal["full", "vendors"]
    # Vendor directory names that exist in the checkout, so their tests can run.
    vendors: tuple[str, ...] = ()
    # Vendor directories the diff deleted. They have no tests left, but a baseline line
    # under them must still go, so the repo invariants scan them.
    deleted_vendors: tuple[str, ...] = ()

    def scan_scope(self) -> tuple[str, ...]:
        """The repo-relative directories a vendors-mode run scans with the repo invariants."""
        return tuple(f"{PLUGIN_DIR}/{name}" for name in sorted(self.vendors + self.deleted_vendors))

    def github_output(self) -> str:
        return "\n".join(
            [
                f"mode={self.mode}",
                f"vendors={' '.join(self.vendors)}",
                f"scan_scope={' '.join(self.scan_scope())}",
            ]
        )


class VendorSelector:
    def __init__(self, repo_root: Path) -> None:
        self.plugin_root = repo_root / PLUGIN_DIR

    def _vendor_of(self, changed_file: str) -> str | None:
        """The vendor directory name a changed file sits in, or None when it sits in no vendor."""
        path = PurePosixPath(changed_file)
        if not path.is_relative_to(PLUGIN_DIR):
            return None
        parts = path.relative_to(PLUGIN_DIR).parts
        # A file directly under the plugin directory (conftest.py, __init__.py, AGENTS.md) has
        # one part. The SDK is shared code, not a vendor.
        if len(parts) < 2 or parts[0] == SDK_DIR:
            return None
        return parts[0]

    def _is_product_doc(self, changed_file: str) -> bool:
        path = PurePosixPath(changed_file)
        return path.suffix == ".md" and path.is_relative_to(PRODUCT_DIR)

    def select(self, changed_files: Iterable[str]) -> VendorSelection:
        changed_files = list(changed_files)
        # The job runs only when its paths filter matched a file, so an empty diff means the
        # diff missed that file. A narrowed run would then test nothing.
        if not changed_files:
            return VendorSelection(mode="full")
        vendors: set[str] = set()
        for changed_file in changed_files:
            if changed_file in VENDOR_REGISTRY_FILES or self._is_product_doc(changed_file):
                continue
            vendor = self._vendor_of(changed_file)
            if vendor is None:
                return VendorSelection(mode="full")
            vendors.add(vendor)
        if not vendors:
            return VendorSelection(mode="full")
        present = tuple(sorted(name for name in vendors if (self.plugin_root / name).is_dir()))
        deleted = tuple(sorted(vendors - set(present)))
        return VendorSelection(mode="vendors", vendors=present, deleted_vendors=deleted)


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("--full", action="store_true", help="select the full run without reading a diff")
    args = parser.parse_args()

    if args.full:
        selection = VendorSelection(mode="full")
    else:
        changed_files = [line.strip() for line in sys.stdin if line.strip()]
        selection = VendorSelector(REPO_ROOT).select(changed_files)
    sys.stdout.write(selection.github_output() + "\n")


if __name__ == "__main__":
    main()