Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(ci): run repo checks when signals scout skills change (#121755)

Phil Haackfe12ef733a82
diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml
index 082200777bb8..fc2ef7df9dd0 100644
--- a/.depot/workflows/ci-backend.yml
+++ b/.depot/workflows/ci-backend.yml
@@ -560,6 +560,16 @@ jobs:
- '.github/workflows/ci-python.yml'
- 'products/ruff.toml'
- '.github/scripts/**'
+ # test_signals_scout_tool_citations.py checks the MCP tool names that the
+ # canonical scout skills cite against the committed tool catalog. The backend
+ # filter matches only the .py files in a skill and nothing under services/mcp/.
+ # The same test fails when a file it reads has no match in this filter.
+ - 'products/signals/skills/**'
+ - 'services/mcp/schema/tool-definitions.json'
+ - 'services/mcp/schema/generated-tool-definitions.json'
+ # _EXTERNAL_COMPANION_SKILL_DIRS in products/signals/backend/scout_harness/lazy_seed.py
+ # names this companion skill from another product.
+ - 'products/replay_vision/skills/exploring-replay-vision-observations/**'
sdk_manifests:
# Everything the SDK major-version guard reads. Lockfiles are listed
# because a floor-only or `*` spec can move to a new major through
diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml
index dece23f00520..30cb7c456a07 100644
--- a/.github/workflows/ci-backend.yml
+++ b/.github/workflows/ci-backend.yml
@@ -424,6 +424,16 @@ jobs:
- '.github/workflows/ci-python.yml'
- 'products/ruff.toml'
- '.github/scripts/**'
+ # test_signals_scout_tool_citations.py checks the MCP tool names that the
+ # canonical scout skills cite against the committed tool catalog. The backend
+ # filter matches only the .py files in a skill and nothing under services/mcp/.
+ # The same test fails when a file it reads has no match in this filter.
+ - 'products/signals/skills/**'
+ - 'services/mcp/schema/tool-definitions.json'
+ - 'services/mcp/schema/generated-tool-definitions.json'
+ # _EXTERNAL_COMPANION_SKILL_DIRS in products/signals/backend/scout_harness/lazy_seed.py
+ # names this companion skill from another product.
+ - 'products/replay_vision/skills/exploring-replay-vision-observations/**'
sdk_manifests:
# Everything the SDK major-version guard reads. Lockfiles are listed
# because a floor-only or `*` spec can move to a new major through
diff --git a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py b/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
index 7887cd822ec1..cf6ec2c7e357 100644
--- a/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
+++ b/posthog/test/repo_invariants/test_signals_scout_tool_citations.py
@@ -30,11 +30,13 @@ Regenerate the baseline (after confirming a rename, or when a body cites a new t
import re
from datetime import date
-from pathlib import Path
+from pathlib import Path, PurePosixPath
import pytest
-from posthog.mcp_tool_definitions import get_mcp_tool_definitions
+import yaml
+
+from posthog.mcp_tool_definitions import _TOOL_DEFINITION_PATHS, get_mcp_tool_definitions
from posthog.test.repo_invariants.ratchet import enforce_baseline
REPO_ROOT = Path(__file__).parents[3]
@@ -176,6 +178,36 @@ def test_baseline_tool_citations_still_resolve() -> None:
)
+def repo_invariants_filter(workflow: str) -> list[str]:
+ parsed = yaml.safe_load((REPO_ROOT / workflow).read_text())
+ [step] = [step for step in parsed["jobs"]["changes"]["steps"] if step.get("id") == "filter"]
+ return yaml.safe_load(step["with"]["filters"])["repo_invariants"]
+
+
+def citation_inputs() -> list[PurePosixPath]:
+ from products.signals.backend.scout_harness.lazy_seed import discover_canonical_skills
+
+ paths = list(_TOOL_DEFINITION_PATHS)
+ for skill in discover_canonical_skills():
+ paths.append(skill.source_path / "SKILL.md")
+ paths.extend(skill.source_path / file.path for file in skill.files)
+ root = REPO_ROOT.resolve()
+ return [PurePosixPath(path.resolve().relative_to(root).as_posix()) for path in paths]
+
+
[email protected]("workflow", [".github/workflows/ci-backend.yml", ".depot/workflows/ci-backend.yml"])
+def test_repo_checks_runs_when_a_citation_input_changes(workflow: str) -> None:
+ globs = repo_invariants_filter(workflow)
+ # full_match keeps `*` inside one path segment, as picomatch in the paths-filter action does.
+ # fnmatch lets `*` cross `/`, so it would pass a glob that the action never fires on.
+ missing = sorted(str(path) for path in citation_inputs() if not any(path.full_match(glob) for glob in globs))
+ assert missing == [], (
+ "Repo checks skips a pull request that changes only these inputs of the scout citation "
+ f"check. Add a glob that covers each one to the repo_invariants paths filter in {workflow}:\n"
+ + "\n".join(f" {path}" for path in missing)
+ )
+
+
if __name__ == "__main__":
import django