:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from __future__ import annotations
import os
import sys
import tempfile
import importlib.util
from pathlib import Path
from types import ModuleType
import unittest
from unittest import mock
from parameterized import parameterized
SCRIPT_PATH = Path(__file__).with_name("snob_backend_test_selection_shadow.py")
def _load_selection_module() -> ModuleType:
spec = importlib.util.spec_from_file_location("snob_backend_test_selection_shadow", SCRIPT_PATH)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
class TestSnobBackendTestSelectionShadow(unittest.TestCase):
def test_classifies_django_api_client_tests(self) -> None:
with tempfile.TemporaryDirectory() as root:
tmp_path = Path(root)
selection = _load_selection_module()
selection.REPO_ROOT = tmp_path
test_path = tmp_path / "products" / "feature_flags" / "backend" / "test" / "test_api.py"
test_path.parent.mkdir(parents=True)
test_path.write_text(
"\n".join(
[
"from rest_framework.test import APIClient",
"",
"def test_feature_flags_endpoint():",
" client = APIClient()",
" client.get('/api/projects/1/feature_flags/')",
]
)
)
features = selection.classify_test_file("products/feature_flags/backend/test/test_api.py")
self.assertTrue(features.imports_api_client)
self.assertTrue(features.calls_http_client)
self.assertTrue(features.uses_api_url)
self.assertTrue(features.is_django_api_test)
self.assertIn("feature_flags", features.api_tokens)
def test_ast_selection_groups_product_api_client_tests(self) -> None:
with tempfile.TemporaryDirectory() as root:
tmp_path = Path(root)
selection = _load_selection_module()
selection.REPO_ROOT = tmp_path
test_path = tmp_path / "products" / "feature_flags" / "backend" / "test" / "test_api.py"
test_path.parent.mkdir(parents=True)
test_path.write_text(
"\n".join(
[
"def test_feature_flags_endpoint(api_client):",
" api_client.get('/api/projects/1/feature_flags/')",
]
)
)
features_by_path = selection.classify_tests()
result = selection.ast_select_tests(
["products/feature_flags/backend/api/feature_flags.py"],
features_by_path,
)
self.assertIn("product_api_client:feature_flags", result.groups)
self.assertIn("product_api_route_tokens:feature_flags", result.groups)
self.assertIn("same_app:products/feature_flags/backend", result.groups)
self.assertEqual(["products/feature_flags/backend/test/test_api.py"], result.tests)
def test_ast_selection_ignores_deleted_test_files(self) -> None:
with tempfile.TemporaryDirectory() as root:
selection = _load_selection_module()
selection.REPO_ROOT = Path(root)
result = selection.ast_select_tests(["posthog/test/test_deleted.py"], {})
self.assertEqual({}, result.groups)
self.assertEqual([], result.tests)
def test_ast_selection_matches_posthog_api_test_by_filename(self) -> None:
with tempfile.TemporaryDirectory() as root:
tmp_path = Path(root)
selection = _load_selection_module()
selection.REPO_ROOT = tmp_path
neighbor = tmp_path / "posthog" / "api" / "test" / "test_project.py"
neighbor.parent.mkdir(parents=True)
neighbor.touch()
result = selection.ast_select_tests(
["posthog/api/project.py"],
{
"posthog/api/test/test_project.py": selection.TestFeatures(
path="posthog/api/test/test_project.py",
imports_api_client=True,
api_tokens=("project",),
),
"posthog/api/test/test_user.py": selection.TestFeatures(
path="posthog/api/test/test_user.py",
imports_api_client=True,
api_tokens=("user",),
),
},
)
self.assertIn("conventional_neighbors", result.groups)
self.assertIn("posthog_api_route_tokens", result.groups)
# same-app fallback includes all tests under posthog/api/
self.assertIn("same_app:posthog/api", result.groups)
self.assertIn("posthog/api/test/test_project.py", result.tests)
self.assertIn("posthog/api/test/test_user.py", result.tests)
def test_snob_selection_filters_to_python_files(self) -> None:
selection = _load_selection_module()
with tempfile.TemporaryDirectory() as root:
selection.REPO_ROOT = Path(root)
selected_test = selection.REPO_ROOT / "posthog" / "api" / "test" / "test_feature_flags.py"
selected_test.parent.mkdir(parents=True)
selected_test.write_text("def test_feature_flags():\n pass\n")
deleted_test = selection.REPO_ROOT / "posthog" / "api" / "test" / "test_deleted.py"
(selection.REPO_ROOT / "pytest.ini").write_text(
"[pytest]\naddopts = --reuse-db --ignore=posthog/user_scripts\n"
)
pytest_ignored_file = selection.REPO_ROOT / "posthog" / "user_scripts" / "aggregate_funnel_test.py"
pytest_ignored_file.parent.mkdir(parents=True)
pytest_ignored_file.write_text("")
seen_changed_files: list[list[str]] = []
fake_snob = ModuleType("snob_lib")
def get_tests(changed_files: list[str]) -> set[str]:
seen_changed_files.append(changed_files)
return {str(selected_test), str(deleted_test), str(pytest_ignored_file)}
fake_snob.get_tests = get_tests # type: ignore[attr-defined]
previous_snob = sys.modules.get("snob_lib")
sys.modules["snob_lib"] = fake_snob
try:
result = selection.snob_select_tests(["posthog/api/feature_flags.py", "frontend/src/index.ts"])
finally:
if previous_snob is None:
del sys.modules["snob_lib"]
else:
sys.modules["snob_lib"] = previous_snob
self.assertEqual([["posthog/api/feature_flags.py"]], seen_changed_files)
self.assertEqual(
{"status": "ok", "tests": ["posthog/api/test/test_feature_flags.py"], "count": 1},
result,
)
def test_signal_handler_change_expands_to_app_and_api_tests(self) -> None:
with tempfile.TemporaryDirectory() as root:
tmp_path = Path(root)
selection = _load_selection_module()
selection.REPO_ROOT = tmp_path
# A signal handler file
handler = tmp_path / "posthog" / "models" / "signal_handlers.py"
handler.parent.mkdir(parents=True)
handler.write_text("from django.db.models.signals import post_save\npost_save.connect(my_handler)\n")
# A test in the same app
test_path = tmp_path / "posthog" / "models" / "test" / "test_models.py"
test_path.parent.mkdir(parents=True)
test_path.write_text("def test_model(): pass\n")
# An API test elsewhere
api_test = tmp_path / "posthog" / "api" / "test" / "test_something.py"
api_test.parent.mkdir(parents=True)
api_test.write_text("from rest_framework.test import APIClient\nclient = APIClient()\n")
features_by_path = selection.classify_tests()
result = selection.ast_select_tests(
["posthog/models/signal_handlers.py"],
features_by_path,
)
self.assertIn("signal_handler_app:posthog/models", result.groups)
self.assertIn("posthog/models/test/test_models.py", result.tests)
def test_middleware_change_expands_to_api_tests(self) -> None:
selection = _load_selection_module()
result = selection.ast_select_tests(
["posthog/gzip_middleware.py"],
{
"posthog/api/test/test_capture.py": selection.TestFeatures(
path="posthog/api/test/test_capture.py",
imports_api_client=True,
),
"posthog/models/test/test_utils.py": selection.TestFeatures(
path="posthog/models/test/test_utils.py",
),
},
)
self.assertIn("middleware_api_tests", result.groups)
self.assertIn("posthog/api/test/test_capture.py", result.tests)
# Non-API tests are NOT included by middleware expansion
self.assertNotIn("posthog/models/test/test_utils.py", result.groups.get("middleware_api_tests", []))
def test_db_router_change_expands_to_api_tests(self) -> None:
selection = _load_selection_module()
result = selection.ast_select_tests(
["posthog/product_db_router.py"],
{
"posthog/api/test/test_user.py": selection.TestFeatures(
path="posthog/api/test/test_user.py",
imports_api_client=True,
),
},
)
self.assertIn("db_router_api_tests", result.groups)
self.assertIn("posthog/api/test/test_user.py", result.tests)
def test_same_app_fallback_includes_sibling_tests(self) -> None:
selection = _load_selection_module()
result = selection.ast_select_tests(
["products/surveys/backend/models.py"],
{
"products/surveys/backend/test/test_api.py": selection.TestFeatures(
path="products/surveys/backend/test/test_api.py",
),
"products/experiments/backend/test/test_api.py": selection.TestFeatures(
path="products/experiments/backend/test/test_api.py",
),
},
)
# Same-app tests included
self.assertIn("products/surveys/backend/test/test_api.py", result.tests)
# Different app tests NOT included
self.assertNotIn("products/experiments/backend/test/test_api.py", result.tests)
def test_too_many_files_signals_full_run(self) -> None:
selection = _load_selection_module()
many_files = [f"posthog/models/model_{i}.py" for i in range(60)]
result = selection.ast_select_tests(many_files, {})
self.assertTrue(any("too many changed files" in r for r in result.full_run_reasons))
def test_high_fanout_file_signals_full_run(self) -> None:
selection = _load_selection_module()
original_path = selection.HIGH_FANOUT_PATH
try:
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
f.write("posthog/redis.py\nposthog/models/team/team.py\n")
selection.HIGH_FANOUT_PATH = Path(f.name)
result = selection.ast_select_tests(["posthog/redis.py"], {})
self.assertTrue(any("high-fanout" in r for r in result.full_run_reasons))
finally:
selection.HIGH_FANOUT_PATH = original_path
def test_changed_tests_do_not_trigger_full_run_patterns(self) -> None:
with tempfile.TemporaryDirectory() as root:
selection = _load_selection_module()
selection.REPO_ROOT = Path(root)
test_path = selection.REPO_ROOT / "posthog" / "test" / "test_version_requirement.py"
test_path.parent.mkdir(parents=True)
test_path.touch()
result = selection.ast_select_tests(
["posthog/test/test_version_requirement.py"],
{
"posthog/test/test_version_requirement.py": selection.TestFeatures(
path="posthog/test/test_version_requirement.py"
)
},
)
self.assertEqual([], result.full_run_reasons)
self.assertEqual({"changed_tests": ["posthog/test/test_version_requirement.py"]}, result.groups)
# ci-backend's `legacy` paths filter routes these into test selection, but none of
# them is Python, so the import graph reaches no test through them. Without a full-run
# pattern the selector returns an empty set and the narrowed run gates on nothing.
@parameterized.expand(
[
("quarantine_lift", ".test_quarantine.json"),
("hogql_parser_sources", "common/hogql_parser/HogQLParser.cpp"),
("hogql_parser_rs_sources", "rust/hogql/parser/src/lib.rs"),
("hogvm", "common/hogvm/python/execute.py"),
("personhog_stubs", "packages/personhog-proto/personhog/types/v1/person_pb2.py"),
("product_manifest", "products/surveys/manifest.tsx"),
]
)
def test_non_python_legacy_inputs_signal_full_run(self, _name: str, path: str) -> None:
selection = _load_selection_module()
result = selection.ast_select_tests([path], {})
self.assertTrue(result.full_run_reasons, f"{path} selected nothing and forced no full run")
@parameterized.expand(
[
("core", "posthog/models/test_a.py", {"core"}),
# POE patterns run in both the Core matrix and the person-on-events matrix.
("poe_runs_in_core_too", "posthog/clickhouse/test_b.py", {"core", "poe"}),
("temporal", "posthog/temporal/tests/test_c.py", {"temporal"}),
# The Temporal invocation runs the product suites alongside posthog/temporal.
("temporal_signals_emission", "products/signals/backend/emission/test_c.py", {"temporal"}),
# Explicitly ignored by the Core invocation, and not a draft-narrowable matrix.
("core_ignored_dags", "posthog/dags/test_e.py", set()),
("core_ignored_repo_invariants", "posthog/test/repo_invariants/test_f.py", set()),
# Product/turbo tests are not part of any draft-narrowable Django matrix.
("turbo_product", "products/warehouse_sources/backend/test_d.py", set()),
]
)
def test_segments_for_test_file_mirrors_matrix_partition(self, _name: str, path: str, expected: set[str]) -> None:
selection = _load_selection_module()
self.assertEqual(selection.segments_for_test_file(path), frozenset(expected))
@parameterized.expand(
[
(
"compat_targets_set",
"posthog/clickhouse ee/clickhouse",
["ee/clickhouse/test_g.py", "posthog/clickhouse/test_b.py"],
),
("no_compat_targets", "", []),
]
)
def test_selected_files_by_segment_reads_compat_targets_from_the_env(
self, _name: str, targets: str, expected_compat: list[str]
) -> None:
selection = _load_selection_module()
selected = [
"ee/clickhouse/test_g.py",
"posthog/clickhouse/test_b.py",
"posthog/dags/test_e.py",
"posthog/models/test_a.py",
"posthog/temporal/tests/test_c.py",
"products/warehouse_sources/backend/test_d.py",
]
with mock.patch.dict(os.environ, {"CLICKHOUSE_COMPAT_PYTEST_TARGETS": targets}):
by_segment = selection.selected_files_by_segment(selected)
self.assertEqual(
by_segment,
{
"core": ["ee/clickhouse/test_g.py", "posthog/clickhouse/test_b.py", "posthog/models/test_a.py"],
"poe": ["ee/clickhouse/test_g.py", "posthog/clickhouse/test_b.py"],
"temporal": ["posthog/temporal/tests/test_c.py"],
"compat": expected_compat,
},
)
def test_narrowable_baseline_excludes_turbo_product_tests(self) -> None:
selection = _load_selection_module()
durations = {
"posthog/models/test_a.py::t1": 100.0,
"posthog/clickhouse/test_b.py::t2": 70.0,
"posthog/temporal/tests/test_c.py::t3": 210.0,
"posthog/dags/test_e.py::t5": 42.0, # ignored path
"products/warehouse_sources/backend/test_d.py::t4": 100_000.0, # turbo-tests
}
# Only the Core/POE/Temporal universe counts; the huge product test and the ignored
# path are excluded, so a draft can never be credited with skipping them.
self.assertEqual(selection.narrowable_baseline_seconds(durations), 380.0)
def test_selected_seconds_by_segment_splits_the_matrix_legs(self) -> None:
selection = _load_selection_module()
durations = {
"posthog/models/test_a.py::t1": 100.0,
"posthog/clickhouse/test_b.py::t2": 70.0,
"posthog/temporal/tests/test_c.py::t3": 210.0,
"posthog/models/test_unselected.py::t4": 5_000.0,
"products/warehouse_sources/backend/test_d.py::t5": 100_000.0,
"posthog/dags/test_e.py::t6": 42.0,
}
selected = [
"posthog/models/test_a.py",
"posthog/clickhouse/test_b.py",
"posthog/temporal/tests/test_c.py",
"products/warehouse_sources/backend/test_d.py",
"posthog/dags/test_e.py",
]
# The clickhouse file runs in both the Core and the person-on-events leg, so its
# seconds count once per leg. Turbo product tests and ignored paths size no Django
# shard, and an unselected file never contributes.
self.assertEqual(
selection.selected_seconds_by_segment(selected, durations),
{"core": 170, "poe": 70, "temporal": 210},
)
if __name__ == "__main__":
unittest.main()