:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): persist versioned filter catalogs (#125958)
Mikayla Thompsone2c10e7130da
diff --git a/.github/scripts/check-idor-model-coverage.py b/.github/scripts/check-idor-model-coverage.pyindex ffe2c08f1681..addd4ffc2665 100644--- a/.github/scripts/check-idor-model-coverage.py+++ b/.github/scripts/check-idor-model-coverage.py@@ -329,6 +329,8 @@ def get_scoped_models() -> tuple[dict[str, set[str]], set[str], set[str], set[st"PromptSequence","UserPromptState",# --- Global catalogs ---+ "SignalFilterSourceDefinition", # instance-global internal source capability metadata+ "SignalFilterCatalogVersion", # immutable shared recipes, never tenant observations"CommunitySkill", # instance-global community skills catalog, synced from GitHub"CommunitySkillFile", # bundled files of a CommunitySkill (scoped via the catalog row)"HogFunctionTemplate",diff --git a/products/signals/backend/filter_catalog.py b/products/signals/backend/filter_catalog.pynew file mode 100644index 000000000000..b6b0ced947c6--- /dev/null+++ b/products/signals/backend/filter_catalog.py@@ -0,0 +1,144 @@+from __future__ import annotations++from collections.abc import Collection, Iterable+from typing import TYPE_CHECKING, TypeVar++from django.core.exceptions import ValidationError+from django.db import models, router, transaction++from posthog.models.utils import UUIDModel++from products.signals.backend.filter_catalog_schema import validate_catalog_bundle++if TYPE_CHECKING:+ from django.db.models.base import ModelBase++CatalogModel = TypeVar("CatalogModel", bound=models.Model)+++class CatalogQuerySet(models.QuerySet[CatalogModel]):+ def update(self, **kwargs: object) -> int:+ raise ValidationError("Catalog records must be written with save()")++ def delete(self) -> tuple[int, dict[str, int]]:+ raise ValidationError("Catalog history must be retained")++ def bulk_create(+ self,+ objs: Iterable[CatalogModel],+ batch_size: int | None = None,+ ignore_conflicts: bool = False,+ update_conflicts: bool = False,+ update_fields: Collection[str] | None = None,+ unique_fields: Collection[str] | None = None,+ ) -> list[CatalogModel]:+ raise ValidationError("Catalog records must be written with save()")+++class InternalCatalogModel(UUIDModel):+ """Instance-global backend metadata; never stores tenant observations or values."""++ objects = CatalogQuerySet.as_manager()++ class Meta:+ abstract = True++ def delete(self, using: str | None = None, keep_parents: bool = False) -> tuple[int, dict[str, int]]:+ raise ValidationError("Catalog history must be retained")+++class SignalFilterSourceDefinition(InternalCatalogModel):+ class AdapterKind(models.TextChoices):+ WAREHOUSE = "warehouse"++ class RolloutState(models.TextChoices):+ DISABLED = "disabled"+ INTERNAL = "internal"+ ENABLED = "enabled"++ source_product = models.CharField(max_length=128)+ source_type = models.CharField(max_length=128)+ adapter_kind = models.CharField(max_length=32, choices=AdapterKind.choices)+ rollout_state = models.CharField(max_length=16, choices=RolloutState.choices, default=RolloutState.DISABLED)+ filter_query_eligible = models.BooleanField(default=False)+ active_catalog_version = models.ForeignKey(+ "signals.SignalFilterCatalogVersion", null=True, blank=True, on_delete=models.PROTECT, related_name="+"+ )+ metadata = models.JSONField(default=dict, blank=True)++ class Meta:+ app_label = "signals"+ base_manager_name = "objects"+ constraints = [+ models.UniqueConstraint(fields=["source_product", "source_type"], name="signals_filter_source_unique"),+ models.CheckConstraint(+ condition=models.Q(filter_query_eligible=False) | models.Q(active_catalog_version__isnull=False),+ name="signals_filter_eligible_catalog",+ ),+ ]++ def save(+ self,+ *,+ force_insert: bool | tuple[ModelBase, ...] = False,+ force_update: bool = False,+ using: str | None = None,+ update_fields: Iterable[str] | None = None,+ ) -> None:+ using = using or router.db_for_write(type(self), instance=self)+ with transaction.atomic(using=using):+ original = (+ type(self).objects.using(using).select_for_update().filter(pk=self.pk).first() if self.pk else None+ )+ if original is not None:+ if (self.source_product, self.source_type, self.adapter_kind) != (+ original.source_product,+ original.source_type,+ original.adapter_kind,+ ):+ raise ValidationError("Source identity and adapter are immutable")+ if self.active_catalog_version_id is not None:+ if (+ not SignalFilterCatalogVersion.objects.using(using)+ .filter(pk=self.active_catalog_version_id, source_id=self.pk)+ .exists()+ ):+ raise ValidationError("Active catalog version must belong to this source")+ if self.filter_query_eligible and self.active_catalog_version_id is None:+ raise ValidationError("Filter eligibility requires an active catalog version")+ self.full_clean()+ super().save(force_insert=force_insert, force_update=force_update, using=using, update_fields=update_fields)++ def activate(self, catalog_version: SignalFilterCatalogVersion) -> None:+ if catalog_version.pk is None or catalog_version._state.adding:+ raise ValidationError("Catalog version must be saved before activation")+ self.active_catalog_version_id = catalog_version.pk+ self.save(update_fields=["active_catalog_version"])+++class SignalFilterCatalogVersion(InternalCatalogModel):+ source = models.ForeignKey(SignalFilterSourceDefinition, on_delete=models.PROTECT, related_name="catalog_versions")+ version = models.PositiveIntegerField()+ bundle = models.JSONField(validators=[validate_catalog_bundle])+ created_at = models.DateTimeField(auto_now_add=True)++ class Meta:+ app_label = "signals"+ base_manager_name = "objects"+ constraints = [+ models.UniqueConstraint(fields=["source", "version"], name="signals_filter_catalog_version_unique"),+ models.CheckConstraint(condition=models.Q(version__gt=0), name="signals_filter_catalog_version_positive"),+ ]++ def save(+ self,+ *,+ force_insert: bool | tuple[ModelBase, ...] = False,+ force_update: bool = False,+ using: str | None = None,+ update_fields: Iterable[str] | None = None,+ ) -> None:+ if not self._state.adding or force_update or update_fields is not None:+ raise ValidationError("Catalog versions are immutable; create a new version")+ self.full_clean()+ super().save(force_insert=True, using=using)diff --git a/products/signals/backend/filter_catalog_schema.py b/products/signals/backend/filter_catalog_schema.pynew file mode 100644index 000000000000..cf337b82a6af--- /dev/null+++ b/products/signals/backend/filter_catalog_schema.py@@ -0,0 +1,95 @@+import json+from typing import Annotated, Literal++from django.core.exceptions import ValidationError++from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator++Identifier = Annotated[str, Field(min_length=1, max_length=128, pattern=r"^[a-zA-Z_][a-zA-Z0-9_]*$")]+Path = Annotated[list[Annotated[str, Field(min_length=1, max_length=128)]], Field(max_length=8)]+++class CatalogSchema(BaseModel):+ model_config = ConfigDict(extra="forbid", strict=True)+++class ColumnRecipe(CatalogSchema):+ kind: Literal["column"]+ column: Identifier+++class JsonPathRecipe(CatalogSchema):+ kind: Literal["json_path"]+ column: Identifier+ path: Path+++class JsonArrayMapRecipe(CatalogSchema):+ kind: Literal["json_array_map"]+ column: Identifier+ array_path: Path+ value_path: Path+++class CatalogField(CatalogSchema):+ name: Identifier+ type: Literal[+ "string",+ "integer",+ "number",+ "boolean",+ "datetime",+ "array<string>",+ "array<integer>",+ "array<number>",+ "array<boolean>",+ ]+ description: Annotated[str, Field(min_length=1, max_length=2048)]+ binding: Annotated[ColumnRecipe | JsonPathRecipe | JsonArrayMapRecipe, Field(discriminator="kind")]+ missing_value: Literal["null", "empty_array"]+ malformed_value: Literal["error"]+ provenance: dict[str, JsonValue] = Field(default_factory=dict)+ overrides: dict[str, JsonValue] = Field(default_factory=dict)++ @model_validator(mode="after")+ def validate_missing_value(self) -> "CatalogField":+ if self.missing_value == "empty_array" and not self.type.startswith("array<"):+ raise ValueError("empty_array requires an array field type")+ if isinstance(self.binding, JsonArrayMapRecipe) and not self.type.startswith("array<"):+ raise ValueError("json_array_map requires an array field type")+ return self+++class CatalogBundle(CatalogSchema):+ schema_version: Literal[1]+ fields: Annotated[list[CatalogField], Field(min_length=1, max_length=256)]+ starter_expressions: Annotated[list[Annotated[str, Field(min_length=1, max_length=4096)]], Field(max_length=32)]+ provenance: dict[str, JsonValue] = Field(default_factory=dict)+ overrides: dict[str, JsonValue] = Field(default_factory=dict)++ @model_validator(mode="after")+ def validate_unique_names(self) -> "CatalogBundle":+ names = [field.name for field in self.fields]+ if len(names) != len(set(names)):+ raise ValueError("Catalog field names must be unique")+ return self+++def validate_catalog_bundle(value: object) -> None:+ try:+ if len(json.dumps(value, allow_nan=False).encode()) > 131072:+ raise ValueError("Catalog bundle exceeds 128 KiB")+ pending = [(value, 0)]+ nodes = 0+ while pending:+ item, depth = pending.pop()+ nodes += 1+ if depth > 16 or nodes > 8192:+ raise ValueError("Catalog bundle is too deeply nested or complex")+ if isinstance(item, dict):+ pending.extend((child, depth + 1) for child in item.values())+ elif isinstance(item, list):+ pending.extend((child, depth + 1) for child in item)+ CatalogBundle.model_validate(value)+ except (ValueError, TypeError, RecursionError) as error:+ raise ValidationError("Invalid catalog bundle: %(reason)s", params={"reason": str(error)}) from errordiff --git a/products/signals/backend/migrations/0221_signal_filter_catalog.py b/products/signals/backend/migrations/0221_signal_filter_catalog.pynew file mode 100644index 000000000000..44f4028e9344--- /dev/null+++ b/products/signals/backend/migrations/0221_signal_filter_catalog.py@@ -0,0 +1,124 @@+# Generated by Django 5.2.18 on 2026-10-29 22:27++import django.db.models.deletion+from django.db import migrations, models++import posthog.uuidt++import products.signals.backend.filter_catalog_schema+++class Migration(migrations.Migration):+ dependencies = [("signals", "0220_signalteamconfig_default_slack_notification_integration")]++ operations = [+ migrations.CreateModel(+ name="SignalFilterSourceDefinition",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("source_product", models.CharField(max_length=128)),+ ("source_type", models.CharField(max_length=128)),+ (+ "adapter_kind",+ models.CharField(choices=[("warehouse", "Warehouse")], max_length=32),+ ),+ (+ "rollout_state",+ models.CharField(+ choices=[+ ("disabled", "Disabled"),+ ("internal", "Internal"),+ ("enabled", "Enabled"),+ ],+ default="disabled",+ max_length=16,+ ),+ ),+ ("filter_query_eligible", models.BooleanField(default=False)),+ ("metadata", models.JSONField(blank=True, default=dict)),+ ],+ options={"base_manager_name": "objects"},+ ),+ migrations.CreateModel(+ name="SignalFilterCatalogVersion",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("version", models.PositiveIntegerField()),+ (+ "bundle",+ models.JSONField(+ validators=[products.signals.backend.filter_catalog_schema.validate_catalog_bundle]+ ),+ ),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ (+ "source",+ models.ForeignKey(+ on_delete=django.db.models.deletion.PROTECT,+ related_name="catalog_versions",+ to="signals.signalfiltersourcedefinition",+ ),+ ),+ ],+ options={"base_manager_name": "objects"},+ ),+ migrations.AddField(+ model_name="signalfiltersourcedefinition",+ name="active_catalog_version",+ field=models.ForeignKey(+ blank=True,+ null=True,+ on_delete=django.db.models.deletion.PROTECT,+ related_name="+",+ to="signals.signalfiltercatalogversion",+ ),+ ),+ migrations.AddConstraint(+ model_name="signalfiltersourcedefinition",+ constraint=models.UniqueConstraint(+ fields=("source_product", "source_type"),+ name="signals_filter_source_unique",+ ),+ ),+ migrations.AddConstraint(+ model_name="signalfiltersourcedefinition",+ constraint=models.CheckConstraint(+ condition=models.Q(+ ("filter_query_eligible", False),+ ("active_catalog_version__isnull", False),+ _connector="OR",+ ),+ name="signals_filter_eligible_catalog",+ ),+ ),+ migrations.AddConstraint(+ model_name="signalfiltercatalogversion",+ constraint=models.UniqueConstraint(+ fields=("source", "version"),+ name="signals_filter_catalog_version_unique",+ ),+ ),+ migrations.AddConstraint(+ model_name="signalfiltercatalogversion",+ constraint=models.CheckConstraint(+ condition=models.Q(("version__gt", 0)),+ name="signals_filter_catalog_version_positive",+ ),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex b24eb9cdc9b1..ec3fdca4697f 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0220_signalteamconfig_default_slack_notification_integration+0221_signal_filter_catalogdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex f53c72a60266..52f8818cf11a 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -54,6 +54,10 @@ from products.signals.backend.enums import (SignalSourceType,signal_source_product_choices,)+from products.signals.backend.filter_catalog import (+ SignalFilterCatalogVersion as SignalFilterCatalogVersion,+ SignalFilterSourceDefinition as SignalFilterSourceDefinition,+)from products.signals.backend.report_checks import MAX_CHECK_TITLE_LENGTHfrom products.signals.backend.report_monitoring_aggregation import MonitoringAggregationdiff --git a/products/signals/backend/test/test_filter_catalog.py b/products/signals/backend/test/test_filter_catalog.pynew file mode 100644index 000000000000..75193d927ca2--- /dev/null+++ b/products/signals/backend/test/test_filter_catalog.py@@ -0,0 +1,208 @@+from copy import deepcopy++from django.core.exceptions import ValidationError+from django.db import IntegrityError, transaction+from django.test import SimpleTestCase, TestCase++from parameterized import parameterized+from pydantic import JsonValue++from products.signals.backend.filter_catalog_schema import validate_catalog_bundle+from products.signals.backend.models import SignalFilterCatalogVersion, SignalFilterSourceDefinition+++def catalog_field(**changes: JsonValue) -> dict[str, JsonValue]:+ return {+ "name": "category_name",+ "type": "string",+ "description": "The candidate's category name.",+ "binding": {"kind": "json_path", "column": "category", "path": ["name"]},+ "missing_value": "null",+ "malformed_value": "error",+ **changes,+ }+++def catalog_bundle(**changes: JsonValue) -> dict[str, JsonValue]:+ return {+ "schema_version": 1,+ "fields": [catalog_field()],+ "starter_expressions": ["category_name IS NULL"],+ "provenance": {"origin": "manual"},+ "overrides": {"descriptions": True},+ **changes,+ }+++class TestCatalogBundle(SimpleTestCase):+ @parameterized.expand(+ [+ ("column", {"kind": "column", "column": "category"}, "string", "null"),+ ("json_path", {"kind": "json_path", "column": "category", "path": ["name"]}, "string", "null"),+ (+ "array",+ {"kind": "json_array_map", "column": "tags", "array_path": ["items"], "value_path": ["name"]},+ "array<string>",+ "empty_array",+ ),+ ]+ )+ def test_supported_recipe_shapes(self, _name: str, binding: JsonValue, field_type: str, missing: str) -> None:+ validate_catalog_bundle(+ catalog_bundle(fields=[catalog_field(binding=binding, type=field_type, missing_value=missing)])+ )++ @parameterized.expand(+ [+ ("unknown_recipe", catalog_bundle(fields=[catalog_field(binding={"kind": "sql", "expression": "1"})])),+ (+ "extra_recipe_key",+ catalog_bundle(fields=[catalog_field(binding={"kind": "column", "column": "category", "sql": "1"})]),+ ),+ (+ "path_depth",+ catalog_bundle(+ fields=[catalog_field(binding={"kind": "json_path", "column": "category", "path": ["x"] * 9})]+ ),+ ),+ ("unknown_type", catalog_bundle(fields=[catalog_field(type="object")])),+ (+ "array_recipe_for_scalar",+ catalog_bundle(+ fields=[+ catalog_field(+ binding={+ "kind": "json_array_map",+ "column": "tags",+ "array_path": [],+ "value_path": ["name"],+ }+ )+ ]+ ),+ ),+ ("missing_array_for_scalar", catalog_bundle(fields=[catalog_field(missing_value="empty_array")])),+ ("malformed_as_missing", catalog_bundle(fields=[catalog_field(malformed_value="null")])),+ ("duplicate_name", catalog_bundle(fields=[catalog_field(), catalog_field()])),+ ("bad_identifier", catalog_bundle(fields=[catalog_field(name="category.name")])),+ ("starter_type", catalog_bundle(starter_expressions=[False])),+ ("unknown_schema", catalog_bundle(schema_version=2)),+ ("too_many_fields", catalog_bundle(fields=[catalog_field(name=f"field_{index}") for index in range(257)])),+ ("oversized", catalog_bundle(provenance={"note": "x" * 131072})),+ ]+ )+ def test_invalid_bundle(self, _name: str, bundle: object) -> None:+ with self.assertRaises(ValidationError):+ validate_catalog_bundle(bundle)++ def test_metadata_nesting_is_bounded(self) -> None:+ value: JsonValue = "origin"+ for _ in range(20):+ value = {"nested": value}+ with self.assertRaises(ValidationError):+ validate_catalog_bundle(catalog_bundle(provenance={"origin": value}))+++class TestFilterCatalogPersistence(TestCase):+ source: SignalFilterSourceDefinition+ catalog: SignalFilterCatalogVersion++ @classmethod+ def setUpTestData(cls) -> None:+ cls.source = SignalFilterSourceDefinition.objects.create(+ source_product="example_tracker", source_type="issue", adapter_kind="warehouse"+ )+ cls.catalog = SignalFilterCatalogVersion.objects.create(source=cls.source, version=1, bundle=catalog_bundle())++ def test_activation_and_rollback_retain_versions(self) -> None:+ second = SignalFilterCatalogVersion.objects.create(source=self.source, version=2, bundle=catalog_bundle())+ self.source.activate(second)+ self.source.refresh_from_db()+ self.assertEqual(self.source.active_catalog_version_id, second.pk)+ self.assertFalse(self.source.filter_query_eligible)+ self.source.activate(self.catalog)+ self.source.refresh_from_db()+ self.assertEqual(self.source.active_catalog_version_id, self.catalog.pk)+ self.assertEqual(self.source.catalog_versions.count(), 2)++ def test_unsaved_version_cannot_be_activated(self) -> None:+ with self.assertRaises(ValidationError):+ self.source.activate(SignalFilterCatalogVersion(source=self.source, version=2, bundle=catalog_bundle()))++ def test_cross_source_activation_preserves_active_version(self) -> None:+ self.source.activate(self.catalog)+ other = SignalFilterSourceDefinition.objects.create(+ source_product="another_tracker", source_type="issue", adapter_kind="warehouse"+ )+ other_catalog = SignalFilterCatalogVersion.objects.create(source=other, version=1, bundle=catalog_bundle())+ with self.assertRaises(ValidationError):+ self.source.activate(other_catalog)+ self.source.refresh_from_db()+ self.assertEqual(self.source.active_catalog_version_id, self.catalog.pk)++ @parameterized.expand(["add", "set"])+ def test_reverse_relation_cannot_reassign_active_version(self, operation: str) -> None:+ self.source.activate(self.catalog)+ other = SignalFilterSourceDefinition.objects.create(+ source_product="another_tracker", source_type="issue", adapter_kind="warehouse"+ )+ with self.assertRaises(ValidationError):+ if operation == "add":+ other.catalog_versions.add(self.catalog)+ else:+ other.catalog_versions.set([self.catalog])+ self.catalog.refresh_from_db()+ self.source.refresh_from_db()+ self.assertEqual(self.catalog.source_id, self.source.pk)+ self.assertEqual(self.source.active_catalog_version_id, self.catalog.pk)++ @parameterized.expand(["bundle", "version", "source_id"])+ def test_published_version_is_immutable(self, field: str) -> None:+ original = deepcopy(self.catalog.bundle)+ setattr(self.catalog, field, {"changed": True} if field == "bundle" else 999)+ with self.assertRaises(ValidationError):+ self.catalog.save()+ self.catalog.refresh_from_db()+ self.assertEqual(self.catalog.bundle, original)+ self.assertEqual(self.catalog.version, 1)+ self.assertEqual(self.catalog.source_id, self.source.pk)++ def test_invalid_bundle_cannot_be_saved(self) -> None:+ with self.assertRaises(ValidationError):+ SignalFilterCatalogVersion.objects.create(source=self.source, version=2, bundle={})++ @parameterized.expand(["source_product", "source_type", "adapter_kind"])+ def test_source_identity_is_immutable(self, field: str) -> None:+ setattr(self.source, field, "changed")+ with self.assertRaises(ValidationError):+ self.source.save()++ def test_eligibility_requires_active_catalog(self) -> None:+ self.source.filter_query_eligible = True+ with self.assertRaises(ValidationError):+ self.source.save()++ def test_duplicate_source_and_version_are_rejected(self) -> None:+ with self.assertRaises(ValidationError):+ SignalFilterSourceDefinition.objects.create(+ source_product="example_tracker", source_type="issue", adapter_kind="warehouse"+ )+ with self.assertRaises(ValidationError):+ SignalFilterCatalogVersion.objects.create(source=self.source, version=1, bundle=catalog_bundle())+ with self.assertRaises(IntegrityError), transaction.atomic():+ duplicate = SignalFilterCatalogVersion(source=self.source, version=1, bundle=catalog_bundle())+ super(SignalFilterCatalogVersion, duplicate).save(force_insert=True)++ def test_bulk_writes_and_deletion_cannot_bypass_invariants(self) -> None:+ with self.assertRaises(ValidationError):+ SignalFilterCatalogVersion.objects.filter(pk=self.catalog.pk).update(bundle={})+ with self.assertRaises(ValidationError):+ SignalFilterCatalogVersion.objects.bulk_create(+ [SignalFilterCatalogVersion(source=self.source, version=2, bundle={})]+ )+ with self.assertRaises(ValidationError):+ SignalFilterSourceDefinition.objects.filter(pk=self.source.pk).update(active_catalog_version_id=999)+ with self.assertRaises(ValidationError):+ self.catalog.delete()+ with self.assertRaises(ValidationError):+ SignalFilterCatalogVersion.objects.filter(pk=self.catalog.pk).delete()