:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(endpoints): use node targets for materialized freshness (#123115)
Jovan Sakovicabae763405b6
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txtindex 52609d5eff6b..8f622608f96e 100644--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt@@ -103,6 +103,7 @@ post_delete:products.cdp.backend.models.plugin.plugin_reload_neededpost_delete:products.cohorts.backend.models.cohort.cohort_people_changedpost_delete:products.cohorts.backend.models.dependencies.clear_team_cohort_dependency_cachepost_delete:products.cohorts.backend.models.dependencies.cohort_deleted+post_delete:products.endpoints.backend.signals.invalidate_endpoint_node_cachepost_delete:products.event_definitions.backend.models.event_definition._invalidate_default_event_info_on_deletepost_delete:products.experiments.backend.facade.change_signals._notify_experiment_deletedpost_delete:products.feature_flags.backend.flags_cache.cohort_changed_flags_cache@@ -222,6 +223,7 @@ post_save:products.data_modeling.backend.signals.sync_saved_query_name_to_nodepost_save:products.data_modeling.backend.signals.trigger_view_semantic_enrichmentpost_save:products.data_warehouse.backend.models.revenue_analytics_config.create_external_data_source_revenue_analytics_configpost_save:products.early_access_features.backend.signals.create_waitlist_survey_on_concept_stage+post_save:products.endpoints.backend.signals.invalidate_endpoint_node_cachepost_save:products.event_definitions.backend.models.event_definition._invalidate_default_event_info_on_savepost_save:products.event_definitions.backend.models.property_definition._invalidate_has_person_email_on_savepost_save:products.experiments.backend.facade.change_signals._notify_experiment_saveddiff --git a/products/endpoints/backend/admin.py b/products/endpoints/backend/admin.pyindex b419f38b3c31..910b2a7016f6 100644--- a/products/endpoints/backend/admin.py+++ b/products/endpoints/backend/admin.py@@ -17,7 +17,6 @@ class EndpointVersionInline(admin.TabularInline):"version","is_active","query_kind",- "data_freshness_seconds","is_materialized_display","saved_query_name","saved_query_status",@@ -234,6 +233,8 @@ class EndpointVersionAdmin(admin.ModelAdmin):super().get_queryset(request).select_related("endpoint", "endpoint__team", "saved_query", "saved_query__table", "created_by")+ # data_freshness_seconds reads the node target, so prefetch it once for the whole page.+ .prefetch_related(EndpointVersion.frequency_nodes_prefetch()))@admin.display(description="Endpoint")diff --git a/products/endpoints/backend/apps.py b/products/endpoints/backend/apps.pyindex ae9b2536dacd..d1512add583b 100644--- a/products/endpoints/backend/apps.py+++ b/products/endpoints/backend/apps.py@@ -6,3 +6,6 @@ class EndpointsConfig(AppConfig):name = "products.endpoints.backend"label = "endpoints"verbose_name = "Endpoints (product)"++ def ready(self) -> None:+ from products.endpoints.backend import signals # noqa: F401diff --git a/products/endpoints/backend/logic/crud.py b/products/endpoints/backend/logic/crud.pyindex e8102c1b8d78..327edc1c14c2 100644--- a/products/endpoints/backend/logic/crud.py+++ b/products/endpoints/backend/logic/crud.py@@ -118,7 +118,7 @@ class EndpointCrudService:version=1,query=query_dict,description=data.description or "",- data_freshness_seconds=(+ cache_ttl_seconds=(data.data_freshness_secondsif data.data_freshness_seconds is not Noneelse DEFAULT_DATA_FRESHNESS_SECONDS@@ -199,6 +199,7 @@ class EndpointCrudService:# get_version raises when no versions exist, so target_version is never None.target_version = target_version_override or current_versionversion_before_update = EndpointVersion.objects.get(pk=target_version.pk)+ previous_data_freshness_seconds = version_before_update.data_freshness_secondswas_materialized = current_version.saved_query_id is not None# Before any write, so a refused update leaves the endpoint as it was.@@ -216,6 +217,9 @@ class EndpointCrudService:# version, not just the current one.for materialized_version in endpoint.versions.filter(saved_query__isnull=False):self.materialization.disable_materialization(endpoint, materialized_version)+ # target_version was loaded before the teardown and still points at the removed saved+ # query, which would make the freshness write below skip the direct query cache duration.+ target_version.refresh_from_db()if data.is_active is not None and not version_targeted:# Activation affects throttle classification — force a lazy re-check.clear_endpoint_materialization_cache(self.team.pk, endpoint.name)@@ -225,25 +229,32 @@ class EndpointCrudService:endpoint, data, target_version, was_materialized)- step = "version_fields"- self._apply_version_field_updates(endpoint, target_version, data, raw_data, version_targeted)-- step = "materialization"- materialization_error = self._reconcile_materialization(- endpoint,- data,- raw_data,- target_version,- version_targeted,- was_materialized=was_materialized,- version_was_created=version_was_created,- old_bucket_overrides=old_bucket_overrides,- )+ # A rejected enable must also roll back changes to the direct query cache duration.+ with transaction.atomic():+ step = "version_fields"+ self._apply_version_field_updates(endpoint, target_version, data, raw_data, version_targeted)++ step = "materialization"+ materialization_error = self._reconcile_materialization(+ endpoint,+ data,+ raw_data,+ target_version,+ version_targeted,+ was_materialized=was_materialized,+ version_was_created=version_was_created,+ old_bucket_overrides=old_bucket_overrides,+ )step = "tags_and_activity"apply_tags(endpoint, data.tags)self._log_update_activity(- endpoint, endpoint_before_update, target_version, version_before_update, version_was_created+ endpoint,+ endpoint_before_update,+ target_version,+ version_before_update,+ version_was_created,+ previous_data_freshness_seconds,)return EndpointUpdateResult(@@ -344,13 +355,13 @@ class EndpointCrudService:if data.description is not None:target_version.description = data.descriptionupdate_fields.append("description")- if "data_freshness_seconds" in raw_data:- target_version.data_freshness_seconds = (+ if "data_freshness_seconds" in raw_data and target_version.saved_query_id is None:+ target_version.cache_ttl_seconds = (data.data_freshness_secondsif data.data_freshness_seconds is not Noneelse DEFAULT_DATA_FRESHNESS_SECONDS)- update_fields.append("data_freshness_seconds")+ update_fields.append("cache_ttl_seconds")if "optional_breakdown_properties" in raw_data:target_version.optional_breakdown_properties = list(data.optional_breakdown_properties or [])update_fields.append("optional_breakdown_properties")@@ -361,9 +372,14 @@ class EndpointCrudService:if update_fields:update_fields.append("updated_at")target_version.save(update_fields=update_fields)- if {"data_freshness_seconds", "optional_breakdown_properties"} & set(update_fields):+ if {"cache_ttl_seconds", "optional_breakdown_properties"} & set(update_fields):# Both fields are part of the cached serving snapshot the throttle classifies from.- clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[target_version.version])+ transaction.on_commit(+ lambda: clear_endpoint_materialization_cache(+ self.team.pk, endpoint.name, versions=[target_version.version]+ ),+ robust=True,+ )def _reconcile_materialization(self,@@ -382,11 +398,18 @@ class EndpointCrudService:Returns a materialization error message when enabling failed after a newversion was already committed (the update itself still succeeds)."""+ requested_freshness = (+ (data.data_freshness_seconds if data.data_freshness_seconds is not None else DEFAULT_DATA_FRESHNESS_SECONDS)+ if "data_freshness_seconds" in raw_data+ else None+ )if version_targeted and not target_version.is_active:# Deactivating a version: tear down its own materialization, never enable.# Checked before the endpoint-active guard so this holds even on an inactive endpoint.if target_version.saved_query_id is not None:- self.materialization.disable_materialization(endpoint, target_version)+ self.materialization.disable_materialization(+ endpoint, target_version, cache_ttl_seconds=requested_freshness+ )return Noneif not endpoint.is_active:@@ -399,7 +422,9 @@ class EndpointCrudService:should_enable = data.is_materialized is True or (data.is_materialized is None and check_was_materialized)if data.is_materialized is False:- self.materialization.disable_materialization(endpoint, target_version)+ self.materialization.disable_materialization(+ endpoint, target_version, cache_ttl_seconds=requested_freshness+ )return Noneif not should_enable:return None@@ -414,7 +439,7 @@ class EndpointCrudService:self.materialization.enable_materialization(endpoint,target_version,- target_version.data_freshness_seconds,+ requested_freshness if requested_freshness is not None else target_version.data_freshness_seconds,bucket_overrides=bucket_overrides,)if (@@ -424,16 +449,21 @@ class EndpointCrudService:):# enable only triggers an immediate Temporal run when it creates the schedule;# changed bucketing on an existing materialization needs an explicit refresh.- try:- materialize_saved_query(target_version.saved_query)- except Exception:- logger.warning(- "failed_to_trigger_materialization_refresh",- team_id=self.team.pk,- endpoint_name=endpoint.name,- saved_query_id=str(target_version.saved_query_id),- bucket_overrides=bucket_overrides,- )+ saved_query = target_version.saved_query++ def refresh_materialization() -> None:+ try:+ materialize_saved_query(saved_query)+ except Exception:+ logger.warning(+ "failed_to_trigger_materialization_refresh",+ team_id=self.team.pk,+ endpoint_name=endpoint.name,+ saved_query_id=str(saved_query.id),+ bucket_overrides=bucket_overrides,+ )++ transaction.on_commit(refresh_materialization)except Exception as e:if not version_was_created:raise@@ -463,6 +493,7 @@ class EndpointCrudService:target_version: EndpointVersion,version_before_update: EndpointVersion,version_was_created: bool,+ previous_data_freshness_seconds: int,) -> None:endpoint_changes = changes_between("Endpoint", previous=endpoint_before_update, current=endpoint)if endpoint_changes:@@ -492,7 +523,22 @@ class EndpointCrudService:),)else:- version_changes = changes_between("EndpointVersion", previous=version_before_update, current=target_version)+ version_changes = [+ change+ for change in changes_between("EndpointVersion", previous=version_before_update, current=target_version)+ if change.field != "cache_ttl_seconds"+ ]+ data_freshness_seconds = target_version.data_freshness_seconds+ if data_freshness_seconds != previous_data_freshness_seconds:+ version_changes.append(+ Change(+ type="EndpointVersion",+ action="changed",+ field="data_freshness_seconds",+ before=previous_data_freshness_seconds,+ after=data_freshness_seconds,+ )+ )if version_changes:self._log_activity(item_id=str(endpoint.id),diff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.pyindex a7af4a078cd1..7a816ddf2ac5 100644--- a/products/endpoints/backend/logic/execution.py+++ b/products/endpoints/backend/logic/execution.py@@ -794,6 +794,7 @@ class EndpointExecutionService(PydanticModelMixin):try:if not saved_query:raise ValidationError("No materialized query found for this endpoint")+ data_freshness_seconds = version.data_freshness_secondsstrategy = strategy_for(endpoint, version, self.team)query_kind = strategy.query_kind@@ -835,16 +836,16 @@ class EndpointExecutionService(PydanticModelMixin):):# Compute dynamic cache TTL: time remaining until data_freshness window expirescache_ttl = None- if materialized_at and version.data_freshness_seconds:+ if materialized_at and data_freshness_seconds:remaining = (- materialized_at + timedelta(seconds=version.data_freshness_seconds) - timezone.now()+ materialized_at + timedelta(seconds=data_freshness_seconds) - timezone.now()).total_seconds()if remaining <= 0:logger.warning("endpoint_materialization_behind_sla",endpoint_name=endpoint.name,team_id=self.team.pk,- data_freshness_seconds=version.data_freshness_seconds,+ data_freshness_seconds=data_freshness_seconds,last_run_at=materialized_at.isoformat(),remaining_seconds=remaining,)@@ -903,9 +904,9 @@ class EndpointExecutionService(PydanticModelMixin):# Freshness relative to the configured target: >1.0 means behind SLA.# Absolute age is meaningless across endpoints with different frequencies.- if materialized_at and version.data_freshness_seconds:+ if materialized_at and data_freshness_seconds:age_seconds = max((timezone.now() - materialized_at).total_seconds(), 0.0)- ENDPOINT_MATERIALIZED_FRESHNESS_RATIO.observe(age_seconds / version.data_freshness_seconds)+ ENDPOINT_MATERIALIZED_FRESHNESS_RATIO.observe(age_seconds / data_freshness_seconds)return resultexcept Exception as e:diff --git a/products/endpoints/backend/logic/materialization.py b/products/endpoints/backend/logic/materialization.pyindex 1019fc98f70c..7a8a70d87fbe 100644--- a/products/endpoints/backend/logic/materialization.py+++ b/products/endpoints/backend/logic/materialization.py@@ -7,6 +7,7 @@ query transforms live in ``products.endpoints.backend.materialization_transforms"""import dataclasses+from datetime import timedeltafrom typing import castfrom django.db import DatabaseError, transaction@@ -39,7 +40,6 @@ from products.data_modeling.backend.facade.api import (sync_saved_query_to_dag,)from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery-from products.endpoints.backend.constants import DATA_FRESHNESS_BUCKETSfrom products.endpoints.backend.logic.activity import EndpointContextfrom products.endpoints.backend.logic.strategies import apply_where_filter, strategy_forfrom products.endpoints.backend.materialization_transforms import (@@ -54,7 +54,6 @@ from products.endpoints.backend.materialization_transforms import (from products.endpoints.backend.metrics import ENDPOINT_MATERIALIZATION_EVENT_TOTALfrom products.endpoints.backend.models import Endpoint, EndpointVersionfrom products.endpoints.backend.rate_limit import clear_endpoint_materialization_cache-from products.warehouse_sources.backend.facade.models import sync_frequency_to_sync_frequency_intervallogger = structlog.get_logger(__name__)@@ -177,7 +176,10 @@ class EndpointMaterializationService:self._enable_materialization_inner(endpoint, version, data_freshness_seconds, bucket_overrides)# The throttle may hold a not-ready snapshot cached before the enable. Drop it so the# first completed run is picked up on the next request, not when the entry expires.- clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version])+ transaction.on_commit(+ lambda: clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version]),+ robust=True,+ )ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="success").inc()if version.saved_query:self._log_materialization_activity(@@ -340,18 +342,18 @@ class EndpointMaterializationService:)saved_query.external_tables = saved_query.s3_tablessaved_query.is_materialized = True- saved_query.sync_frequency_interval = sync_frequency_to_sync_frequency_interval(- DATA_FRESHNESS_BUCKETS[data_freshness_seconds]- )+ saved_query.sync_frequency_interval = timedelta(seconds=data_freshness_seconds)saved_query.save()- def disable_materialization(self, endpoint: Endpoint, version: EndpointVersion) -> None:+ def disable_materialization(+ self, endpoint: Endpoint, version: EndpointVersion, *, cache_ttl_seconds: int | None = None+ ) -> None:"""Disable materialization for an endpoint version."""if version.saved_query:saved_query_id = str(version.saved_query.id)saved_query_name = version.saved_query.nametry:- version.disable_materialization()+ version.disable_materialization(cache_ttl_seconds=cache_ttl_seconds)except Exception:ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="disable", status="error").inc()raisediff --git a/products/endpoints/backend/logic/validation.py b/products/endpoints/backend/logic/validation.pyindex e88dfb1cc972..8a6b83536081 100644--- a/products/endpoints/backend/logic/validation.py+++ b/products/endpoints/backend/logic/validation.py@@ -54,6 +54,18 @@ def validate_data_freshness(data_freshness_seconds: int | None) -> None:)+def _is_current_data_freshness(+ data_freshness_seconds: int | None, endpoint: Endpoint | None, version_number: int | None+) -> bool:+ """True when the value equals what the API already reports for the version being updated."""+ if data_freshness_seconds is None or endpoint is None:+ return False+ try:+ return endpoint.get_version(version_number).data_freshness_seconds == data_freshness_seconds+ except EndpointVersion.DoesNotExist:+ return False++def validate_bucket_overrides(bucket_overrides: dict[str, str] | None) -> None:"""Raise ValidationError if any bucket override value is not in SUPPORTED_BUCKET_FUNCTIONS."""if not bucket_overrides:@@ -310,7 +322,8 @@ def validate_update_request(version_number: int | None = None,) -> None:"""Validate an update payload against the endpoint's resulting state."""- validate_data_freshness(data.data_freshness_seconds)+ if not _is_current_data_freshness(data.data_freshness_seconds, endpoint, version_number):+ validate_data_freshness(data.data_freshness_seconds)# Determine final states after this request (for validation)will_be_active = data.is_active if data.is_active is not None else (endpoint.is_active if endpoint else True)diff --git a/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.py b/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.pynew file mode 100644index 000000000000..883167114721--- /dev/null+++ b/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.py@@ -0,0 +1,28 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("endpoints", "0034_drop_orphaned_endpoint_saved_query_fk"),+ ]++ operations = [+ migrations.SeparateDatabaseAndState(+ state_operations=[+ migrations.AlterField(+ model_name="endpointversion",+ name="data_freshness_seconds",+ field=models.IntegerField(+ db_column="data_freshness_seconds",+ default=86400,+ help_text="How long direct query results remain in the cache, in seconds.",+ ),+ ),+ migrations.RenameField(+ model_name="endpointversion",+ old_name="data_freshness_seconds",+ new_name="cache_ttl_seconds",+ ),+ ],+ ),+ ]diff --git a/products/endpoints/backend/migrations/max_migration.txt b/products/endpoints/backend/migrations/max_migration.txtindex c367bab5367e..0cab9717b391 100644--- a/products/endpoints/backend/migrations/max_migration.txt+++ b/products/endpoints/backend/migrations/max_migration.txt@@ -1 +1 @@-0034_drop_orphaned_endpoint_saved_query_fk+0035_endpoint_cache_ttldiff --git a/products/endpoints/backend/models.py b/products/endpoints/backend/models.pyindex 0b64dcab96c9..7c868414d143 100644--- a/products/endpoints/backend/models.py+++ b/products/endpoints/backend/models.py@@ -22,6 +22,8 @@ from posthog.models.user import Userfrom posthog.models.utils import CreatedMetaFields, DeletedMetaFields, UpdatedMetaFields, UUIDTModelfrom posthog.schema_enums import ProductKey+from products.data_modeling.backend.facade.api import declared_targets_by_saved_query, get_declared_target+logger = logging.getLogger(__name__)@@ -199,9 +201,10 @@ class EndpointVersion(UpdatedMetaFields, models.Model):related_name="endpoint_versions_created",)- data_freshness_seconds = models.IntegerField(+ cache_ttl_seconds = models.IntegerField(default=86400,- help_text="How fresh the data should be, in seconds. Controls cache TTL and materialization sync frequency.",+ db_column="data_freshness_seconds",+ help_text="How long direct query results remain in the cache, in seconds.",)saved_query = models.ForeignKey("data_modeling.DataWarehouseSavedQuery",@@ -289,14 +292,45 @@ class EndpointVersion(UpdatedMetaFields, models.Model):self.bucket_overrides = bucket_overridesself.save(update_fields=["saved_query", "bucket_overrides", "updated_at"])- def disable_materialization(self) -> None:+ # Attribute that `frequency_nodes_prefetch` fills and `data_freshness_seconds` reads.+ FREQUENCY_NODES_ATTR = "prefetched_frequency_nodes"++ @classmethod+ def frequency_nodes_prefetch(cls, saved_query_path: str = "saved_query") -> models.Prefetch:+ """Prefetch the nodes holding the declared freshness target, for use with `data_freshness_seconds`."""+ from products.data_modeling.backend.facade.models import Node++ return models.Prefetch(+ f"{saved_query_path}__node_set",+ queryset=Node.objects.only("saved_query_id", "properties"),+ to_attr=cls.FREQUENCY_NODES_ATTR,+ )++ @property+ def data_freshness_seconds(self) -> int:+ if self.saved_query_id is None:+ return self.cache_ttl_seconds++ nodes = getattr(self.saved_query, self.FREQUENCY_NODES_ATTR, None)+ if nodes is None:+ target = declared_targets_by_saved_query(self.team_id or self.endpoint.team_id, [self.saved_query_id]).get(+ str(self.saved_query_id)+ )+ else:+ target = next((target for node in nodes if (target := get_declared_target(node)) is not None), None)+ if target is not None:+ return int(target.total_seconds())+ return self.cache_ttl_seconds++ def disable_materialization(self, *, cache_ttl_seconds: int | None = None) -> None:"""Disable materialization: revert and soft-delete the saved query, clear version fields."""if not self.saved_query:return+ self.cache_ttl_seconds = self.data_freshness_seconds if cache_ttl_seconds is None else cache_ttl_secondsself.saved_query.revert_materialization()self.saved_query.soft_delete()self.saved_query = None- self.save(update_fields=["saved_query", "updated_at"])+ self.save(update_fields=["saved_query", "cache_ttl_seconds", "updated_at"])@propertydef is_materialized(self) -> bool:@@ -360,7 +394,7 @@ class Endpoint(Taggable, CreatedMetaFields, UpdatedMetaFields, DeletedMetaFieldsEndpoints allow creating reusable query endpoints like:/api/projects/{team_id}/endpoints/{endpoint_name}/run- Query, description, data_freshness_seconds, and materialization settings are stored+ Query, description, cache duration, and materialization settings are storedin EndpointVersion, allowing per-version configuration."""@@ -457,7 +491,7 @@ class Endpoint(Taggable, CreatedMetaFields, UpdatedMetaFields, DeletedMetaFieldsversion=self.current_version,query=query,created_by=user,- data_freshness_seconds=previous_data_freshness,+ cache_ttl_seconds=previous_data_freshness,description=previous_description,optional_breakdown_properties=pruned_optional_breakdowns,columns=columns,diff --git a/products/endpoints/backend/presentation/serializers.py b/products/endpoints/backend/presentation/serializers.pyindex 589d0911c46b..92cd3cb473ec 100644--- a/products/endpoints/backend/presentation/serializers.py+++ b/products/endpoints/backend/presentation/serializers.py@@ -152,7 +152,10 @@ class EndpointResponseSerializer(serializers.Serializer):help_text="Whether the endpoint can be executed via the API.",)data_freshness_seconds = serializers.IntegerField(- help_text="How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800.",+ help_text=(+ "Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, "+ "which can differ from the standard freshness choices. Other endpoints use the cache duration."+ ),)endpoint_path = serializers.CharField(help_text="Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run).",diff --git a/products/endpoints/backend/presentation/views/api.py b/products/endpoints/backend/presentation/views/api.pyindex 2e75eb5298df..e94588ffb23c 100644--- a/products/endpoints/backend/presentation/views/api.py+++ b/products/endpoints/backend/presentation/views/api.py@@ -336,6 +336,7 @@ class EndpointViewSet(latest_jobs = DataModelingJob.objects.filter(engine=DataModelingJob.Engine.CLICKHOUSE).order_by("-last_run_at")latest_completed_jobs = latest_jobs.filter(status=DataModelingJob.Status.COMPLETED)return queryset.select_related("saved_query", "team").prefetch_related(+ EndpointVersion.frequency_nodes_prefetch(),Prefetch("saved_query__datamodelingjob_set", queryset=latest_jobs[:1], to_attr="prefetched_latest_jobs"),Prefetch("saved_query__datamodelingjob_set",diff --git a/products/endpoints/backend/signals.py b/products/endpoints/backend/signals.pynew file mode 100644index 000000000000..eef1fa81b84f--- /dev/null+++ b/products/endpoints/backend/signals.py@@ -0,0 +1,32 @@+from django.db import transaction+from django.db.models.signals import post_delete, post_save+from django.dispatch import receiver++from products.data_modeling.backend.facade.api import endpoint_link+from products.data_modeling.backend.facade.models import Node, NodeType+from products.endpoints.backend.models import EndpointVersion+++def _clear_endpoint_cache(team_id: int, name: str, version: int) -> None:+ # The rate limiter imports query strategies, which must stay out of Django startup.+ from products.endpoints.backend.rate_limit import clear_endpoint_materialization_cache # noqa: PLC0415++ clear_endpoint_materialization_cache(team_id, name, versions=[version])+++@receiver(post_save, sender=Node)+@receiver(post_delete, sender=Node)+def invalidate_endpoint_node_cache(sender: type[Node], instance: Node, **kwargs: object) -> None:+ link = endpoint_link(instance.properties)+ if link is not None:+ # robust: a Redis failure must not skip the schedule reconciliation callbacks registered after this one.+ transaction.on_commit(lambda: _clear_endpoint_cache(instance.team_id, link.name, link.version), robust=True)+ elif instance.type == NodeType.ENDPOINT and instance.saved_query_id is not None:++ def clear_legacy_endpoint_cache() -> None:+ for name, version in EndpointVersion.objects.filter(+ endpoint__team_id=instance.team_id, saved_query_id=instance.saved_query_id+ ).values_list("endpoint__name", "version"):+ _clear_endpoint_cache(instance.team_id, name, version)++ transaction.on_commit(clear_legacy_endpoint_cache, robust=True)diff --git a/products/endpoints/backend/tests/conftest.py b/products/endpoints/backend/tests/conftest.pyindex c508355d0479..19c97a1ba957 100644--- a/products/endpoints/backend/tests/conftest.py+++ b/products/endpoints/backend/tests/conftest.py@@ -36,7 +36,7 @@ def create_endpoint_with_version(version=current_version,query=query,description=description,- data_freshness_seconds=data_freshness_seconds,+ cache_ttl_seconds=data_freshness_seconds,created_by=created_by,)return endpointdiff --git a/products/endpoints/backend/tests/test_endpoint.py b/products/endpoints/backend/tests/test_endpoint.pyindex 4e4bfcce639d..f27819e12937 100644--- a/products/endpoints/backend/tests/test_endpoint.py+++ b/products/endpoints/backend/tests/test_endpoint.py@@ -1253,9 +1253,10 @@ class TestEndpoint(ClickhouseTestMixin, APIBaseTest):# Update to different data freshnessupdated_data: dict[str, int | None] = {"data_freshness_seconds": 21600}- response = self.client.patch(- f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", updated_data, format="json"- )+ with self.captureOnCommitCallbacks(execute=True):+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", updated_data, format="json"+ )self.assertEqual(response.status_code, status.HTTP_200_OK)self.assertEqual(response.json()["data_freshness_seconds"], 21600)# The freshness target is part of the throttle's cached snapshot, so the edit drops it.@@ -1689,11 +1690,13 @@ class TestMaterializationPreview(ClickhouseTestMixin, APIBaseTest):# Change bucket_overrides — should trigger an immediate refreshwith mock.patch("products.endpoints.backend.logic.crud.materialize_saved_query") as mock_trigger:- response = self.client.patch(- f"/api/environments/{self.team.id}/endpoints/trigger-bucket/",- {"bucket_overrides": {"timestamp": "hour"}},- format="json",- )+ with self.captureOnCommitCallbacks(execute=True):+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/trigger-bucket/",+ {"bucket_overrides": {"timestamp": "hour"}},+ format="json",+ )+ mock_trigger.assert_not_called()assert response.status_code == status.HTTP_200_OK, response.json()version = EndpointVersion.objects.get(endpoint__name="trigger-bucket", endpoint__team=self.team, version=1)assert version.bucket_overrides == {"timestamp": "hour"}diff --git a/products/endpoints/backend/tests/test_endpoint_materialization.py b/products/endpoints/backend/tests/test_endpoint_materialization.pyindex 26511013f99e..95803ad590b9 100644--- a/products/endpoints/backend/tests/test_endpoint_materialization.py+++ b/products/endpoints/backend/tests/test_endpoint_materialization.py@@ -26,7 +26,11 @@ from posthog.models.team.extensions import get_or_create_team_extensionfrom posthog.models.team.team_revenue_analytics_config import TeamRevenueAnalyticsConfigfrom posthog.sync import database_sync_to_async-from products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, get_declared_target+from products.data_modeling.backend.facade.api import (+ UnsatisfiableFrequencyError,+ get_declared_target,+ set_declared_target,+)from products.data_modeling.backend.facade.models import DAG, DataModelingJob, DataWarehouseSavedQuery, Nodefrom products.endpoints.backend.insight_transformers import transform_materialized_insight_responsefrom products.endpoints.backend.logic.execution import EndpointExecutionService@@ -134,11 +138,12 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):set_endpoint_materialization_ready(self.team.pk, endpoint.name, False)set_endpoint_materialization_ready(self.team.pk, endpoint.name, False, version=1)- response = self.client.patch(- f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",- {"is_materialized": True},- format="json",- )+ with self.captureOnCommitCallbacks(execute=True):+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True},+ format="json",+ )self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())# A snapshot cached before the enable would hold the inline rate until it expired.@@ -216,6 +221,213 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):DataWarehouseSavedQuery.objects.filter(team=self.team, name=f"{endpoint.name}_v{version.version}").exists())+ def test_refused_freshness_change_keeps_the_previous_freshness(self):+ endpoint = create_endpoint_with_version(+ name="refused_freshness_endpoint",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ is_active=True,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True, "data_freshness_seconds": 86400},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())++ with mock.patch.object(+ DataWarehouseSavedQuery,+ "schedule_materialization",+ side_effect=UnsatisfiableFrequencyError("15min is finer than the 24h upstream source"),+ ):+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"data_freshness_seconds": 900},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())+ version = EndpointVersion.objects.get(endpoint=endpoint, version=1)+ assert version.saved_query is not None+ self.assertEqual(version.data_freshness_seconds, 86400)+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), timedelta(hours=24))++ @parameterized.expand([(86400,), (2592000,)])+ def test_node_target_controls_api_freshness_and_metadata_updates(self, node_target_seconds):+ endpoint = create_endpoint_with_version(+ name="seeded_daily_endpoint",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ is_active=True,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True, "data_freshness_seconds": 21600},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ version = EndpointVersion.objects.get(endpoint=endpoint, version=1)+ node_target = timedelta(seconds=node_target_seconds)+ set_declared_target(Node.objects.get(saved_query=version.saved_query), node_target)++ response = self.client.get(f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/")+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"description": "Updated description"},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), node_target)++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"data_freshness_seconds": node_target_seconds},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)++ def test_materialized_freshness_update_does_not_write_the_cache_ttl(self):+ endpoint = create_endpoint_with_version(+ name="node_freshness_endpoint",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True, "data_freshness_seconds": 21600},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ version = endpoint.get_version()+ previous_cache_ttl = version.cache_ttl_seconds++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"data_freshness_seconds": 43200},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], 43200)+ version.refresh_from_db()+ self.assertEqual(version.cache_ttl_seconds, previous_cache_ttl)+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), timedelta(hours=12))+ activity = ActivityLog.objects.filter(+ team_id=self.team.pk, item_id=str(endpoint.id), scope="EndpointVersion", activity="version_updated"+ ).latest("created_at")+ assert activity.detail is not None+ self.assertIn(+ {+ "type": "EndpointVersion",+ "action": "changed",+ "field": "data_freshness_seconds",+ "before": 21600,+ "after": 43200,+ },+ activity.detail["changes"],+ )++ @parameterized.expand(+ [+ (None, 43200, {"is_materialized": False}),+ (900, 900, {"is_materialized": False}),+ (900, 900, {"is_active": False, "version": 1}),+ ]+ )+ def test_disabling_materialization_preserves_or_updates_freshness(+ self, requested_freshness, expected_freshness, update+ ):+ endpoint = create_endpoint_with_version(+ name="disabled_node_freshness",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ version = endpoint.get_version()+ set_declared_target(Node.objects.get(saved_query=version.saved_query), timedelta(hours=12))+ update = dict(update)+ if requested_freshness is not None:+ update["data_freshness_seconds"] = requested_freshness++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", update, format="json"+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], expected_freshness)+ version.refresh_from_db()+ self.assertIsNone(version.saved_query_id)+ self.assertEqual(version.cache_ttl_seconds, expected_freshness)++ def test_new_version_inherits_the_node_target(self):+ endpoint = create_endpoint_with_version(+ name="inherited_node_freshness",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ version = endpoint.get_version()+ set_declared_target(Node.objects.get(saved_query=version.saved_query), timedelta(hours=12))++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"query": {"kind": "HogQLQuery", "query": "SELECT count() FROM events"}},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ self.assertEqual(response.json()["data_freshness_seconds"], 43200)+ new_version = endpoint.get_version()+ self.assertEqual(new_version.version, 2)+ self.assertEqual(new_version.data_freshness_seconds, 43200)++ @parameterized.expand([(True,), (False,)])+ def test_node_target_change_invalidates_readiness_after_commit(self, has_endpoint_link):+ endpoint = create_endpoint_with_version(+ name="node_freshness_cache",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ version = endpoint.get_version()+ node = Node.objects.get(saved_query=version.saved_query)+ if not has_endpoint_link:+ node.properties.pop("endpoint", None)+ node.save(update_fields=["properties"])+ set_endpoint_materialization_ready(self.team.pk, endpoint.name, True)+ set_endpoint_materialization_ready(self.team.pk, endpoint.name, True, version=1)++ with self.captureOnCommitCallbacks(execute=True):+ set_declared_target(node, timedelta(hours=12))+ self.assertTrue(is_endpoint_materialization_ready(self.team.pk, endpoint.name))+ self.assertTrue(is_endpoint_materialization_ready(self.team.pk, endpoint.name, version=1))++ self.assertIsNone(is_endpoint_materialization_ready(self.team.pk, endpoint.name))+ self.assertIsNone(is_endpoint_materialization_ready(self.team.pk, endpoint.name, version=1))+def test_enable_fails_as_a_request_error_when_the_dag_sync_leaves_no_node(self):# the endpoint path swallows a dag sync failure, and scheduling then disables itself rather# than raising, so without a check the enable reports success on an endpoint that nothing@@ -2425,6 +2637,32 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):self.assertIsNone(v1.saved_query_id, "deactivated version kept its materialization")self.assertTrue(DataWarehouseSavedQuery.objects.get(id=saved_query_id).deleted)+ def test_deactivating_endpoint_with_freshness_keeps_requested_cache_duration(self):+ endpoint = create_endpoint_with_version(+ name="endpoint-deactivate-freshness",+ team=self.team,+ query=self.sample_hogql_query,+ created_by=self.user,+ is_active=True,+ )+ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_materialized": True, "data_freshness_seconds": 21600},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())++ response = self.client.patch(+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",+ {"is_active": False, "data_freshness_seconds": 43200},+ format="json",+ )+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())++ version = endpoint.get_version()+ self.assertIsNone(version.saved_query_id)+ self.assertEqual(version.cache_ttl_seconds, 43200)+def test_deactivating_endpoint_disables_all_versions_materialization(self):"""Deactivating the endpoint must tear down every materialized version, not just the current one."""endpoint = create_endpoint_with_version(diff --git a/products/endpoints/backend/tests/test_rate_limit.py b/products/endpoints/backend/tests/test_rate_limit.pyindex fa5db66c8a6a..587eba076803 100644--- a/products/endpoints/backend/tests/test_rate_limit.py+++ b/products/endpoints/backend/tests/test_rate_limit.py@@ -339,7 +339,7 @@ def _create_ready_materialized_endpoint(team, user, name: str, materialized_at)query={"kind": "HogQLQuery", "query": "SELECT 1"},created_by=user,saved_query=saved_query,- data_freshness_seconds=3600,+ cache_ttl_seconds=3600,)@@ -605,7 +605,7 @@ class TestMaterializationStateCacheTimeout(APIBaseTest):query={"kind": "HogQLQuery", "query": "SELECT 1"},created_by=self.user,saved_query=saved_query,- data_freshness_seconds=3600,+ cache_ttl_seconds=3600,)with patch("products.endpoints.backend.rate_limit.cache.set", wraps=cache.set) as cache_set:diff --git a/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx b/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsxindex 35e0041c8fc5..42433414e712 100644--- a/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx+++ b/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx@@ -17,6 +17,7 @@ import {import { AccessControlAction } from 'lib/components/AccessControlAction'import { CodeSnippet, Language } from 'lib/components/CodeSnippet'import { LemonField } from 'lib/lemon-ui/LemonField'+import { humanFriendlyDuration } from 'lib/utils/durations'import { MaterializationStatusModal } from 'scenes/data-warehouse/saved_queries/MaterializationStatusModal'import { AccessControlLevel, AccessControlResourceType } from '~/types'@@ -96,6 +97,12 @@ export function EndpointConfiguration(): JSX.Element {const effectiveDataFreshness =dataFreshness ?? viewingVersion?.data_freshness_seconds ?? endpoint.data_freshness_seconds ?? 86400+ const dataFreshnessOptions = DATA_FRESHNESS_OPTIONS.some((option) => option.value === effectiveDataFreshness)+ ? DATA_FRESHNESS_OPTIONS+ : [+ ...DATA_FRESHNESS_OPTIONS,+ { value: effectiveDataFreshness, label: humanFriendlyDuration(effectiveDataFreshness) },+ ]const baseIsMaterialized = viewingVersion?.is_materialized ?? endpoint.is_materializedconst isMaterialized = localIsMaterialized ?? baseIsMaterializedconst materializationExpanded = leftActiveKeys.includes('materialization')@@ -148,7 +155,7 @@ export function EndpointConfiguration(): JSX.Element {<LemonSelectvalue={effectiveDataFreshness}onChange={setDataFreshness}- options={DATA_FRESHNESS_OPTIONS}+ options={dataFreshnessOptions}/></AccessControlAction></div>diff --git a/products/endpoints/frontend/generated/api.schemas.ts b/products/endpoints/frontend/generated/api.schemas.tsindex 74e97b28345e..5f5fa4541ab0 100644--- a/products/endpoints/frontend/generated/api.schemas.ts+++ b/products/endpoints/frontend/generated/api.schemas.ts@@ -130,7 +130,7 @@ export interface EndpointResponseApi {query: unknown/** Whether the endpoint can be executed via the API. */is_active: boolean- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */data_freshness_seconds: number/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */endpoint_path: string@@ -287,7 +287,7 @@ export interface EndpointVersionResponseApi {query: unknown/** Whether the endpoint can be executed via the API. */is_active: boolean- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */data_freshness_seconds: number/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */endpoint_path: stringdiff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 11e7cdeec398..1aed61ba28b7 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -42603,7 +42603,7 @@ export namespace Schemas {query: unknown;/** Whether the endpoint can be executed via the API. */is_active: boolean;- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */data_freshness_seconds: number;/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */endpoint_path: string;@@ -42738,7 +42738,7 @@ export namespace Schemas {query: unknown;/** Whether the endpoint can be executed via the API. */is_active: boolean;- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */data_freshness_seconds: number;/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */endpoint_path: string;