Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(endpoints): use node targets for materialized freshness (#123115)

Jovan Sakovicabae763405b6
diff --git a/posthog/test/repo_invariants/setup_receivers_baseline.txt b/posthog/test/repo_invariants/setup_receivers_baseline.txt
index 52609d5eff6b..8f622608f96e 100644
--- a/posthog/test/repo_invariants/setup_receivers_baseline.txt
+++ b/posthog/test/repo_invariants/setup_receivers_baseline.txt
@@ -103,6 +103,7 @@ post_delete:products.cdp.backend.models.plugin.plugin_reload_needed
post_delete:products.cohorts.backend.models.cohort.cohort_people_changed
post_delete:products.cohorts.backend.models.dependencies.clear_team_cohort_dependency_cache
post_delete:products.cohorts.backend.models.dependencies.cohort_deleted
+post_delete:products.endpoints.backend.signals.invalidate_endpoint_node_cache
post_delete:products.event_definitions.backend.models.event_definition._invalidate_default_event_info_on_delete
post_delete:products.experiments.backend.facade.change_signals._notify_experiment_deleted
post_delete:products.feature_flags.backend.flags_cache.cohort_changed_flags_cache
@@ -222,6 +223,7 @@ post_save:products.data_modeling.backend.signals.sync_saved_query_name_to_node
post_save:products.data_modeling.backend.signals.trigger_view_semantic_enrichment
post_save:products.data_warehouse.backend.models.revenue_analytics_config.create_external_data_source_revenue_analytics_config
post_save:products.early_access_features.backend.signals.create_waitlist_survey_on_concept_stage
+post_save:products.endpoints.backend.signals.invalidate_endpoint_node_cache
post_save:products.event_definitions.backend.models.event_definition._invalidate_default_event_info_on_save
post_save:products.event_definitions.backend.models.property_definition._invalidate_has_person_email_on_save
post_save:products.experiments.backend.facade.change_signals._notify_experiment_saved
diff --git a/products/endpoints/backend/admin.py b/products/endpoints/backend/admin.py
index b419f38b3c31..910b2a7016f6 100644
--- a/products/endpoints/backend/admin.py
+++ b/products/endpoints/backend/admin.py
@@ -17,7 +17,6 @@ class EndpointVersionInline(admin.TabularInline):
"version",
"is_active",
"query_kind",
- "data_freshness_seconds",
"is_materialized_display",
"saved_query_name",
"saved_query_status",
@@ -234,6 +233,8 @@ class EndpointVersionAdmin(admin.ModelAdmin):
super()
.get_queryset(request)
.select_related("endpoint", "endpoint__team", "saved_query", "saved_query__table", "created_by")
+ # data_freshness_seconds reads the node target, so prefetch it once for the whole page.
+ .prefetch_related(EndpointVersion.frequency_nodes_prefetch())
)
@admin.display(description="Endpoint")
diff --git a/products/endpoints/backend/apps.py b/products/endpoints/backend/apps.py
index ae9b2536dacd..d1512add583b 100644
--- a/products/endpoints/backend/apps.py
+++ b/products/endpoints/backend/apps.py
@@ -6,3 +6,6 @@ class EndpointsConfig(AppConfig):
name = "products.endpoints.backend"
label = "endpoints"
verbose_name = "Endpoints (product)"
+
+ def ready(self) -> None:
+ from products.endpoints.backend import signals # noqa: F401
diff --git a/products/endpoints/backend/logic/crud.py b/products/endpoints/backend/logic/crud.py
index e8102c1b8d78..327edc1c14c2 100644
--- a/products/endpoints/backend/logic/crud.py
+++ b/products/endpoints/backend/logic/crud.py
@@ -118,7 +118,7 @@ class EndpointCrudService:
version=1,
query=query_dict,
description=data.description or "",
- data_freshness_seconds=(
+ cache_ttl_seconds=(
data.data_freshness_seconds
if data.data_freshness_seconds is not None
else DEFAULT_DATA_FRESHNESS_SECONDS
@@ -199,6 +199,7 @@ class EndpointCrudService:
# get_version raises when no versions exist, so target_version is never None.
target_version = target_version_override or current_version
version_before_update = EndpointVersion.objects.get(pk=target_version.pk)
+ previous_data_freshness_seconds = version_before_update.data_freshness_seconds
was_materialized = current_version.saved_query_id is not None
# Before any write, so a refused update leaves the endpoint as it was.
@@ -216,6 +217,9 @@ class EndpointCrudService:
# version, not just the current one.
for materialized_version in endpoint.versions.filter(saved_query__isnull=False):
self.materialization.disable_materialization(endpoint, materialized_version)
+ # target_version was loaded before the teardown and still points at the removed saved
+ # query, which would make the freshness write below skip the direct query cache duration.
+ target_version.refresh_from_db()
if data.is_active is not None and not version_targeted:
# Activation affects throttle classification — force a lazy re-check.
clear_endpoint_materialization_cache(self.team.pk, endpoint.name)
@@ -225,25 +229,32 @@ class EndpointCrudService:
endpoint, data, target_version, was_materialized
)
- step = "version_fields"
- self._apply_version_field_updates(endpoint, target_version, data, raw_data, version_targeted)
-
- step = "materialization"
- materialization_error = self._reconcile_materialization(
- endpoint,
- data,
- raw_data,
- target_version,
- version_targeted,
- was_materialized=was_materialized,
- version_was_created=version_was_created,
- old_bucket_overrides=old_bucket_overrides,
- )
+ # A rejected enable must also roll back changes to the direct query cache duration.
+ with transaction.atomic():
+ step = "version_fields"
+ self._apply_version_field_updates(endpoint, target_version, data, raw_data, version_targeted)
+
+ step = "materialization"
+ materialization_error = self._reconcile_materialization(
+ endpoint,
+ data,
+ raw_data,
+ target_version,
+ version_targeted,
+ was_materialized=was_materialized,
+ version_was_created=version_was_created,
+ old_bucket_overrides=old_bucket_overrides,
+ )
step = "tags_and_activity"
apply_tags(endpoint, data.tags)
self._log_update_activity(
- endpoint, endpoint_before_update, target_version, version_before_update, version_was_created
+ endpoint,
+ endpoint_before_update,
+ target_version,
+ version_before_update,
+ version_was_created,
+ previous_data_freshness_seconds,
)
return EndpointUpdateResult(
@@ -344,13 +355,13 @@ class EndpointCrudService:
if data.description is not None:
target_version.description = data.description
update_fields.append("description")
- if "data_freshness_seconds" in raw_data:
- target_version.data_freshness_seconds = (
+ if "data_freshness_seconds" in raw_data and target_version.saved_query_id is None:
+ target_version.cache_ttl_seconds = (
data.data_freshness_seconds
if data.data_freshness_seconds is not None
else DEFAULT_DATA_FRESHNESS_SECONDS
)
- update_fields.append("data_freshness_seconds")
+ update_fields.append("cache_ttl_seconds")
if "optional_breakdown_properties" in raw_data:
target_version.optional_breakdown_properties = list(data.optional_breakdown_properties or [])
update_fields.append("optional_breakdown_properties")
@@ -361,9 +372,14 @@ class EndpointCrudService:
if update_fields:
update_fields.append("updated_at")
target_version.save(update_fields=update_fields)
- if {"data_freshness_seconds", "optional_breakdown_properties"} & set(update_fields):
+ if {"cache_ttl_seconds", "optional_breakdown_properties"} & set(update_fields):
# Both fields are part of the cached serving snapshot the throttle classifies from.
- clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[target_version.version])
+ transaction.on_commit(
+ lambda: clear_endpoint_materialization_cache(
+ self.team.pk, endpoint.name, versions=[target_version.version]
+ ),
+ robust=True,
+ )
def _reconcile_materialization(
self,
@@ -382,11 +398,18 @@ class EndpointCrudService:
Returns a materialization error message when enabling failed after a new
version was already committed (the update itself still succeeds).
"""
+ requested_freshness = (
+ (data.data_freshness_seconds if data.data_freshness_seconds is not None else DEFAULT_DATA_FRESHNESS_SECONDS)
+ if "data_freshness_seconds" in raw_data
+ else None
+ )
if version_targeted and not target_version.is_active:
# Deactivating a version: tear down its own materialization, never enable.
# Checked before the endpoint-active guard so this holds even on an inactive endpoint.
if target_version.saved_query_id is not None:
- self.materialization.disable_materialization(endpoint, target_version)
+ self.materialization.disable_materialization(
+ endpoint, target_version, cache_ttl_seconds=requested_freshness
+ )
return None
if not endpoint.is_active:
@@ -399,7 +422,9 @@ class EndpointCrudService:
should_enable = data.is_materialized is True or (data.is_materialized is None and check_was_materialized)
if data.is_materialized is False:
- self.materialization.disable_materialization(endpoint, target_version)
+ self.materialization.disable_materialization(
+ endpoint, target_version, cache_ttl_seconds=requested_freshness
+ )
return None
if not should_enable:
return None
@@ -414,7 +439,7 @@ class EndpointCrudService:
self.materialization.enable_materialization(
endpoint,
target_version,
- target_version.data_freshness_seconds,
+ requested_freshness if requested_freshness is not None else target_version.data_freshness_seconds,
bucket_overrides=bucket_overrides,
)
if (
@@ -424,16 +449,21 @@ class EndpointCrudService:
):
# enable only triggers an immediate Temporal run when it creates the schedule;
# changed bucketing on an existing materialization needs an explicit refresh.
- try:
- materialize_saved_query(target_version.saved_query)
- except Exception:
- logger.warning(
- "failed_to_trigger_materialization_refresh",
- team_id=self.team.pk,
- endpoint_name=endpoint.name,
- saved_query_id=str(target_version.saved_query_id),
- bucket_overrides=bucket_overrides,
- )
+ saved_query = target_version.saved_query
+
+ def refresh_materialization() -> None:
+ try:
+ materialize_saved_query(saved_query)
+ except Exception:
+ logger.warning(
+ "failed_to_trigger_materialization_refresh",
+ team_id=self.team.pk,
+ endpoint_name=endpoint.name,
+ saved_query_id=str(saved_query.id),
+ bucket_overrides=bucket_overrides,
+ )
+
+ transaction.on_commit(refresh_materialization)
except Exception as e:
if not version_was_created:
raise
@@ -463,6 +493,7 @@ class EndpointCrudService:
target_version: EndpointVersion,
version_before_update: EndpointVersion,
version_was_created: bool,
+ previous_data_freshness_seconds: int,
) -> None:
endpoint_changes = changes_between("Endpoint", previous=endpoint_before_update, current=endpoint)
if endpoint_changes:
@@ -492,7 +523,22 @@ class EndpointCrudService:
),
)
else:
- version_changes = changes_between("EndpointVersion", previous=version_before_update, current=target_version)
+ version_changes = [
+ change
+ for change in changes_between("EndpointVersion", previous=version_before_update, current=target_version)
+ if change.field != "cache_ttl_seconds"
+ ]
+ data_freshness_seconds = target_version.data_freshness_seconds
+ if data_freshness_seconds != previous_data_freshness_seconds:
+ version_changes.append(
+ Change(
+ type="EndpointVersion",
+ action="changed",
+ field="data_freshness_seconds",
+ before=previous_data_freshness_seconds,
+ after=data_freshness_seconds,
+ )
+ )
if version_changes:
self._log_activity(
item_id=str(endpoint.id),
diff --git a/products/endpoints/backend/logic/execution.py b/products/endpoints/backend/logic/execution.py
index a7af4a078cd1..7a816ddf2ac5 100644
--- a/products/endpoints/backend/logic/execution.py
+++ b/products/endpoints/backend/logic/execution.py
@@ -794,6 +794,7 @@ class EndpointExecutionService(PydanticModelMixin):
try:
if not saved_query:
raise ValidationError("No materialized query found for this endpoint")
+ data_freshness_seconds = version.data_freshness_seconds
strategy = strategy_for(endpoint, version, self.team)
query_kind = strategy.query_kind
@@ -835,16 +836,16 @@ class EndpointExecutionService(PydanticModelMixin):
):
# Compute dynamic cache TTL: time remaining until data_freshness window expires
cache_ttl = None
- if materialized_at and version.data_freshness_seconds:
+ if materialized_at and data_freshness_seconds:
remaining = (
- materialized_at + timedelta(seconds=version.data_freshness_seconds) - timezone.now()
+ materialized_at + timedelta(seconds=data_freshness_seconds) - timezone.now()
).total_seconds()
if remaining <= 0:
logger.warning(
"endpoint_materialization_behind_sla",
endpoint_name=endpoint.name,
team_id=self.team.pk,
- data_freshness_seconds=version.data_freshness_seconds,
+ data_freshness_seconds=data_freshness_seconds,
last_run_at=materialized_at.isoformat(),
remaining_seconds=remaining,
)
@@ -903,9 +904,9 @@ class EndpointExecutionService(PydanticModelMixin):
# Freshness relative to the configured target: >1.0 means behind SLA.
# Absolute age is meaningless across endpoints with different frequencies.
- if materialized_at and version.data_freshness_seconds:
+ if materialized_at and data_freshness_seconds:
age_seconds = max((timezone.now() - materialized_at).total_seconds(), 0.0)
- ENDPOINT_MATERIALIZED_FRESHNESS_RATIO.observe(age_seconds / version.data_freshness_seconds)
+ ENDPOINT_MATERIALIZED_FRESHNESS_RATIO.observe(age_seconds / data_freshness_seconds)
return result
except Exception as e:
diff --git a/products/endpoints/backend/logic/materialization.py b/products/endpoints/backend/logic/materialization.py
index 1019fc98f70c..7a8a70d87fbe 100644
--- a/products/endpoints/backend/logic/materialization.py
+++ b/products/endpoints/backend/logic/materialization.py
@@ -7,6 +7,7 @@ query transforms live in ``products.endpoints.backend.materialization_transforms
"""
import dataclasses
+from datetime import timedelta
from typing import cast
from django.db import DatabaseError, transaction
@@ -39,7 +40,6 @@ from products.data_modeling.backend.facade.api import (
sync_saved_query_to_dag,
)
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
-from products.endpoints.backend.constants import DATA_FRESHNESS_BUCKETS
from products.endpoints.backend.logic.activity import EndpointContext
from products.endpoints.backend.logic.strategies import apply_where_filter, strategy_for
from products.endpoints.backend.materialization_transforms import (
@@ -54,7 +54,6 @@ from products.endpoints.backend.materialization_transforms import (
from products.endpoints.backend.metrics import ENDPOINT_MATERIALIZATION_EVENT_TOTAL
from products.endpoints.backend.models import Endpoint, EndpointVersion
from products.endpoints.backend.rate_limit import clear_endpoint_materialization_cache
-from products.warehouse_sources.backend.facade.models import sync_frequency_to_sync_frequency_interval
logger = structlog.get_logger(__name__)
@@ -177,7 +176,10 @@ class EndpointMaterializationService:
self._enable_materialization_inner(endpoint, version, data_freshness_seconds, bucket_overrides)
# The throttle may hold a not-ready snapshot cached before the enable. Drop it so the
# first completed run is picked up on the next request, not when the entry expires.
- clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version])
+ transaction.on_commit(
+ lambda: clear_endpoint_materialization_cache(self.team.pk, endpoint.name, versions=[version.version]),
+ robust=True,
+ )
ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="enable", status="success").inc()
if version.saved_query:
self._log_materialization_activity(
@@ -340,18 +342,18 @@ class EndpointMaterializationService:
)
saved_query.external_tables = saved_query.s3_tables
saved_query.is_materialized = True
- saved_query.sync_frequency_interval = sync_frequency_to_sync_frequency_interval(
- DATA_FRESHNESS_BUCKETS[data_freshness_seconds]
- )
+ saved_query.sync_frequency_interval = timedelta(seconds=data_freshness_seconds)
saved_query.save()
- def disable_materialization(self, endpoint: Endpoint, version: EndpointVersion) -> None:
+ def disable_materialization(
+ self, endpoint: Endpoint, version: EndpointVersion, *, cache_ttl_seconds: int | None = None
+ ) -> None:
"""Disable materialization for an endpoint version."""
if version.saved_query:
saved_query_id = str(version.saved_query.id)
saved_query_name = version.saved_query.name
try:
- version.disable_materialization()
+ version.disable_materialization(cache_ttl_seconds=cache_ttl_seconds)
except Exception:
ENDPOINT_MATERIALIZATION_EVENT_TOTAL.labels(action="disable", status="error").inc()
raise
diff --git a/products/endpoints/backend/logic/validation.py b/products/endpoints/backend/logic/validation.py
index e88dfb1cc972..8a6b83536081 100644
--- a/products/endpoints/backend/logic/validation.py
+++ b/products/endpoints/backend/logic/validation.py
@@ -54,6 +54,18 @@ def validate_data_freshness(data_freshness_seconds: int | None) -> None:
)
+def _is_current_data_freshness(
+ data_freshness_seconds: int | None, endpoint: Endpoint | None, version_number: int | None
+) -> bool:
+ """True when the value equals what the API already reports for the version being updated."""
+ if data_freshness_seconds is None or endpoint is None:
+ return False
+ try:
+ return endpoint.get_version(version_number).data_freshness_seconds == data_freshness_seconds
+ except EndpointVersion.DoesNotExist:
+ return False
+
+
def validate_bucket_overrides(bucket_overrides: dict[str, str] | None) -> None:
"""Raise ValidationError if any bucket override value is not in SUPPORTED_BUCKET_FUNCTIONS."""
if not bucket_overrides:
@@ -310,7 +322,8 @@ def validate_update_request(
version_number: int | None = None,
) -> None:
"""Validate an update payload against the endpoint's resulting state."""
- validate_data_freshness(data.data_freshness_seconds)
+ if not _is_current_data_freshness(data.data_freshness_seconds, endpoint, version_number):
+ validate_data_freshness(data.data_freshness_seconds)
# Determine final states after this request (for validation)
will_be_active = data.is_active if data.is_active is not None else (endpoint.is_active if endpoint else True)
diff --git a/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.py b/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.py
new file mode 100644
index 000000000000..883167114721
--- /dev/null
+++ b/products/endpoints/backend/migrations/0035_endpoint_cache_ttl.py
@@ -0,0 +1,28 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("endpoints", "0034_drop_orphaned_endpoint_saved_query_fk"),
+ ]
+
+ operations = [
+ migrations.SeparateDatabaseAndState(
+ state_operations=[
+ migrations.AlterField(
+ model_name="endpointversion",
+ name="data_freshness_seconds",
+ field=models.IntegerField(
+ db_column="data_freshness_seconds",
+ default=86400,
+ help_text="How long direct query results remain in the cache, in seconds.",
+ ),
+ ),
+ migrations.RenameField(
+ model_name="endpointversion",
+ old_name="data_freshness_seconds",
+ new_name="cache_ttl_seconds",
+ ),
+ ],
+ ),
+ ]
diff --git a/products/endpoints/backend/migrations/max_migration.txt b/products/endpoints/backend/migrations/max_migration.txt
index c367bab5367e..0cab9717b391 100644
--- a/products/endpoints/backend/migrations/max_migration.txt
+++ b/products/endpoints/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0034_drop_orphaned_endpoint_saved_query_fk
+0035_endpoint_cache_ttl
diff --git a/products/endpoints/backend/models.py b/products/endpoints/backend/models.py
index 0b64dcab96c9..7c868414d143 100644
--- a/products/endpoints/backend/models.py
+++ b/products/endpoints/backend/models.py
@@ -22,6 +22,8 @@ from posthog.models.user import User
from posthog.models.utils import CreatedMetaFields, DeletedMetaFields, UpdatedMetaFields, UUIDTModel
from posthog.schema_enums import ProductKey
+from products.data_modeling.backend.facade.api import declared_targets_by_saved_query, get_declared_target
+
logger = logging.getLogger(__name__)
@@ -199,9 +201,10 @@ class EndpointVersion(UpdatedMetaFields, models.Model):
related_name="endpoint_versions_created",
)
- data_freshness_seconds = models.IntegerField(
+ cache_ttl_seconds = models.IntegerField(
default=86400,
- help_text="How fresh the data should be, in seconds. Controls cache TTL and materialization sync frequency.",
+ db_column="data_freshness_seconds",
+ help_text="How long direct query results remain in the cache, in seconds.",
)
saved_query = models.ForeignKey(
"data_modeling.DataWarehouseSavedQuery",
@@ -289,14 +292,45 @@ class EndpointVersion(UpdatedMetaFields, models.Model):
self.bucket_overrides = bucket_overrides
self.save(update_fields=["saved_query", "bucket_overrides", "updated_at"])
- def disable_materialization(self) -> None:
+ # Attribute that `frequency_nodes_prefetch` fills and `data_freshness_seconds` reads.
+ FREQUENCY_NODES_ATTR = "prefetched_frequency_nodes"
+
+ @classmethod
+ def frequency_nodes_prefetch(cls, saved_query_path: str = "saved_query") -> models.Prefetch:
+ """Prefetch the nodes holding the declared freshness target, for use with `data_freshness_seconds`."""
+ from products.data_modeling.backend.facade.models import Node
+
+ return models.Prefetch(
+ f"{saved_query_path}__node_set",
+ queryset=Node.objects.only("saved_query_id", "properties"),
+ to_attr=cls.FREQUENCY_NODES_ATTR,
+ )
+
+ @property
+ def data_freshness_seconds(self) -> int:
+ if self.saved_query_id is None:
+ return self.cache_ttl_seconds
+
+ nodes = getattr(self.saved_query, self.FREQUENCY_NODES_ATTR, None)
+ if nodes is None:
+ target = declared_targets_by_saved_query(self.team_id or self.endpoint.team_id, [self.saved_query_id]).get(
+ str(self.saved_query_id)
+ )
+ else:
+ target = next((target for node in nodes if (target := get_declared_target(node)) is not None), None)
+ if target is not None:
+ return int(target.total_seconds())
+ return self.cache_ttl_seconds
+
+ def disable_materialization(self, *, cache_ttl_seconds: int | None = None) -> None:
"""Disable materialization: revert and soft-delete the saved query, clear version fields."""
if not self.saved_query:
return
+ self.cache_ttl_seconds = self.data_freshness_seconds if cache_ttl_seconds is None else cache_ttl_seconds
self.saved_query.revert_materialization()
self.saved_query.soft_delete()
self.saved_query = None
- self.save(update_fields=["saved_query", "updated_at"])
+ self.save(update_fields=["saved_query", "cache_ttl_seconds", "updated_at"])
@property
def is_materialized(self) -> bool:
@@ -360,7 +394,7 @@ class Endpoint(Taggable, CreatedMetaFields, UpdatedMetaFields, DeletedMetaFields
Endpoints allow creating reusable query endpoints like:
/api/projects/{team_id}/endpoints/{endpoint_name}/run
- Query, description, data_freshness_seconds, and materialization settings are stored
+ Query, description, cache duration, and materialization settings are stored
in EndpointVersion, allowing per-version configuration.
"""
@@ -457,7 +491,7 @@ class Endpoint(Taggable, CreatedMetaFields, UpdatedMetaFields, DeletedMetaFields
version=self.current_version,
query=query,
created_by=user,
- data_freshness_seconds=previous_data_freshness,
+ cache_ttl_seconds=previous_data_freshness,
description=previous_description,
optional_breakdown_properties=pruned_optional_breakdowns,
columns=columns,
diff --git a/products/endpoints/backend/presentation/serializers.py b/products/endpoints/backend/presentation/serializers.py
index 589d0911c46b..92cd3cb473ec 100644
--- a/products/endpoints/backend/presentation/serializers.py
+++ b/products/endpoints/backend/presentation/serializers.py
@@ -152,7 +152,10 @@ class EndpointResponseSerializer(serializers.Serializer):
help_text="Whether the endpoint can be executed via the API.",
)
data_freshness_seconds = serializers.IntegerField(
- help_text="How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800.",
+ help_text=(
+ "Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, "
+ "which can differ from the standard freshness choices. Other endpoints use the cache duration."
+ ),
)
endpoint_path = serializers.CharField(
help_text="Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run).",
diff --git a/products/endpoints/backend/presentation/views/api.py b/products/endpoints/backend/presentation/views/api.py
index 2e75eb5298df..e94588ffb23c 100644
--- a/products/endpoints/backend/presentation/views/api.py
+++ b/products/endpoints/backend/presentation/views/api.py
@@ -336,6 +336,7 @@ class EndpointViewSet(
latest_jobs = DataModelingJob.objects.filter(engine=DataModelingJob.Engine.CLICKHOUSE).order_by("-last_run_at")
latest_completed_jobs = latest_jobs.filter(status=DataModelingJob.Status.COMPLETED)
return queryset.select_related("saved_query", "team").prefetch_related(
+ EndpointVersion.frequency_nodes_prefetch(),
Prefetch("saved_query__datamodelingjob_set", queryset=latest_jobs[:1], to_attr="prefetched_latest_jobs"),
Prefetch(
"saved_query__datamodelingjob_set",
diff --git a/products/endpoints/backend/signals.py b/products/endpoints/backend/signals.py
new file mode 100644
index 000000000000..eef1fa81b84f
--- /dev/null
+++ b/products/endpoints/backend/signals.py
@@ -0,0 +1,32 @@
+from django.db import transaction
+from django.db.models.signals import post_delete, post_save
+from django.dispatch import receiver
+
+from products.data_modeling.backend.facade.api import endpoint_link
+from products.data_modeling.backend.facade.models import Node, NodeType
+from products.endpoints.backend.models import EndpointVersion
+
+
+def _clear_endpoint_cache(team_id: int, name: str, version: int) -> None:
+ # The rate limiter imports query strategies, which must stay out of Django startup.
+ from products.endpoints.backend.rate_limit import clear_endpoint_materialization_cache # noqa: PLC0415
+
+ clear_endpoint_materialization_cache(team_id, name, versions=[version])
+
+
+@receiver(post_save, sender=Node)
+@receiver(post_delete, sender=Node)
+def invalidate_endpoint_node_cache(sender: type[Node], instance: Node, **kwargs: object) -> None:
+ link = endpoint_link(instance.properties)
+ if link is not None:
+ # robust: a Redis failure must not skip the schedule reconciliation callbacks registered after this one.
+ transaction.on_commit(lambda: _clear_endpoint_cache(instance.team_id, link.name, link.version), robust=True)
+ elif instance.type == NodeType.ENDPOINT and instance.saved_query_id is not None:
+
+ def clear_legacy_endpoint_cache() -> None:
+ for name, version in EndpointVersion.objects.filter(
+ endpoint__team_id=instance.team_id, saved_query_id=instance.saved_query_id
+ ).values_list("endpoint__name", "version"):
+ _clear_endpoint_cache(instance.team_id, name, version)
+
+ transaction.on_commit(clear_legacy_endpoint_cache, robust=True)
diff --git a/products/endpoints/backend/tests/conftest.py b/products/endpoints/backend/tests/conftest.py
index c508355d0479..19c97a1ba957 100644
--- a/products/endpoints/backend/tests/conftest.py
+++ b/products/endpoints/backend/tests/conftest.py
@@ -36,7 +36,7 @@ def create_endpoint_with_version(
version=current_version,
query=query,
description=description,
- data_freshness_seconds=data_freshness_seconds,
+ cache_ttl_seconds=data_freshness_seconds,
created_by=created_by,
)
return endpoint
diff --git a/products/endpoints/backend/tests/test_endpoint.py b/products/endpoints/backend/tests/test_endpoint.py
index 4e4bfcce639d..f27819e12937 100644
--- a/products/endpoints/backend/tests/test_endpoint.py
+++ b/products/endpoints/backend/tests/test_endpoint.py
@@ -1253,9 +1253,10 @@ class TestEndpoint(ClickhouseTestMixin, APIBaseTest):
# Update to different data freshness
updated_data: dict[str, int | None] = {"data_freshness_seconds": 21600}
- response = self.client.patch(
- f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", updated_data, format="json"
- )
+ with self.captureOnCommitCallbacks(execute=True):
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", updated_data, format="json"
+ )
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(response.json()["data_freshness_seconds"], 21600)
# The freshness target is part of the throttle's cached snapshot, so the edit drops it.
@@ -1689,11 +1690,13 @@ class TestMaterializationPreview(ClickhouseTestMixin, APIBaseTest):
# Change bucket_overrides — should trigger an immediate refresh
with mock.patch("products.endpoints.backend.logic.crud.materialize_saved_query") as mock_trigger:
- response = self.client.patch(
- f"/api/environments/{self.team.id}/endpoints/trigger-bucket/",
- {"bucket_overrides": {"timestamp": "hour"}},
- format="json",
- )
+ with self.captureOnCommitCallbacks(execute=True):
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/trigger-bucket/",
+ {"bucket_overrides": {"timestamp": "hour"}},
+ format="json",
+ )
+ mock_trigger.assert_not_called()
assert response.status_code == status.HTTP_200_OK, response.json()
version = EndpointVersion.objects.get(endpoint__name="trigger-bucket", endpoint__team=self.team, version=1)
assert version.bucket_overrides == {"timestamp": "hour"}
diff --git a/products/endpoints/backend/tests/test_endpoint_materialization.py b/products/endpoints/backend/tests/test_endpoint_materialization.py
index 26511013f99e..95803ad590b9 100644
--- a/products/endpoints/backend/tests/test_endpoint_materialization.py
+++ b/products/endpoints/backend/tests/test_endpoint_materialization.py
@@ -26,7 +26,11 @@ from posthog.models.team.extensions import get_or_create_team_extension
from posthog.models.team.team_revenue_analytics_config import TeamRevenueAnalyticsConfig
from posthog.sync import database_sync_to_async
-from products.data_modeling.backend.facade.api import UnsatisfiableFrequencyError, get_declared_target
+from products.data_modeling.backend.facade.api import (
+ UnsatisfiableFrequencyError,
+ get_declared_target,
+ set_declared_target,
+)
from products.data_modeling.backend.facade.models import DAG, DataModelingJob, DataWarehouseSavedQuery, Node
from products.endpoints.backend.insight_transformers import transform_materialized_insight_response
from products.endpoints.backend.logic.execution import EndpointExecutionService
@@ -134,11 +138,12 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):
set_endpoint_materialization_ready(self.team.pk, endpoint.name, False)
set_endpoint_materialization_ready(self.team.pk, endpoint.name, False, version=1)
- response = self.client.patch(
- f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
- {"is_materialized": True},
- format="json",
- )
+ with self.captureOnCommitCallbacks(execute=True):
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True},
+ format="json",
+ )
self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
# A snapshot cached before the enable would hold the inline rate until it expired.
@@ -216,6 +221,213 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):
DataWarehouseSavedQuery.objects.filter(team=self.team, name=f"{endpoint.name}_v{version.version}").exists()
)
+ def test_refused_freshness_change_keeps_the_previous_freshness(self):
+ endpoint = create_endpoint_with_version(
+ name="refused_freshness_endpoint",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ is_active=True,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True, "data_freshness_seconds": 86400},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+
+ with mock.patch.object(
+ DataWarehouseSavedQuery,
+ "schedule_materialization",
+ side_effect=UnsatisfiableFrequencyError("15min is finer than the 24h upstream source"),
+ ):
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"data_freshness_seconds": 900},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())
+ version = EndpointVersion.objects.get(endpoint=endpoint, version=1)
+ assert version.saved_query is not None
+ self.assertEqual(version.data_freshness_seconds, 86400)
+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), timedelta(hours=24))
+
+ @parameterized.expand([(86400,), (2592000,)])
+ def test_node_target_controls_api_freshness_and_metadata_updates(self, node_target_seconds):
+ endpoint = create_endpoint_with_version(
+ name="seeded_daily_endpoint",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ is_active=True,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True, "data_freshness_seconds": 21600},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ version = EndpointVersion.objects.get(endpoint=endpoint, version=1)
+ node_target = timedelta(seconds=node_target_seconds)
+ set_declared_target(Node.objects.get(saved_query=version.saved_query), node_target)
+
+ response = self.client.get(f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/")
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"description": "Updated description"},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)
+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), node_target)
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"data_freshness_seconds": node_target_seconds},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], node_target_seconds)
+
+ def test_materialized_freshness_update_does_not_write_the_cache_ttl(self):
+ endpoint = create_endpoint_with_version(
+ name="node_freshness_endpoint",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True, "data_freshness_seconds": 21600},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ version = endpoint.get_version()
+ previous_cache_ttl = version.cache_ttl_seconds
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"data_freshness_seconds": 43200},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], 43200)
+ version.refresh_from_db()
+ self.assertEqual(version.cache_ttl_seconds, previous_cache_ttl)
+ self.assertEqual(get_declared_target(Node.objects.get(saved_query=version.saved_query)), timedelta(hours=12))
+ activity = ActivityLog.objects.filter(
+ team_id=self.team.pk, item_id=str(endpoint.id), scope="EndpointVersion", activity="version_updated"
+ ).latest("created_at")
+ assert activity.detail is not None
+ self.assertIn(
+ {
+ "type": "EndpointVersion",
+ "action": "changed",
+ "field": "data_freshness_seconds",
+ "before": 21600,
+ "after": 43200,
+ },
+ activity.detail["changes"],
+ )
+
+ @parameterized.expand(
+ [
+ (None, 43200, {"is_materialized": False}),
+ (900, 900, {"is_materialized": False}),
+ (900, 900, {"is_active": False, "version": 1}),
+ ]
+ )
+ def test_disabling_materialization_preserves_or_updates_freshness(
+ self, requested_freshness, expected_freshness, update
+ ):
+ endpoint = create_endpoint_with_version(
+ name="disabled_node_freshness",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ version = endpoint.get_version()
+ set_declared_target(Node.objects.get(saved_query=version.saved_query), timedelta(hours=12))
+ update = dict(update)
+ if requested_freshness is not None:
+ update["data_freshness_seconds"] = requested_freshness
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/", update, format="json"
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], expected_freshness)
+ version.refresh_from_db()
+ self.assertIsNone(version.saved_query_id)
+ self.assertEqual(version.cache_ttl_seconds, expected_freshness)
+
+ def test_new_version_inherits_the_node_target(self):
+ endpoint = create_endpoint_with_version(
+ name="inherited_node_freshness",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ version = endpoint.get_version()
+ set_declared_target(Node.objects.get(saved_query=version.saved_query), timedelta(hours=12))
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"query": {"kind": "HogQLQuery", "query": "SELECT count() FROM events"}},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ self.assertEqual(response.json()["data_freshness_seconds"], 43200)
+ new_version = endpoint.get_version()
+ self.assertEqual(new_version.version, 2)
+ self.assertEqual(new_version.data_freshness_seconds, 43200)
+
+ @parameterized.expand([(True,), (False,)])
+ def test_node_target_change_invalidates_readiness_after_commit(self, has_endpoint_link):
+ endpoint = create_endpoint_with_version(
+ name="node_freshness_cache",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ version = endpoint.get_version()
+ node = Node.objects.get(saved_query=version.saved_query)
+ if not has_endpoint_link:
+ node.properties.pop("endpoint", None)
+ node.save(update_fields=["properties"])
+ set_endpoint_materialization_ready(self.team.pk, endpoint.name, True)
+ set_endpoint_materialization_ready(self.team.pk, endpoint.name, True, version=1)
+
+ with self.captureOnCommitCallbacks(execute=True):
+ set_declared_target(node, timedelta(hours=12))
+ self.assertTrue(is_endpoint_materialization_ready(self.team.pk, endpoint.name))
+ self.assertTrue(is_endpoint_materialization_ready(self.team.pk, endpoint.name, version=1))
+
+ self.assertIsNone(is_endpoint_materialization_ready(self.team.pk, endpoint.name))
+ self.assertIsNone(is_endpoint_materialization_ready(self.team.pk, endpoint.name, version=1))
+
def test_enable_fails_as_a_request_error_when_the_dag_sync_leaves_no_node(self):
# the endpoint path swallows a dag sync failure, and scheduling then disables itself rather
# than raising, so without a check the enable reports success on an endpoint that nothing
@@ -2425,6 +2637,32 @@ class TestEndpointMaterialization(ClickhouseTestMixin, APIBaseTest):
self.assertIsNone(v1.saved_query_id, "deactivated version kept its materialization")
self.assertTrue(DataWarehouseSavedQuery.objects.get(id=saved_query_id).deleted)
+ def test_deactivating_endpoint_with_freshness_keeps_requested_cache_duration(self):
+ endpoint = create_endpoint_with_version(
+ name="endpoint-deactivate-freshness",
+ team=self.team,
+ query=self.sample_hogql_query,
+ created_by=self.user,
+ is_active=True,
+ )
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_materialized": True, "data_freshness_seconds": 21600},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+
+ response = self.client.patch(
+ f"/api/environments/{self.team.id}/endpoints/{endpoint.name}/",
+ {"is_active": False, "data_freshness_seconds": 43200},
+ format="json",
+ )
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+
+ version = endpoint.get_version()
+ self.assertIsNone(version.saved_query_id)
+ self.assertEqual(version.cache_ttl_seconds, 43200)
+
def test_deactivating_endpoint_disables_all_versions_materialization(self):
"""Deactivating the endpoint must tear down every materialized version, not just the current one."""
endpoint = create_endpoint_with_version(
diff --git a/products/endpoints/backend/tests/test_rate_limit.py b/products/endpoints/backend/tests/test_rate_limit.py
index fa5db66c8a6a..587eba076803 100644
--- a/products/endpoints/backend/tests/test_rate_limit.py
+++ b/products/endpoints/backend/tests/test_rate_limit.py
@@ -339,7 +339,7 @@ def _create_ready_materialized_endpoint(team, user, name: str, materialized_at)
query={"kind": "HogQLQuery", "query": "SELECT 1"},
created_by=user,
saved_query=saved_query,
- data_freshness_seconds=3600,
+ cache_ttl_seconds=3600,
)
@@ -605,7 +605,7 @@ class TestMaterializationStateCacheTimeout(APIBaseTest):
query={"kind": "HogQLQuery", "query": "SELECT 1"},
created_by=self.user,
saved_query=saved_query,
- data_freshness_seconds=3600,
+ cache_ttl_seconds=3600,
)
with patch("products.endpoints.backend.rate_limit.cache.set", wraps=cache.set) as cache_set:
diff --git a/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx b/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx
index 35e0041c8fc5..42433414e712 100644
--- a/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx
+++ b/products/endpoints/frontend/endpoint-tabs/EndpointConfiguration.tsx
@@ -17,6 +17,7 @@ import {
import { AccessControlAction } from 'lib/components/AccessControlAction'
import { CodeSnippet, Language } from 'lib/components/CodeSnippet'
import { LemonField } from 'lib/lemon-ui/LemonField'
+import { humanFriendlyDuration } from 'lib/utils/durations'
import { MaterializationStatusModal } from 'scenes/data-warehouse/saved_queries/MaterializationStatusModal'
import { AccessControlLevel, AccessControlResourceType } from '~/types'
@@ -96,6 +97,12 @@ export function EndpointConfiguration(): JSX.Element {
const effectiveDataFreshness =
dataFreshness ?? viewingVersion?.data_freshness_seconds ?? endpoint.data_freshness_seconds ?? 86400
+ const dataFreshnessOptions = DATA_FRESHNESS_OPTIONS.some((option) => option.value === effectiveDataFreshness)
+ ? DATA_FRESHNESS_OPTIONS
+ : [
+ ...DATA_FRESHNESS_OPTIONS,
+ { value: effectiveDataFreshness, label: humanFriendlyDuration(effectiveDataFreshness) },
+ ]
const baseIsMaterialized = viewingVersion?.is_materialized ?? endpoint.is_materialized
const isMaterialized = localIsMaterialized ?? baseIsMaterialized
const materializationExpanded = leftActiveKeys.includes('materialization')
@@ -148,7 +155,7 @@ export function EndpointConfiguration(): JSX.Element {
<LemonSelect
value={effectiveDataFreshness}
onChange={setDataFreshness}
- options={DATA_FRESHNESS_OPTIONS}
+ options={dataFreshnessOptions}
/>
</AccessControlAction>
</div>
diff --git a/products/endpoints/frontend/generated/api.schemas.ts b/products/endpoints/frontend/generated/api.schemas.ts
index 74e97b28345e..5f5fa4541ab0 100644
--- a/products/endpoints/frontend/generated/api.schemas.ts
+++ b/products/endpoints/frontend/generated/api.schemas.ts
@@ -130,7 +130,7 @@ export interface EndpointResponseApi {
query: unknown
/** Whether the endpoint can be executed via the API. */
is_active: boolean
- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */
+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */
data_freshness_seconds: number
/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */
endpoint_path: string
@@ -287,7 +287,7 @@ export interface EndpointVersionResponseApi {
query: unknown
/** Whether the endpoint can be executed via the API. */
is_active: boolean
- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */
+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */
data_freshness_seconds: number
/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */
endpoint_path: string
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 11e7cdeec398..1aed61ba28b7 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -42603,7 +42603,7 @@ export namespace Schemas {
query: unknown;
/** Whether the endpoint can be executed via the API. */
is_active: boolean;
- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */
+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */
data_freshness_seconds: number;
/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */
endpoint_path: string;
@@ -42738,7 +42738,7 @@ export namespace Schemas {
query: unknown;
/** Whether the endpoint can be executed via the API. */
is_active: boolean;
- /** How fresh the data is, in seconds. One of: 900, 1800, 3600, 21600, 43200, 86400, 604800. */
+ /** Effective data freshness, in seconds. Materialized endpoints use the node's refresh target, which can differ from the standard freshness choices. Other endpoints use the cache duration. */
data_freshness_seconds: number;
/** Relative API path to execute this endpoint (e.g. /api/projects/{team_id}/endpoints/{name}/run). */
endpoint_path: string;