Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(insights): refresh the all-time start date after a view changes (#127976)

Thomas Obermüller14bd137cd02e
diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
index 564eef77e57a..7157cda05591 100644
--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py
+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
@@ -13,6 +13,7 @@ from unittest.mock import MagicMock, patch
from django.core.cache import cache
from django.test import override_settings
+from django.utils import timezone
from dateutil import parser
from parameterized import parameterized
@@ -38,7 +39,7 @@ from posthog.models.instance_setting import override_instance_config
from posthog.models.team import WeekStartDay
from products.actions.backend.models.action import Action
-from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobStatus, DataWarehouseSavedQuery
@override_settings(IN_UNIT_TESTING=True)
@@ -552,29 +553,37 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
# date_to raised "can't compare offset-naive and offset-aware datetimes" here.
self.assertGreater(len(query_date_range.all_values()), 0)
- @override_settings(IN_UNIT_TESTING=False)
- @patch("posthog.hogql_queries.utils.timestamp_utils._get_earliest_timestamp_from_node")
- def test_multi_node_propagates_query_tags_to_threads(self, mock_node):
- # Multiple nodes resolve their earliest timestamp via ThreadPoolExecutor, which does not
- # inherit contextvars. Without copying the context, the worker threads' sync_execute calls
- # run untagged and raise UntaggedQueryError in dev (DEBUG and not TEST).
- captured: dict[str, object] = {}
+ @parameterized.expand([("view_edited",), ("view_materialized_again",)])
+ def test_data_warehouse_view_change_refreshes_cached_earliest_timestamp(self, change):
+ view = DataWarehouseSavedQuery.objects.create(
+ team=self.team, name="orders_view", query={"kind": "HogQLQuery", "query": "SELECT 1 AS id"}
+ )
+ node = DataWarehouseNode(
+ id="orders_view",
+ table_name="orders_view",
+ id_field="id",
+ distinct_id_field="id",
+ timestamp_field="created_at",
+ )
+ before_change = datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC)
+ after_change = datetime.datetime(2022, 1, 1, tzinfo=datetime.UTC)
- def capture(team, node, user=None):
- captured[node.table_name] = get_query_tags().product
- return datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)
+ with patch("posthog.hogql_queries.utils.timestamp_utils.execute_hogql_query") as mock_execute:
+ mock_execute.return_value.results = [[before_change]]
+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), before_change)
- mock_node.side_effect = capture
- nodes = [
- DataWarehouseNode(id="a", table_name="a", id_field="id", distinct_id_field="id", timestamp_field="ts"),
- DataWarehouseNode(id="b", table_name="b", id_field="id", distinct_id_field="id", timestamp_field="ts"),
- ]
+ mock_execute.return_value.results = [[after_change]]
+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), before_change)
- with tags_context(product=Product.MARKETING_ANALYTICS, feature=Feature.QUERY):
- get_earliest_timestamp_from_series(self.team, nodes)
+ if change == "view_edited":
+ view.query = {"kind": "HogQLQuery", "query": "SELECT 2 AS id"}
+ view.save()
+ else:
+ DataModelingJob.objects.create(
+ team=self.team, saved_query=view, status=DataModelingJobStatus.COMPLETED, last_run_at=timezone.now()
+ )
- self.assertEqual(captured["a"], Product.MARKETING_ANALYTICS)
- self.assertEqual(captured["b"], Product.MARKETING_ANALYTICS)
+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), after_change)
def test_cached_raw_value_is_coerced_on_read(self):
# Entries cached before the coercion fix shipped hold a raw str/date for up to the
diff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.py
index 8dfdb586b930..960f1dbb5114 100644
--- a/posthog/hogql_queries/utils/timestamp_utils.py
+++ b/posthog/hogql_queries/utils/timestamp_utils.py
@@ -1,11 +1,8 @@
-import contextvars
from collections.abc import Sequence
-from concurrent.futures import ThreadPoolExecutor
from contextlib import AbstractContextManager
from datetime import date, datetime, timedelta, tzinfo
-from typing import Any, Optional, Union, cast
+from typing import Any, Optional, Union
-from django.conf import settings
from django.core.cache import cache
from django.utils import timezone
@@ -32,6 +29,7 @@ from posthog.hogql.property import action_to_expr
from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context
+from posthog.hogql_queries.utils.caller_context import map_in_caller_context
from posthog.hogql_queries.utils.query_date_range import QueryDateRange
from posthog.models import Team, User
from posthog.models.event import DEFAULT_EARLIEST_TIME_DELTA
@@ -40,6 +38,7 @@ from posthog.models.team import WeekStartDay
from posthog.utils import get_safe_cache
from products.actions.backend.models.action import Action
+from products.data_modeling.backend.facade import api as data_modeling_api
EARLIEST_TIMESTAMP_CACHE_TTL = 24 * 60 * 60
EARLIEST_EVENT_TIMESTAMP = datetime.fromisoformat("1980-01-01T00:00:00Z")
@@ -113,6 +112,13 @@ def _get_event_earliest_timestamp_query(team: Team, node: Union[EventsNode, Acti
)
+def _data_warehouse_cache_key_suffix(team: Team, table_name: str) -> str:
+ # A view returns different rows after someone edits its HogQL or it materializes again, so its
+ # earliest timestamp is valid only for the revision it was read from. A warehouse table has no revision.
+ revision = data_modeling_api.saved_query_data_revision(team.pk, table_name)
+ return f"_{revision}" if revision else ""
+
+
def _get_earliest_timestamp_cache_key(
team: Team,
node: Union[
@@ -131,11 +137,14 @@ def _get_earliest_timestamp_cache_key(
# Use the same cache key as EventsNode(event=None) since they return the same result
return f"earliest_timestamp_event_{team.pk}"
case DataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):
- return f"earliest_timestamp_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"
+ suffix = _data_warehouse_cache_key_suffix(team, table_name)
+ return f"earliest_timestamp_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"
case FunnelsDataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):
- return f"earliest_timestamp_funnels_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"
+ suffix = _data_warehouse_cache_key_suffix(team, table_name)
+ return f"earliest_timestamp_funnels_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"
case LifecycleDataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):
- return f"earliest_timestamp_lifecycle_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"
+ suffix = _data_warehouse_cache_key_suffix(team, table_name)
+ return f"earliest_timestamp_lifecycle_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"
case ActionsNode(id=action_id):
return f"earliest_timestamp_action_{team.pk}_{action_id}"
case EventsNode(event=None):
@@ -286,19 +295,12 @@ def get_earliest_timestamp_from_series(
else:
nodes.append(node)
- if len(nodes) == 1 or settings.IN_UNIT_TESTING:
- timestamps = [_get_earliest_timestamp_from_node(team, node, user) for node in nodes]
-
- else:
- with ThreadPoolExecutor(max_workers=min(len(nodes), 4)) as executor:
- # ThreadPoolExecutor does not inherit contextvars (query tags) by default; copy the
- # current context into each worker so tagged sync_execute calls don't fail untagged.
- futures = [
- executor.submit(contextvars.copy_context().run, _get_earliest_timestamp_from_node, team, node, user)
- for node in nodes
- ]
- timestamps = [cast(datetime, future.result()) for future in futures]
-
+ timestamps = map_in_caller_context(
+ lambda node: _get_earliest_timestamp_from_node(team, node, user),
+ nodes,
+ max_workers=4,
+ thread_name_prefix="earliest_timestamp",
+ )
return min(timestamps)
diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.py
index c37db9f3f689..3f429ae6ba05 100644
--- a/products/data_modeling/backend/facade/api.py
+++ b/products/data_modeling/backend/facade/api.py
@@ -47,6 +47,7 @@ _LAZY = {
"get_saved_query_ids_for_nodes": "logic.saved_query_reads",
"get_saved_query_summary": "logic.saved_query_reads",
"get_saved_query_sql": "logic.saved_query_reads",
+ "saved_query_data_revision": "logic.saved_query_reads",
"all_saved_query_columns": "logic.saved_query_reads",
"all_saved_query_names": "logic.saved_query_reads",
"allowed_saved_query_ids": "logic.saved_query_reads",
diff --git a/products/data_modeling/backend/logic/saved_query_reads.py b/products/data_modeling/backend/logic/saved_query_reads.py
index 59a758240362..0fb532bab059 100644
--- a/products/data_modeling/backend/logic/saved_query_reads.py
+++ b/products/data_modeling/backend/logic/saved_query_reads.py
@@ -26,6 +26,7 @@ from ..models.datawarehouse_saved_query import DataWarehouseSavedQuery
from ..models.edge import Edge
from ..models.node import Node, NodeType
from .incremental import get_incremental_config
+from .node_suspension import query_fingerprint
from .saved_query_freshness import saved_query_materialized_at
from .tier_run_report import MATERIALIZING_TYPES
@@ -108,6 +109,21 @@ def get_saved_query_sql(team_id: int, saved_query_id: UUID | str) -> str | None:
return sql if isinstance(sql, str) else None
+def saved_query_data_revision(team_id: int, name: str) -> str | None:
+ """A token that changes when the rows the named saved query returns can change: an edit to its
+ HogQL, or a new materialization. None when no saved query in this team has this name."""
+ saved_query = (
+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, name=name)
+ .exclude(deleted=True)
+ .only("id", "team_id", "query", "last_run_at")
+ .first()
+ )
+ if saved_query is None:
+ return None
+ materialized_at = saved_query_materialized_at(saved_query)
+ return f"{query_fingerprint(saved_query.query) or ''}:{materialized_at.isoformat() if materialized_at else ''}"
+
+
def all_saved_query_names(team_id: int) -> dict[str, str]:
"""The current name of every saved query in this team that still resolves. One query."""
rows = DataWarehouseSavedQuery.objects.filter(team_id=team_id).exclude(deleted=True).values_list("id", "name")