:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(insights): refresh the all-time start date after a view changes (#127976)
Thomas Obermüller14bd137cd02e
diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.pyindex 564eef77e57a..7157cda05591 100644--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py@@ -13,6 +13,7 @@ from unittest.mock import MagicMock, patchfrom django.core.cache import cachefrom django.test import override_settings+from django.utils import timezonefrom dateutil import parserfrom parameterized import parameterized@@ -38,7 +39,7 @@ from posthog.models.instance_setting import override_instance_configfrom posthog.models.team import WeekStartDayfrom products.actions.backend.models.action import Action-from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+from products.data_modeling.backend.facade.models import DataModelingJob, DataModelingJobStatus, DataWarehouseSavedQuery@override_settings(IN_UNIT_TESTING=True)@@ -552,29 +553,37 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):# date_to raised "can't compare offset-naive and offset-aware datetimes" here.self.assertGreater(len(query_date_range.all_values()), 0)- @override_settings(IN_UNIT_TESTING=False)- @patch("posthog.hogql_queries.utils.timestamp_utils._get_earliest_timestamp_from_node")- def test_multi_node_propagates_query_tags_to_threads(self, mock_node):- # Multiple nodes resolve their earliest timestamp via ThreadPoolExecutor, which does not- # inherit contextvars. Without copying the context, the worker threads' sync_execute calls- # run untagged and raise UntaggedQueryError in dev (DEBUG and not TEST).- captured: dict[str, object] = {}+ @parameterized.expand([("view_edited",), ("view_materialized_again",)])+ def test_data_warehouse_view_change_refreshes_cached_earliest_timestamp(self, change):+ view = DataWarehouseSavedQuery.objects.create(+ team=self.team, name="orders_view", query={"kind": "HogQLQuery", "query": "SELECT 1 AS id"}+ )+ node = DataWarehouseNode(+ id="orders_view",+ table_name="orders_view",+ id_field="id",+ distinct_id_field="id",+ timestamp_field="created_at",+ )+ before_change = datetime.datetime(2024, 6, 1, tzinfo=datetime.UTC)+ after_change = datetime.datetime(2022, 1, 1, tzinfo=datetime.UTC)- def capture(team, node, user=None):- captured[node.table_name] = get_query_tags().product- return datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC)+ with patch("posthog.hogql_queries.utils.timestamp_utils.execute_hogql_query") as mock_execute:+ mock_execute.return_value.results = [[before_change]]+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), before_change)- mock_node.side_effect = capture- nodes = [- DataWarehouseNode(id="a", table_name="a", id_field="id", distinct_id_field="id", timestamp_field="ts"),- DataWarehouseNode(id="b", table_name="b", id_field="id", distinct_id_field="id", timestamp_field="ts"),- ]+ mock_execute.return_value.results = [[after_change]]+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), before_change)- with tags_context(product=Product.MARKETING_ANALYTICS, feature=Feature.QUERY):- get_earliest_timestamp_from_series(self.team, nodes)+ if change == "view_edited":+ view.query = {"kind": "HogQLQuery", "query": "SELECT 2 AS id"}+ view.save()+ else:+ DataModelingJob.objects.create(+ team=self.team, saved_query=view, status=DataModelingJobStatus.COMPLETED, last_run_at=timezone.now()+ )- self.assertEqual(captured["a"], Product.MARKETING_ANALYTICS)- self.assertEqual(captured["b"], Product.MARKETING_ANALYTICS)+ self.assertEqual(get_earliest_timestamp_from_series(self.team, [node]), after_change)def test_cached_raw_value_is_coerced_on_read(self):# Entries cached before the coercion fix shipped hold a raw str/date for up to thediff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.pyindex 8dfdb586b930..960f1dbb5114 100644--- a/posthog/hogql_queries/utils/timestamp_utils.py+++ b/posthog/hogql_queries/utils/timestamp_utils.py@@ -1,11 +1,8 @@-import contextvarsfrom collections.abc import Sequence-from concurrent.futures import ThreadPoolExecutorfrom contextlib import AbstractContextManagerfrom datetime import date, datetime, timedelta, tzinfo-from typing import Any, Optional, Union, cast+from typing import Any, Optional, Union-from django.conf import settingsfrom django.core.cache import cachefrom django.utils import timezone@@ -32,6 +29,7 @@ from posthog.hogql.property import action_to_exprfrom posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import Feature, Product, get_query_tags, tags_context+from posthog.hogql_queries.utils.caller_context import map_in_caller_contextfrom posthog.hogql_queries.utils.query_date_range import QueryDateRangefrom posthog.models import Team, Userfrom posthog.models.event import DEFAULT_EARLIEST_TIME_DELTA@@ -40,6 +38,7 @@ from posthog.models.team import WeekStartDayfrom posthog.utils import get_safe_cachefrom products.actions.backend.models.action import Action+from products.data_modeling.backend.facade import api as data_modeling_apiEARLIEST_TIMESTAMP_CACHE_TTL = 24 * 60 * 60EARLIEST_EVENT_TIMESTAMP = datetime.fromisoformat("1980-01-01T00:00:00Z")@@ -113,6 +112,13 @@ def _get_event_earliest_timestamp_query(team: Team, node: Union[EventsNode, Acti)+def _data_warehouse_cache_key_suffix(team: Team, table_name: str) -> str:+ # A view returns different rows after someone edits its HogQL or it materializes again, so its+ # earliest timestamp is valid only for the revision it was read from. A warehouse table has no revision.+ revision = data_modeling_api.saved_query_data_revision(team.pk, table_name)+ return f"_{revision}" if revision else ""++def _get_earliest_timestamp_cache_key(team: Team,node: Union[@@ -131,11 +137,14 @@ def _get_earliest_timestamp_cache_key(# Use the same cache key as EventsNode(event=None) since they return the same resultreturn f"earliest_timestamp_event_{team.pk}"case DataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):- return f"earliest_timestamp_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"+ suffix = _data_warehouse_cache_key_suffix(team, table_name)+ return f"earliest_timestamp_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"case FunnelsDataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):- return f"earliest_timestamp_funnels_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"+ suffix = _data_warehouse_cache_key_suffix(team, table_name)+ return f"earliest_timestamp_funnels_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"case LifecycleDataWarehouseNode(table_name=table_name, timestamp_field=timestamp_field):- return f"earliest_timestamp_lifecycle_data_warehouse_{team.pk}_{table_name}_{timestamp_field}"+ suffix = _data_warehouse_cache_key_suffix(team, table_name)+ return f"earliest_timestamp_lifecycle_data_warehouse_{team.pk}_{table_name}_{timestamp_field}{suffix}"case ActionsNode(id=action_id):return f"earliest_timestamp_action_{team.pk}_{action_id}"case EventsNode(event=None):@@ -286,19 +295,12 @@ def get_earliest_timestamp_from_series(else:nodes.append(node)- if len(nodes) == 1 or settings.IN_UNIT_TESTING:- timestamps = [_get_earliest_timestamp_from_node(team, node, user) for node in nodes]-- else:- with ThreadPoolExecutor(max_workers=min(len(nodes), 4)) as executor:- # ThreadPoolExecutor does not inherit contextvars (query tags) by default; copy the- # current context into each worker so tagged sync_execute calls don't fail untagged.- futures = [- executor.submit(contextvars.copy_context().run, _get_earliest_timestamp_from_node, team, node, user)- for node in nodes- ]- timestamps = [cast(datetime, future.result()) for future in futures]-+ timestamps = map_in_caller_context(+ lambda node: _get_earliest_timestamp_from_node(team, node, user),+ nodes,+ max_workers=4,+ thread_name_prefix="earliest_timestamp",+ )return min(timestamps)diff --git a/products/data_modeling/backend/facade/api.py b/products/data_modeling/backend/facade/api.pyindex c37db9f3f689..3f429ae6ba05 100644--- a/products/data_modeling/backend/facade/api.py+++ b/products/data_modeling/backend/facade/api.py@@ -47,6 +47,7 @@ _LAZY = {"get_saved_query_ids_for_nodes": "logic.saved_query_reads","get_saved_query_summary": "logic.saved_query_reads","get_saved_query_sql": "logic.saved_query_reads",+ "saved_query_data_revision": "logic.saved_query_reads","all_saved_query_columns": "logic.saved_query_reads","all_saved_query_names": "logic.saved_query_reads","allowed_saved_query_ids": "logic.saved_query_reads",diff --git a/products/data_modeling/backend/logic/saved_query_reads.py b/products/data_modeling/backend/logic/saved_query_reads.pyindex 59a758240362..0fb532bab059 100644--- a/products/data_modeling/backend/logic/saved_query_reads.py+++ b/products/data_modeling/backend/logic/saved_query_reads.py@@ -26,6 +26,7 @@ from ..models.datawarehouse_saved_query import DataWarehouseSavedQueryfrom ..models.edge import Edgefrom ..models.node import Node, NodeTypefrom .incremental import get_incremental_config+from .node_suspension import query_fingerprintfrom .saved_query_freshness import saved_query_materialized_atfrom .tier_run_report import MATERIALIZING_TYPES@@ -108,6 +109,21 @@ def get_saved_query_sql(team_id: int, saved_query_id: UUID | str) -> str | None:return sql if isinstance(sql, str) else None+def saved_query_data_revision(team_id: int, name: str) -> str | None:+ """A token that changes when the rows the named saved query returns can change: an edit to its+ HogQL, or a new materialization. None when no saved query in this team has this name."""+ saved_query = (+ DataWarehouseSavedQuery.objects.filter(team_id=team_id, name=name)+ .exclude(deleted=True)+ .only("id", "team_id", "query", "last_run_at")+ .first()+ )+ if saved_query is None:+ return None+ materialized_at = saved_query_materialized_at(saved_query)+ return f"{query_fingerprint(saved_query.query) or ''}:{materialized_at.isoformat() if materialized_at else ''}"++def all_saved_query_names(team_id: int) -> dict[str, str]:"""The current name of every saved query in this team that still resolves. One query."""rows = DataWarehouseSavedQuery.objects.filter(team_id=team_id).exclude(deleted=True).values_list("id", "name")