:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(insights): read string warehouse timestamps as dates for all time (#127979)
Thomas Obermüllerc58540c26abe
diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.pyindex 55ca7c46b45a..564eef77e57a 100644--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py@@ -38,6 +38,7 @@ from posthog.models.instance_setting import override_instance_configfrom posthog.models.team import WeekStartDayfrom products.actions.backend.models.action import Action+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery@override_settings(IN_UNIT_TESTING=True)@@ -465,6 +466,44 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):# Must be timezone-aware so it can be compared against the tz-aware date_to.self.assertIsNotNone(result.tzinfo)+ @parameterized.expand(+ [+ (+ "string_column_parses_as_dates",+ "formatDateTime(timestamp, '%m/%d/%Y')",+ {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True},+ datetime.datetime(2023, 12, 30, tzinfo=datetime.UTC),+ ),+ (+ "datetime64_column_keeps_subseconds",+ "timestamp",+ {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')", "valid": True},+ datetime.datetime(2023, 12, 30, 0, 0, 0, 500000, tzinfo=datetime.UTC),+ ),+ ]+ )+ def test_data_warehouse_all_time_reads_earliest_by_column_type(self, _name, created_at_expr, column, expected):+ for timestamp in ("2023-12-30T00:00:00.500Z", "2026-01-15T09:00:00Z"):+ _create_event(team=self.team, event="order_placed", distinct_id="person1", timestamp=timestamp)+ flush_persons_and_events()+ DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="orders_view",+ query={"kind": "HogQLQuery", "query": f"SELECT {created_at_expr} AS created_at FROM events"},+ columns={"created_at": column},+ )+ node = DataWarehouseNode(+ id="orders_view",+ table_name="orders_view",+ id_field="created_at",+ distinct_id_field="created_at",+ timestamp_field="created_at",+ )++ earliest_timestamp = get_earliest_timestamp_from_series(self.team, [node])++ self.assertEqual(earliest_timestamp, expected)+@parameterized.expand([("string_timestamp", "2022-03-15 08:00:00", datetime.datetime(2022, 3, 15, 8, 0, 0, tzinfo=datetime.UTC)),@@ -475,6 +514,12 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):def test_data_warehouse_all_time_resolves_string_timestamp(self, _name, raw_value, expected):# Data warehouse tables can return a non-datetime min(timestamp); it must be# coerced before reaching QueryDateRange, which calls .strftime() and compares with <.+ DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="dw_table",+ query={"kind": "HogQLQuery", "query": "SELECT '' AS ts"},+ columns={"ts": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},+ )node = DataWarehouseNode(id="dw_table",table_name="dw_table",diff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.pyindex d2ec736b46db..8dfdb586b930 100644--- a/posthog/hogql_queries/utils/timestamp_utils.py+++ b/posthog/hogql_queries/utils/timestamp_utils.py@@ -25,6 +25,8 @@ from posthog.schema import (from posthog.hogql import astfrom posthog.hogql.ast import SelectQueryfrom posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.database.models import StringDatabaseFieldfrom posthog.hogql.modifiers import create_default_modifiers_for_teamfrom posthog.hogql.property import action_to_exprfrom posthog.hogql.query import execute_hogql_query@@ -47,15 +49,23 @@ UNFILTERED_EARLIEST_TIMESTAMP_FLOOR = datetime.fromisoformat("2015-01-01T00:00:0def _get_data_warehouse_earliest_timestamp_query(node: Union[DataWarehouseNode, FunnelsDataWarehouseNode, LifecycleDataWarehouseNode],+ database: Database,) -> SelectQuery:"""Get the select query for the earliest timestamp from a DataWarehouseNode.:param node: The DataWarehouseNode+ :param database: The HogQL database the query runs against, used to read the timestamp field's type:return: A SelectQuery object that retrieves the earliest timestamp from the specified data warehouse table."""+ timestamp: ast.Expr = ast.Field(chain=[node.timestamp_field])+ # Insights compare a String timestamp column as parsed dates. min() on the raw String sorts it as+ # text, so for a format like MM/DD/YYYY it returns a later date and "All time" drops older rows.+ # Other types keep their native min, because toDateTime cuts a DateTime64 to whole seconds.+ if isinstance(database.get_table(node.table_name).fields.get(node.timestamp_field), StringDatabaseField):+ timestamp = ast.Call(name="toDateTime", args=[timestamp])return ast.SelectQuery(- select=[ast.Call(name="min", args=[ast.Field(chain=[node.timestamp_field])])],+ select=[ast.Call(name="min", args=[timestamp])],select_from=ast.JoinExpr(table=ast.Field(chain=[node.table_name])),)@@ -221,18 +231,25 @@ def _get_earliest_timestamp_from_node(# datetime values written after the fix and repairs the stale ones.return _coerce_to_datetime(cached_result, team.timezone_info)- if (- isinstance(node, DataWarehouseNode)- or isinstance(node, FunnelsDataWarehouseNode)- or isinstance(node, LifecycleDataWarehouseNode)- ):- query = _get_data_warehouse_earliest_timestamp_query(node)- else:- query = _get_event_earliest_timestamp_query(team, node)-earliest_timestamp = EARLIEST_EVENT_TIMESTAMPwith _earliest_timestamp_query_tags():- result = execute_hogql_query(query=query, team=team, user=user, modifiers=modifiers)+ if (+ isinstance(node, DataWarehouseNode)+ or isinstance(node, FunnelsDataWarehouseNode)+ or isinstance(node, LifecycleDataWarehouseNode)+ ):+ database = Database.create_for(team=team, user=user, modifiers=modifiers)+ result = execute_hogql_query(+ query=_get_data_warehouse_earliest_timestamp_query(node, database),+ team=team,+ user=user,+ modifiers=modifiers,+ context=HogQLContext(team_id=team.pk, user=user, database=database),+ )+ else:+ result = execute_hogql_query(+ query=_get_event_earliest_timestamp_query(team, node), team=team, user=user, modifiers=modifiers+ )if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)