Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(insights): read string warehouse timestamps as dates for all time (#127979)

Thomas Obermüllerc58540c26abe
diff --git a/posthog/hogql_queries/utils/test/test_timestamp_utils.py b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
index 55ca7c46b45a..564eef77e57a 100644
--- a/posthog/hogql_queries/utils/test/test_timestamp_utils.py
+++ b/posthog/hogql_queries/utils/test/test_timestamp_utils.py
@@ -38,6 +38,7 @@ from posthog.models.instance_setting import override_instance_config
from posthog.models.team import WeekStartDay
from products.actions.backend.models.action import Action
+from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
@override_settings(IN_UNIT_TESTING=True)
@@ -465,6 +466,44 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
# Must be timezone-aware so it can be compared against the tz-aware date_to.
self.assertIsNotNone(result.tzinfo)
+ @parameterized.expand(
+ [
+ (
+ "string_column_parses_as_dates",
+ "formatDateTime(timestamp, '%m/%d/%Y')",
+ {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True},
+ datetime.datetime(2023, 12, 30, tzinfo=datetime.UTC),
+ ),
+ (
+ "datetime64_column_keeps_subseconds",
+ "timestamp",
+ {"hogql": "DateTimeDatabaseField", "clickhouse": "DateTime64(6, 'UTC')", "valid": True},
+ datetime.datetime(2023, 12, 30, 0, 0, 0, 500000, tzinfo=datetime.UTC),
+ ),
+ ]
+ )
+ def test_data_warehouse_all_time_reads_earliest_by_column_type(self, _name, created_at_expr, column, expected):
+ for timestamp in ("2023-12-30T00:00:00.500Z", "2026-01-15T09:00:00Z"):
+ _create_event(team=self.team, event="order_placed", distinct_id="person1", timestamp=timestamp)
+ flush_persons_and_events()
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="orders_view",
+ query={"kind": "HogQLQuery", "query": f"SELECT {created_at_expr} AS created_at FROM events"},
+ columns={"created_at": column},
+ )
+ node = DataWarehouseNode(
+ id="orders_view",
+ table_name="orders_view",
+ id_field="created_at",
+ distinct_id_field="created_at",
+ timestamp_field="created_at",
+ )
+
+ earliest_timestamp = get_earliest_timestamp_from_series(self.team, [node])
+
+ self.assertEqual(earliest_timestamp, expected)
+
@parameterized.expand(
[
("string_timestamp", "2022-03-15 08:00:00", datetime.datetime(2022, 3, 15, 8, 0, 0, tzinfo=datetime.UTC)),
@@ -475,6 +514,12 @@ class TestTimestampUtils(APIBaseTest, ClickhouseDestroyTablesMixin):
def test_data_warehouse_all_time_resolves_string_timestamp(self, _name, raw_value, expected):
# Data warehouse tables can return a non-datetime min(timestamp); it must be
# coerced before reaching QueryDateRange, which calls .strftime() and compares with <.
+ DataWarehouseSavedQuery.objects.create(
+ team=self.team,
+ name="dw_table",
+ query={"kind": "HogQLQuery", "query": "SELECT '' AS ts"},
+ columns={"ts": {"hogql": "StringDatabaseField", "clickhouse": "String", "valid": True}},
+ )
node = DataWarehouseNode(
id="dw_table",
table_name="dw_table",
diff --git a/posthog/hogql_queries/utils/timestamp_utils.py b/posthog/hogql_queries/utils/timestamp_utils.py
index d2ec736b46db..8dfdb586b930 100644
--- a/posthog/hogql_queries/utils/timestamp_utils.py
+++ b/posthog/hogql_queries/utils/timestamp_utils.py
@@ -25,6 +25,8 @@ from posthog.schema import (
from posthog.hogql import ast
from posthog.hogql.ast import SelectQuery
from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.database.models import StringDatabaseField
from posthog.hogql.modifiers import create_default_modifiers_for_team
from posthog.hogql.property import action_to_expr
from posthog.hogql.query import execute_hogql_query
@@ -47,15 +49,23 @@ UNFILTERED_EARLIEST_TIMESTAMP_FLOOR = datetime.fromisoformat("2015-01-01T00:00:0
def _get_data_warehouse_earliest_timestamp_query(
node: Union[DataWarehouseNode, FunnelsDataWarehouseNode, LifecycleDataWarehouseNode],
+ database: Database,
) -> SelectQuery:
"""
Get the select query for the earliest timestamp from a DataWarehouseNode.
:param node: The DataWarehouseNode
+ :param database: The HogQL database the query runs against, used to read the timestamp field's type
:return: A SelectQuery object that retrieves the earliest timestamp from the specified data warehouse table.
"""
+ timestamp: ast.Expr = ast.Field(chain=[node.timestamp_field])
+ # Insights compare a String timestamp column as parsed dates. min() on the raw String sorts it as
+ # text, so for a format like MM/DD/YYYY it returns a later date and "All time" drops older rows.
+ # Other types keep their native min, because toDateTime cuts a DateTime64 to whole seconds.
+ if isinstance(database.get_table(node.table_name).fields.get(node.timestamp_field), StringDatabaseField):
+ timestamp = ast.Call(name="toDateTime", args=[timestamp])
return ast.SelectQuery(
- select=[ast.Call(name="min", args=[ast.Field(chain=[node.timestamp_field])])],
+ select=[ast.Call(name="min", args=[timestamp])],
select_from=ast.JoinExpr(table=ast.Field(chain=[node.table_name])),
)
@@ -221,18 +231,25 @@ def _get_earliest_timestamp_from_node(
# datetime values written after the fix and repairs the stale ones.
return _coerce_to_datetime(cached_result, team.timezone_info)
- if (
- isinstance(node, DataWarehouseNode)
- or isinstance(node, FunnelsDataWarehouseNode)
- or isinstance(node, LifecycleDataWarehouseNode)
- ):
- query = _get_data_warehouse_earliest_timestamp_query(node)
- else:
- query = _get_event_earliest_timestamp_query(team, node)
-
earliest_timestamp = EARLIEST_EVENT_TIMESTAMP
with _earliest_timestamp_query_tags():
- result = execute_hogql_query(query=query, team=team, user=user, modifiers=modifiers)
+ if (
+ isinstance(node, DataWarehouseNode)
+ or isinstance(node, FunnelsDataWarehouseNode)
+ or isinstance(node, LifecycleDataWarehouseNode)
+ ):
+ database = Database.create_for(team=team, user=user, modifiers=modifiers)
+ result = execute_hogql_query(
+ query=_get_data_warehouse_earliest_timestamp_query(node, database),
+ team=team,
+ user=user,
+ modifiers=modifiers,
+ context=HogQLContext(team_id=team.pk, user=user, database=database),
+ )
+ else:
+ result = execute_hogql_query(
+ query=_get_event_earliest_timestamp_query(team, node), team=team, user=user, modifiers=modifiers
+ )
if result and len(result.results) > 0 and len(result.results[0]) > 0 and result.results[0][0] is not None:
earliest_timestamp = _coerce_to_datetime(result.results[0][0], team.timezone_info)