:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(query): cache api key sql results by the time window they read (#125610)
Vasco de Krijgerf4d91ccfc12c
diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.pyindex 150d66308cda..50ebf373fcd1 100644--- a/posthog/hogql_queries/hogql_query_runner.py+++ b/posthog/hogql_queries/hogql_query_runner.py@@ -1,5 +1,5 @@from collections.abc import Callable-from datetime import datetime+from datetime import UTC, datetimefrom functools import cached_propertyfrom typing import Any, Optional, cast@@ -27,15 +27,24 @@ from posthog.hogql.metadata import get_table_namesfrom posthog.hogql.parser import CacheOrigin, parse_selectfrom posthog.hogql.placeholders import find_placeholders, replace_placeholdersfrom posthog.hogql.query import execute_hogql_query+from posthog.hogql.resolver import resolve_typesfrom posthog.hogql.user_query_validator import validate_user_queryfrom posthog.hogql.variables import replace_variables+from posthog.hogql.visitor import clone_exprfrom posthog import settings as app_settingsfrom posthog.caching.utils import ThresholdMode, staleness_threshold_map-from posthog.clickhouse.query_tagging import tag_contains_user_hogql+from posthog.clickhouse.query_tagging import get_query_tag_value, is_api_key_access_method, tag_contains_user_hogqlfrom posthog.event_usage import AnalyticsPropsfrom posthog.hogql_queries.paginators import HogQLHasMorePaginator-from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionMode+from posthog.hogql_queries.query_runner import CACHE_ENTRY_FRESHNESS_KEY, AnalyticsQueryRunner, ExecutionMode+from posthog.hogql_queries.sql_freshness.classifier import (+ SqlFreshness,+ classify,+ freshness_from_record,+ freshness_record,+)+from posthog.hogql_queries.sql_freshness.rules import FreshnessReason, Rejectfrom posthog.models import Userfrom posthog.models.activity_logging.retention import get_activity_log_lookback_restriction@@ -49,12 +58,14 @@ from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLM_INFORMATION_SCHEMA_PREFIX = "system.information_schema."_ACTIVITY_LOGS_TABLE = "system.activity_logs"+API_SQL_WINDOW_CACHE_AGE_FLAG = "api-sql-window-cache-age"class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):query: HogQLQuerycached_response: CachedHogQLQueryResponsesettings: Optional[HogQLGlobalSettings]+ _sql_classification: SqlFreshness | Reject | None = None# p95 duration of a query service HogQL query is 2.78secQUERY_SERVICE_MAX_EXECUTION_TIME = 10@@ -75,12 +86,73 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):# A direct connection runs on another database, so the events route never applies to it.return not self.query.connectionId- # Treat SQL query caching like day insightdef cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:if last_refresh is None:return None+ freshness = self._sql_classification+ if freshness is not None and self._api_sql_freshness_enabled:+ expiry = super().cache_target_age(last_refresh, lazy=lazy)+ if isinstance(freshness, SqlFreshness) and expiry is not None:+ return max(expiry, freshness.fresh_until)+ return expiryreturn last_refresh + staleness_threshold_map[ThresholdMode.LAZY if lazy else ThresholdMode.DEFAULT]["day"]+ @cached_property+ def _api_sql_freshness_enabled(self) -> bool:+ return (+ self._reads_events_route()+ and is_api_key_access_method(get_query_tag_value("access_method"))+ and self._team_flag_enabled_locally(API_SQL_WINDOW_CACHE_AGE_FLAG)+ )++ def _is_stale(self, last_refresh: Optional[datetime], lazy: bool = False) -> bool:+ freshness = self._sql_classification+ default_stale = super()._is_stale(last_refresh, lazy=lazy)+ if isinstance(freshness, SqlFreshness) and last_refresh is not None and self._api_sql_freshness_enabled:+ return default_stale and datetime.now(UTC) > freshness.fresh_until+ return default_stale++ def _cache_entry_with_freshness(self, response: dict[str, Any]) -> dict[str, Any]:+ if not isinstance(self._sql_classification, SqlFreshness):+ return response+ return {**response, CACHE_ENTRY_FRESHNESS_KEY: freshness_record(self._sql_classification)}++ def _read_cache_entry_freshness(self, record: Any) -> None:+ self._sql_classification = freshness_from_record(record)++ def _staleness_event_properties(self) -> dict[str, Any]:+ classification = self._sql_classification+ if (+ classification is None+ or getattr(self, "_cache_age_override", None) is not None+ or not self._api_sql_freshness_enabled+ ):+ return {}+ if isinstance(classification, Reject):+ return {"sql_cache_reason": classification.reason.value}+ return {f"sql_cache_{key}": value for key, value in freshness_record(classification).items()}++ def _classify_sql_freshness(self, query: ast.SelectQuery | ast.SelectSetQuery) -> SqlFreshness | Reject:+ if "events" not in {name.lower() for name in self._queried_table_names}:+ return Reject(reason=FreshnessReason.NO_EVENTS)+ try:+ placeholders = find_placeholders(parse_select(self.query.query))+ if (+ placeholders.has_filters+ or placeholders.placeholder_expressions+ or any(chain[:1] == ["variables"] for chain in placeholders.placeholder_fields)+ ):+ return Reject(reason=FreshnessReason.UNSUPPORTED_PLACEHOLDER)+ context = self.build_hogql_context(enable_select_queries=True, modifiers=self.modifiers)+ resolved = resolve_types(clone_expr(query), context, dialect="hogql")+ except Exception:+ return Reject(reason=FreshnessReason.UNPARSEABLE)+ return classify(+ resolved,+ now=datetime.now(UTC),+ timezone=UTC if self.modifiers.convertToProjectTimezone is False else self.team.timezone_info,+ )+def _validate_direct_connection(self, *, user: Optional[User] = None, force: bool = False) -> None:if not force and self._direct_connection_validated:return@@ -280,8 +352,16 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):and self.team.pk not in app_settings.API_QUERIES_LEGACY_TEAM_LIST)+ def _build_query(self) -> ast.SelectQuery | ast.SelectSetQuery:+ query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()+ if self._api_sql_freshness_enabled:+ with self.timings.measure("sql_freshness"):+ self._sql_classification = self._classify_sql_freshness(query)+ return query+def _calculate(self) -> HogQLQueryResponse:tag_contains_user_hogql()+ self._sql_classification = Noneif self._capped_for_query_service():assert self.settings is not None# p95 threads is 102, limiting to 60 (below global max_threads of 64)@@ -325,7 +405,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):trino_canary=self._trino_canary,)- query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()+ query = self._build_query()if self.is_query_service:validate_user_query(query, team=self.team)diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.pyindex bb7c9d77eb5c..3e3ad93e80b9 100644--- a/posthog/hogql_queries/query_runner.py+++ b/posthog/hogql_queries/query_runner.py@@ -304,6 +304,7 @@ def _contains_user_hogql_label() -> str:EXTENDED_CACHE_AGE = timedelta(days=1)+CACHE_ENTRY_FRESHNESS_KEY = "_freshness"QUERY_HASH_VERSION = 1@@ -2292,6 +2293,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):if raw_results is not None:cached_response_candidate["results"] = []cached_response_candidate["is_cached"] = True+ self._read_cache_entry_freshness(cached_response_candidate.pop(CACHE_ENTRY_FRESHNESS_KEY, None))# When rolling out schema changes, cached responses may not match the new schema.# Trigger recomputation in this case.try:@@ -2320,7 +2322,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):if custom_names_modified:# Update cache with patched response so subsequent requests get the updated namescache_manager.store_result(- response=cached_response.model_dump(),+ response=self._cache_entry_with_freshness(cached_response.model_dump()),target_age=cached_response.cache_target_age,)@@ -2837,6 +2839,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):**query_run.event_properties(self.modifiers),"cache_hit": isinstance(results, self.cached_response_type),"cache_age_override": self._cache_age_override,+ **self._staleness_event_properties(),"response_time_ms": query_run.elapsed_ms(),"clickhouse_rows_read": cached_query_scan.rows_read if cached_query_scan else None,"clickhouse_duration_ms": cached_query_scan.duration_ms if cached_query_scan else None,@@ -2899,7 +2902,6 @@ class QueryRunner(ABC, Generic[Q, R, CR]):CachedResponse: type[CR] = self.cached_response_typelast_refresh = datetime.now(UTC)- target_age = self.cache_target_age(last_refresh=last_refresh)# Avoid affecting cache key# Add user based modifiers here, primarily for user specific feature flagging@@ -2967,6 +2969,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):SURVEY_QUERY_EXECUTION_DURATION.labels(**survey_query_metric_labels).observe(query_duration_seconds)self._phase = RunPhase(name="finish", stats=query_stats, query_duration_ms=query_duration_ms)+ target_age = self.cache_target_age(last_refresh=last_refresh)fresh_response_dict: dict[str, Any] = {**query_result.model_dump(),"is_cached": False,@@ -3065,7 +3068,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):if cacheable:with self.timings.measure("cache_write"):stored = cache_manager.store_result(- response=fresh_response_dict,+ response=self._cache_entry_with_freshness(fresh_response_dict),# This would be a possible place to decide to not ever keep this cache warm# Example: Not for super quickly calculated insights# Set target_age to None in that case@@ -3091,6 +3094,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):"last_refresh": last_refresh.isoformat(),"cache_write_success": stored,"cache_age_override": getattr(self, "_cache_age_override", None),+ **self._staleness_event_properties(),"calculation_trigger": query_run.trigger,"response_time_ms": query_run.elapsed_ms(),"query_duration_ms": query_duration_ms,@@ -3453,6 +3457,21 @@ class QueryRunner(ABC, Generic[Q, R, CR]):return Truereturn datetime.now(UTC) > last_refresh + timedelta(seconds=override)+ def _staleness_event_properties(self) -> dict[str, Any]:+ return {}++ def _cache_entry_with_freshness(self, response: dict[str, Any]) -> dict[str, Any]:+ """Adds the runner's freshness decision to a cache entry before it is stored.++ A cache hit returns before the runner compiles its query, so the runner cannot work out+ freshness again on that request. A runner that decides freshness from the compiled query+ stores the decision under CACHE_ENTRY_FRESHNESS_KEY and reads it back on the hit.+ """+ return response++ def _read_cache_entry_freshness(self, record: Any) -> None:+ """Restores the decision that `_cache_entry_with_freshness` stored, before the response is parsed."""+def _is_stale(self, last_refresh: Optional[datetime], lazy: bool = False) -> bool:query_date_range = getattr(self, "query_date_range", None)date_to = query_date_range.date_to() if query_date_range else Nonediff --git a/posthog/hogql_queries/sql_freshness/__init__.py b/posthog/hogql_queries/sql_freshness/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/posthog/hogql_queries/sql_freshness/classifier.py b/posthog/hogql_queries/sql_freshness/classifier.pynew file mode 100644index 000000000000..246b5cf763dd--- /dev/null+++ b/posthog/hogql_queries/sql_freshness/classifier.py@@ -0,0 +1,63 @@+from datetime import UTC, datetime, tzinfo+from typing import Any, Optional++from dateutil.parser import isoparse++from posthog.hogql import ast++from posthog.dataclasses import frozen+from posthog.exceptions_capture import capture_exception+from posthog.hogql_queries.sql_freshness.facts import extract_facts+from posthog.hogql_queries.sql_freshness.rules import (+ FreshnessReason,+ Reject,+ SqlWindowClass,+ fresh_until,+ reject_reason,+ shortest_lived_window,+)+++@frozen+class SqlFreshness:+ window: SqlWindowClass+ fresh_until: datetime+ reason: FreshnessReason+++def classify(+ resolved_query: ast.SelectQuery | ast.SelectSetQuery, *, now: datetime, timezone: tzinfo+) -> SqlFreshness | Reject:+ try:+ facts = extract_facts(resolved_query, now=now, timezone=timezone)+ reason = reject_reason(facts)+ if reason is not None:+ return Reject(reason=reason)+ until, binding_reason = fresh_until(facts)+ window = shortest_lived_window(facts)+ except Exception as error:+ capture_exception(error, {"context": "sql_freshness_classification"})+ return Reject(reason=FreshnessReason.UNPARSEABLE)+ return SqlFreshness(window=window, fresh_until=until.astimezone(UTC), reason=binding_reason)+++def freshness_record(freshness: SqlFreshness) -> dict[str, str]:+ return {+ "window": freshness.window.value,+ "fresh_until": freshness.fresh_until.isoformat(),+ "reason": freshness.reason.value,+ }+++def freshness_from_record(record: Any) -> Optional[SqlFreshness]:+ try:+ fresh_until = isoparse(record["fresh_until"])+ if fresh_until.tzinfo is None:+ return None+ return SqlFreshness(+ window=SqlWindowClass(record["window"]),+ fresh_until=fresh_until,+ reason=FreshnessReason(record["reason"]),+ )+ except (AttributeError, KeyError, OverflowError, TypeError, ValueError):+ return Nonediff --git a/posthog/hogql_queries/sql_freshness/facts.py b/posthog/hogql_queries/sql_freshness/facts.pynew file mode 100644index 000000000000..6464964658ed--- /dev/null+++ b/posthog/hogql_queries/sql_freshness/facts.py@@ -0,0 +1,432 @@+from datetime import UTC, datetime, tzinfo+from typing import Any, Optional+from zoneinfo import ZoneInfo++from dateutil.parser import isoparse++from posthog.hogql import ast+from posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.groups import GroupsTable+from posthog.hogql.database.schema.person_distinct_ids import PersonDistinctIdsTable+from posthog.hogql.database.schema.persons import PersonsTable+from posthog.hogql.database.schema.sessions_v1 import SessionsTableV1+from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2+from posthog.hogql.database.schema.sessions_v3 import SessionsTableV3+from posthog.hogql.database.schema.util.where_clause_extractor import (+ contains_in_cohort,+ top_level_conjuncts,+ unwrap_table_alias_type,+)+from posthog.hogql.functions.clickhouse.datetime import DATE_GENERATOR_FUNCTIONS+from posthog.hogql.functions.mapping import find_hogql_function+from posthog.hogql.transforms.order_by_pushdown import unwrap_alias+from posthog.hogql.visitor import TraversingVisitor++from posthog.dataclasses import frozen+from posthog.interval_specs import INTERVAL_SPECS, PERIOD_MAP++_SUB_DAY_UNITS = frozenset({"second", "minute", "hour"})+_JOINABLE_TABLES = (+ PersonsTable,+ PersonDistinctIdsTable,+ GroupsTable,+ SessionsTableV1,+ SessionsTableV2,+ SessionsTableV3,+)+_EXPRESSION_FIELDS = frozenset({"person_id"})+_TIMESTAMP_COLUMN = "timestamp"+_LATEST_EVENT_FUNCTION_PREFIXES = ("max", "argmax", "anylast", "last_value")+_ZONE_ARGUMENT_POSITION = {"toDateTime": 1, "toDateTime64": 2, "parseDateTimeBestEffort": 1}+_NOW_FUNCTIONS = frozenset({"now", "now64"})+_EARLIEST_DATETIME = datetime(1970, 1, 1, tzinfo=UTC)+_LATEST_DATETIME = datetime(2106, 2, 7, 6, 28, 15, tzinfo=UTC)+_CLOCK_FUNCTIONS = {id(meta): meta.clickhouse_name for meta in DATE_GENERATOR_FUNCTIONS.values()} | {+ id(find_hogql_function("nowInBlock")): "nowInBlock"+}+_LOWER_BOUND_OPS = frozenset({ast.CompareOperationOp.Gt, ast.CompareOperationOp.GtEq})+_FLIPPED_OPS = {+ ast.CompareOperationOp.Gt: ast.CompareOperationOp.Lt,+ ast.CompareOperationOp.GtEq: ast.CompareOperationOp.LtEq,+ ast.CompareOperationOp.Lt: ast.CompareOperationOp.Gt,+ ast.CompareOperationOp.LtEq: ast.CompareOperationOp.GtEq,+ ast.CompareOperationOp.Eq: ast.CompareOperationOp.Eq,+}+++@frozen+class Window:+ start: datetime+ end: Optional[datetime]+++@frozen+class _Bound:+ start: Optional[datetime]+ end: Optional[datetime]+++@frozen+class QueryFacts:+ now: datetime+ start_of_today: datetime+ windows: tuple[Window, ...]+ unsupported_field: bool+ join: bool+ foreign_source: bool+ unbounded_select: bool+ unsupported_bound: bool+ clock_outside_bounds: bool+ reads_now: bool+ reads_latest_event: bool+++class _UnsupportedBound(Exception):+ pass+++def _clock_name(call: ast.Call) -> Optional[str]:+ meta = find_hogql_function(call.name)+ return _CLOCK_FUNCTIONS.get(id(meta)) if meta is not None else None+++def _is_events_table(table_type: Any) -> bool:+ table_type = unwrap_table_alias_type(table_type)+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)+++def _is_subquery_type(table_type: Any) -> bool:+ return isinstance(+ table_type, ast.SelectQueryType | ast.SelectSetQueryType | ast.SelectQueryAliasType | ast.CTETableType+ )+++def _reads_supported_table(table_type: Any) -> bool:+ if isinstance(table_type, ast.VirtualTableType):+ return _reads_supported_table(table_type.table_type)+ if isinstance(table_type, ast.LazyJoinType):+ return isinstance(table_type.lazy_join.join_table, _JOINABLE_TABLES) and _reads_supported_table(+ table_type.table_type+ )+ return _is_events_table(table_type)+++def _is_supported_field_type(field_type: Any) -> bool:+ if isinstance(field_type, ast.FieldAliasType):+ return True+ if isinstance(field_type, ast.PropertyType):+ return _is_supported_field_type(field_type.field_type)+ if isinstance(field_type, ast.ExpressionFieldType):+ return field_type.name in _EXPRESSION_FIELDS and _reads_supported_table(field_type.table_type)+ if not isinstance(field_type, ast.FieldType):+ return False+ return _is_subquery_type(field_type.table_type) or _reads_supported_table(field_type.table_type)+++def _is_events_timestamp(expr: ast.Expr) -> bool:+ expr = unwrap_alias(expr)+ field_type = expr.type if isinstance(expr, ast.Field) else None+ return (+ isinstance(field_type, ast.FieldType)+ and field_type.name == _TIMESTAMP_COLUMN+ and _is_events_table(field_type.table_type)+ )+++def _int_constant(expr: ast.Expr) -> Optional[int]:+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):+ return expr.value+ return None+++def _string_constant(expr: ast.Expr) -> Optional[str]:+ return expr.value if isinstance(expr, ast.Constant) and isinstance(expr.value, str) else None+++def _parse_instant(value: Optional[str], timezone: tzinfo) -> Optional[datetime]:+ if value is None or not (len(value) >= 10 and value[4] == "-" and value[7] == "-" and value[:4].isdigit()):+ return None+ try:+ parsed = isoparse(value)+ if parsed.tzinfo:+ return parsed.astimezone(timezone)+ local = parsed.replace(tzinfo=timezone)+ return local if _wall_time_exists(local) else None+ except (OverflowError, ValueError):+ return None+++def _wall_time_exists(at: datetime) -> bool:+ return at.astimezone(UTC).astimezone(at.tzinfo).replace(tzinfo=None) == at.replace(tzinfo=None)+++def _repeats_wall_time(at: datetime, timezone: tzinfo) -> bool:+ return any(at.astimezone(zone).fold == 1 for zone in (timezone, at.tzinfo))+++def _shifted(at: datetime, unit: str, amount: int) -> Optional[datetime]:+ if unit in _SUB_DAY_UNITS:+ return (at.astimezone(UTC) + PERIOD_MAP[unit] * amount).astimezone(at.tzinfo)+ moved = at + PERIOD_MAP[unit] * amount+ return moved if _wall_time_exists(moved) else None+++class _TimestampFinder(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.found = False++ def visit_field(self, node: ast.Field) -> None:+ if _is_events_timestamp(node):+ self.found = True++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ pass+++def _mentions_events_timestamp(expr: ast.Expr) -> bool:+ finder = _TimestampFinder()+ finder.visit(expr)+ return finder.found+++def _subquery_column_types(table_type: Any, name: str) -> list[ast.Type]:+ if isinstance(table_type, ast.SelectQueryAliasType | ast.CTETableType | ast.SelectViewType):+ return _subquery_column_types(table_type.select_query_type, name)+ if isinstance(table_type, ast.SelectSetQueryType):+ return [column for query_type in table_type.types for column in _subquery_column_types(query_type, name)]+ if isinstance(table_type, ast.SelectQueryType) and name in table_type.columns:+ return [table_type.columns[name]]+ return []+++class _DerivedTimestampFinder(TraversingVisitor):+ def __init__(self, alias_exprs: dict[int, ast.Expr]) -> None:+ super().__init__()+ self.alias_exprs = alias_exprs+ self.followed: set[int] = set()+ self.found = False++ def visit_field(self, node: ast.Field) -> None:+ self.found = self.found or self._derives_from_events_timestamp(node.type)++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ pass++ def _derives_from_events_timestamp(self, field_type: Any) -> bool:+ if id(field_type) in self.followed:+ return False+ self.followed.add(id(field_type))+ if isinstance(field_type, ast.FieldAliasType):+ alias_expr = self.alias_exprs.get(id(field_type))+ if alias_expr is not None:+ self.visit(alias_expr)+ return self.found+ return self._derives_from_events_timestamp(field_type.type)+ if not isinstance(field_type, ast.FieldType):+ return False+ if field_type.name == _TIMESTAMP_COLUMN and _is_events_table(field_type.table_type):+ return True+ return any(+ self._derives_from_events_timestamp(column)+ for column in _subquery_column_types(field_type.table_type, field_type.name)+ )+++class _QueryScan(TraversingVisitor):+ def __init__(self) -> None:+ super().__init__()+ self.selects: list[ast.SelectQuery] = []+ self.clock_calls: set[int] = set()+ self.unsupported_field = False+ self.alias_exprs: dict[int, ast.Expr] = {}+ self.latest_event_candidates: list[ast.Expr] = []++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ self.selects.append(node)+ for order in node.order_by or []:+ position = _int_constant(order.expr)+ if order.order == "DESC" and position is not None and 0 < position <= len(node.select):+ self.latest_event_candidates.append(node.select[position - 1])+ super().visit_select_query(node)++ def visit_field(self, node: ast.Field) -> None:+ if not isinstance(node.type, ast.BaseTableType) and not _is_supported_field_type(node.type):+ self.unsupported_field = True++ def visit_call(self, node: ast.Call) -> None:+ if _clock_name(node) is not None:+ self.clock_calls.add(id(node))+ if node.name.lower().startswith(_LATEST_EVENT_FUNCTION_PREFIXES):+ self.latest_event_candidates.extend(node.args)+ super().visit_call(node)++ def visit_window_function(self, node: ast.WindowFunction) -> None:+ if node.name.lower().startswith(_LATEST_EVENT_FUNCTION_PREFIXES):+ window_order = node.over_expr.order_by if node.over_expr is not None else None+ self.latest_event_candidates.extend(+ [*(node.exprs or []), *(node.args or []), *(order.expr for order in window_order or [])]+ )+ super().visit_window_function(node)++ def visit_order_expr(self, node: ast.OrderExpr) -> None:+ if node.order == "DESC":+ self.latest_event_candidates.append(node.expr)+ super().visit_order_expr(node)++ def visit_alias(self, node: ast.Alias) -> None:+ self.alias_exprs[id(node.type)] = node.expr+ super().visit_alias(node)++ def reads_latest_event(self) -> bool:+ finder = _DerivedTimestampFinder(self.alias_exprs)+ for candidate in self.latest_event_candidates:+ finder.visit(candidate)+ return finder.found+++class _Evaluator:+ def __init__(self, *, now: datetime, timezone: tzinfo) -> None:+ self.now = now+ self.timezone = timezone+ self.consumed_clock_calls: set[int] = set()+ self.reads_now = False++ def evaluate(self, expr: ast.Expr) -> Optional[datetime]:+ at = self._value(unwrap_alias(expr))+ return at if at is not None and _EARLIEST_DATETIME <= at <= _LATEST_DATETIME else None++ def _value(self, expr: ast.Expr) -> Optional[datetime]:+ if isinstance(expr, ast.Constant):+ return _parse_instant(_string_constant(expr), self.timezone)+ if isinstance(expr, ast.Call):+ return self._call(expr)+ if isinstance(expr, ast.ArithmeticOperation) and expr.op in (+ ast.ArithmeticOperationOp.Add,+ ast.ArithmeticOperationOp.Sub,+ ):+ if isinstance(unwrap_alias(expr.left), ast.Constant):+ return None+ base = self.evaluate(expr.left)+ step = _interval(expr.right)+ if base is None or step is None:+ return None+ unit, amount = step+ try:+ return _shifted(base, unit, -amount if expr.op == ast.ArithmeticOperationOp.Sub else amount)+ except (OverflowError, ValueError):+ return None+ return None++ def _call(self, call: ast.Call) -> Optional[datetime]:+ if _clock_name(call) in _NOW_FUNCTIONS and not call.args:+ self.consumed_clock_calls.add(id(call))+ self.reads_now = True+ return self.now+ epoch = _int_constant(call.args[0]) if len(call.args) == 1 else None+ if epoch is not None and call.name in ("toDateTime", "fromUnixTimestamp"):+ timezone = self.timezone if call.name == "toDateTime" else UTC+ try:+ return datetime.fromtimestamp(epoch, timezone)+ except (OverflowError, ValueError, OSError):+ return None+ zone_position = _ZONE_ARGUMENT_POSITION.get(call.name)+ if zone_position is None or not call.args or len(call.args) > zone_position + 1:+ return None+ if call.name == "toDateTime64" and len(call.args) > 1 and _int_constant(call.args[1]) is None:+ return None+ zone_name = _string_constant(call.args[zone_position]) if len(call.args) > zone_position else None+ if len(call.args) > zone_position and not zone_name:+ return None+ try:+ timezone = ZoneInfo(zone_name) if zone_name else self.timezone+ except (KeyError, ValueError):+ return None+ return _parse_instant(_string_constant(call.args[0]), timezone)+++def _interval(expr: ast.Expr) -> Optional[tuple[str, int]]:+ if isinstance(expr, ast.Call) and expr.name.startswith("toInterval") and len(expr.args) == 1:+ unit = expr.name.removeprefix("toInterval").lower()+ amount = _int_constant(expr.args[0])+ if unit in PERIOD_MAP and amount is not None:+ return unit, amount+ return None+++def _bound(+ evaluator: _Evaluator, column: ast.Expr, op: ast.CompareOperationOp, value_expr: ast.Expr+) -> Optional[_Bound]:+ if not _is_events_timestamp(column):+ return None+ at = evaluator.evaluate(value_expr)+ if at is None or _repeats_wall_time(at, evaluator.timezone):+ raise _UnsupportedBound()+ if op == ast.CompareOperationOp.Eq:+ return _Bound(start=at, end=at)+ return _Bound(start=at, end=None) if op in _LOWER_BOUND_OPS else _Bound(start=None, end=at)+++def _bounds_from_conjunct(evaluator: _Evaluator, conjunct: ast.Expr) -> _Bound:+ if isinstance(conjunct, ast.CompareOperation) and conjunct.op in _FLIPPED_OPS:+ bound = _bound(evaluator, conjunct.left, conjunct.op, conjunct.right)+ if bound is None:+ bound = _bound(evaluator, conjunct.right, _FLIPPED_OPS[conjunct.op], conjunct.left)+ if bound is not None:+ return bound+ raise _UnsupportedBound()+++def _select_bounds(evaluator: _Evaluator, select: ast.SelectQuery) -> list[_Bound]:+ return [+ _bounds_from_conjunct(evaluator, conjunct)+ for clause in (select.where, select.prewhere)+ if clause is not None+ for conjunct in top_level_conjuncts(clause)+ if _mentions_events_timestamp(conjunct)+ ]+++def extract_facts(+ resolved_query: ast.SelectQuery | ast.SelectSetQuery, *, now: datetime, timezone: tzinfo+) -> QueryFacts:+ now = now.astimezone(timezone)+ scan = _QueryScan()+ scan.visit(resolved_query)+ evaluator = _Evaluator(now=now, timezone=timezone)+ windows: list[Window] = []+ has_join = foreign_source = unbounded_select = unsupported_bound = False+ for select in scan.selects:+ join = select.select_from+ if join is None:+ continue+ has_join = has_join or join.next_join is not None+ if _is_subquery_type(join.type):+ continue+ if not _is_events_table(join.type):+ foreign_source = True+ continue+ try:+ bounds = _select_bounds(evaluator, select)+ except _UnsupportedBound:+ unsupported_bound = True+ continue+ starts = [bound.start.astimezone(UTC) for bound in bounds if bound.start is not None]+ ends = [bound.end.astimezone(UTC) for bound in bounds if bound.end is not None]+ if not starts:+ unbounded_select = True+ continue+ windows.append(Window(start=max(starts), end=min(ends) if ends else None))+ return QueryFacts(+ now=now.astimezone(UTC),+ start_of_today=INTERVAL_SPECS["day"].align(now, None),+ windows=tuple(windows),+ unsupported_field=scan.unsupported_field,+ join=has_join,+ foreign_source=foreign_source or contains_in_cohort(resolved_query),+ unbounded_select=unbounded_select or not windows,+ unsupported_bound=unsupported_bound,+ clock_outside_bounds=bool(scan.clock_calls - evaluator.consumed_clock_calls),+ reads_now=evaluator.reads_now,+ reads_latest_event=scan.reads_latest_event(),+ )diff --git a/posthog/hogql_queries/sql_freshness/rules.py b/posthog/hogql_queries/sql_freshness/rules.pynew file mode 100644index 000000000000..14ef0e7ac984--- /dev/null+++ b/posthog/hogql_queries/sql_freshness/rules.py@@ -0,0 +1,95 @@+from datetime import datetime, timedelta+from enum import StrEnum+from typing import Optional++from posthog.dataclasses import frozen+from posthog.hogql_queries.sql_freshness.facts import QueryFacts, Window+++class SqlWindowClass(StrEnum):+ FIXED_PAST = "fixed_past"+ LIVE = "live"+ UP_TO_NOW_SHORT = "up_to_now_short"+ UP_TO_NOW_LONG = "up_to_now_long"+++SQL_WINDOW_MAX_AGE: dict[SqlWindowClass, timedelta] = {+ SqlWindowClass.FIXED_PAST: timedelta(hours=6),+ SqlWindowClass.LIVE: timedelta(minutes=5),+ SqlWindowClass.UP_TO_NOW_SHORT: timedelta(minutes=30),+ SqlWindowClass.UP_TO_NOW_LONG: timedelta(hours=1),+}++LIVE_WINDOW_MAX_LENGTH = timedelta(hours=1)+SHORT_WINDOW_MAX_LENGTH = timedelta(days=1)+FIXED_PAST_SETTLE_TIME = timedelta(hours=1)+++class FreshnessReason(StrEnum):+ WINDOW_AGE = "window_age"+ NOW_ANCHORED = "now_anchored"+ LATEST_EVENT = "latest_event"+ NO_EVENTS = "no_events"+ UNSUPPORTED_FIELD = "unsupported_field"+ JOIN = "join"+ FOREIGN_SOURCE = "foreign_source"+ UNBOUNDED = "unbounded"+ UNSUPPORTED_BOUND = "unsupported_bound"+ CLOCK_OUTSIDE_BOUNDS = "clock_outside_bounds"+ UNPARSEABLE = "unparseable"+ UNSUPPORTED_PLACEHOLDER = "unsupported_placeholder"+++@frozen+class Reject:+ reason: FreshnessReason+++def window_class(window: Window, facts: QueryFacts) -> SqlWindowClass:+ if (+ window.end is not None+ and window.end <= facts.start_of_today+ and window.end <= facts.now - FIXED_PAST_SETTLE_TIME+ ):+ return SqlWindowClass.FIXED_PAST+ return _class_for_length(facts.now - window.start)+++def _class_for_length(length: timedelta) -> SqlWindowClass:+ if length <= LIVE_WINDOW_MAX_LENGTH:+ return SqlWindowClass.LIVE+ if length <= SHORT_WINDOW_MAX_LENGTH:+ return SqlWindowClass.UP_TO_NOW_SHORT+ return SqlWindowClass.UP_TO_NOW_LONG+++def shortest_lived_window(facts: QueryFacts) -> SqlWindowClass:+ return min((window_class(window, facts) for window in facts.windows), key=SQL_WINDOW_MAX_AGE.__getitem__)+++def reject_reason(facts: QueryFacts) -> Optional[FreshnessReason]:+ checks = (+ (facts.unsupported_field, FreshnessReason.UNSUPPORTED_FIELD),+ (facts.join, FreshnessReason.JOIN),+ (facts.foreign_source, FreshnessReason.FOREIGN_SOURCE),+ (facts.unsupported_bound, FreshnessReason.UNSUPPORTED_BOUND),+ (facts.unbounded_select, FreshnessReason.UNBOUNDED),+ (facts.clock_outside_bounds, FreshnessReason.CLOCK_OUTSIDE_BOUNDS),+ )+ return next((reason for failed, reason in checks if failed), None)+++def fresh_until(facts: QueryFacts) -> tuple[datetime, FreshnessReason]:+ """Every limit that applies holds at once, so the earliest one sets the expiry."""+ limits = [(facts.now + SQL_WINDOW_MAX_AGE[shortest_lived_window(facts)], FreshnessReason.WINDOW_AGE)]+ if facts.reads_now:+ limits += [+ (facts.now + SQL_WINDOW_MAX_AGE[_class_for_length(window.end - window.start)], FreshnessReason.NOW_ANCHORED)+ for window in facts.windows+ if window.end is not None and window_class(window, facts) == SqlWindowClass.FIXED_PAST+ ]+ if facts.reads_latest_event and any(+ window_class(window, facts) != SqlWindowClass.FIXED_PAST for window in facts.windows+ ):+ limits.append((facts.now + SQL_WINDOW_MAX_AGE[SqlWindowClass.LIVE], FreshnessReason.LATEST_EVENT))+ return min(limits, key=lambda limit: limit[0])diff --git a/posthog/hogql_queries/sql_freshness/test/test_classifier.py b/posthog/hogql_queries/sql_freshness/test/test_classifier.pynew file mode 100644index 000000000000..5fa9f9e29d4f--- /dev/null+++ b/posthog/hogql_queries/sql_freshness/test/test_classifier.py@@ -0,0 +1,395 @@+from datetime import UTC, datetime, timedelta+from typing import Any+from zoneinfo import ZoneInfo++from posthog.test.base import APIBaseTest, BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events++from parameterized import parameterized++from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.parser import parse_select+from posthog.hogql.query import execute_hogql_query+from posthog.hogql.resolver import resolve_types++from posthog.hogql_queries.sql_freshness.classifier import SqlFreshness, classify, freshness_from_record+from posthog.hogql_queries.sql_freshness.facts import extract_facts+from posthog.hogql_queries.sql_freshness.rules import SQL_WINDOW_MAX_AGE, FreshnessReason, Reject, SqlWindowClass++NOW = datetime(2026, 10, 29, 15, 0, tzinfo=UTC)+AMSTERDAM = ZoneInfo("Europe/Amsterdam")+PAST_DAY = "select count() from events where timestamp >= '2026-10-28' and timestamp < '2026-10-29'"+ACCEPTED_BOUNDS = [+ ("date_literal", "'2026-03-29'", "Europe/Amsterdam"),+ ("datetime_literal", "'2026-03-29 03:30:00'", "Europe/Amsterdam"),+ ("utc_literal", "'2026-10-29T08:00:00Z'", "Europe/Amsterdam"),+ ("zoned_literal", "toDateTime('2026-10-29 08:00:00', 'Asia/Tokyo')", "UTC"),+ ("zoned_datetime64", "toDateTime64('2026-10-29 08:00:00', 6, 'Asia/Tokyo')", "UTC"),+ ("datetime64_with_precision_only", "toDateTime64('2026-10-29 08:00:00', 3)", "Europe/Amsterdam"),+ ("best_effort_with_offset", "parseDateTimeBestEffort('2026-10-29T08:00:00Z')", "Europe/Amsterdam"),+ ("best_effort_with_zone", "parseDateTimeBestEffort('2026-10-29 08:00:00', 'Asia/Tokyo')", "UTC"),+ ("epoch", "toDateTime(1761955200)", "Europe/Amsterdam"),+ ("from_unix_timestamp", "fromUnixTimestamp(1761955200)", "Europe/Amsterdam"),+ ("epoch_plus_a_day_across_a_fall_back", "toDateTime(1792836000) + interval 1 day", "Europe/Amsterdam"),+ ("from_unix_timestamp_plus_a_month", "fromUnixTimestamp(1769815800) + interval 1 month", "Europe/Amsterdam"),+ ("day_interval_across_a_fall_back", "toDateTime('2026-10-24 12:00:00') + interval 1 day", "Europe/Amsterdam"),+ ("hour_interval_across_a_fall_back", "toDateTime('2026-10-25 01:30:00') + interval 3 hour", "Europe/Amsterdam"),+ ("month_interval_at_a_month_end", "toDateTime('2026-01-31 12:00:00') + interval 1 month", "Europe/Amsterdam"),+ ("wall_time_in_a_dst_fold", "'2026-10-25 02:30:00'", "Europe/Amsterdam"),+ (+ "hour_interval_across_a_spring_forward",+ "toDateTime('2026-03-29 01:30:00') + interval 1 hour",+ "Europe/Amsterdam",+ ),+ ("now_minus_hours", "now() - interval 6 hour", "Europe/Amsterdam"),+ ("now_minus_days", "now() - interval 7 day", "Europe/Amsterdam"),+]+++def _resolve(sql: str, team: Any) -> Any:+ context = HogQLContext(team_id=team.pk, database=Database.create_for(team=team), enable_select_queries=True)+ return resolve_types(parse_select(sql), context, dialect="hogql")+++class TestClassify(BaseTest):+ def _classify(self, sql: str, now: datetime = NOW, timezone: Any = None) -> SqlFreshness | Reject:+ return classify(_resolve(sql, self.team), now=now, timezone=timezone or self.team.timezone_info)++ @parameterized.expand(+ [+ (+ "last_30_minutes",+ "select count() from events where timestamp > now() - interval 30 minute",+ SqlWindowClass.LIVE,+ ),+ (+ "last_6_hours_with_alias",+ "select count() as c from events where timestamp >= now() - interval 6 hour order by c",+ SqlWindowClass.UP_TO_NOW_SHORT,+ ),+ (+ "last_7_days",+ "select event, count() from events where timestamp > now() - interval 7 day group by event",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "cte_of_events",+ "with recent as (select event from events where timestamp > now() - interval 7 day) "+ "select count() from recent",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "or_without_time",+ "select count() from events where (event = 'a' or event = 'b') and timestamp > now() - interval 2 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "person_id",+ "select uniq(person_id) from events where timestamp > now() - interval 7 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "person_properties",+ "select count() from events where timestamp > now() - interval 7 day and person.properties.email = 'a'",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "group_properties",+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "+ "and group_0.properties.plan = 'x'",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "session_properties",+ "select count() from events where timestamp > now() - interval 2 day and session.$session_duration > 10",+ SqlWindowClass.UP_TO_NOW_LONG,+ ),+ (+ "absolute_past_range",+ "select count() from events where timestamp >= '2026-10-01' and timestamp <= '2026-10-15'",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "zoned_literal_uses_its_timezone",+ "select count() from events where timestamp >= '2026-10-01' "+ "and timestamp < toDateTime('2026-10-29 08:00:00', 'Asia/Tokyo')",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "epoch_seconds",+ "select count() from events where timestamp >= toDateTime(1759276800) "+ "and timestamp < fromUnixTimestamp(1760486400)",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "literal_plus_days",+ "select count() from events where timestamp >= toDateTime('2026-10-01 00:00:00') "+ "and timestamp < toDateTime('2026-10-01 00:00:00') + interval 14 day",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "now_anchored_past_window_moves",+ "select count() from events where timestamp > now() - interval 9 day and timestamp < now() - interval 2 day",+ SqlWindowClass.FIXED_PAST,+ {"fresh_until": NOW + timedelta(hours=1)},+ ),+ (+ "narrow_now_anchored_window_expires_within_its_width",+ "select count() from events where timestamp >= now() - interval 25 hour and timestamp < now() - interval 24 hour",+ SqlWindowClass.FIXED_PAST,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_event_time_up_to_now",+ "select max(timestamp) from events where timestamp > now() - interval 2 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_rows_up_to_now",+ "select event from events where timestamp > now() - interval 7 day order by timestamp desc limit 10",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_rows_by_an_alias",+ "select timestamp as ts from events where timestamp > now() - interval 7 day order by ts desc limit 10",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_value_by_time",+ "select argMax(event, timestamp) from events where timestamp > now() - interval 7 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_event_time_with_a_condition",+ "select maxIf(timestamp, event = 'signup') from events where timestamp > now() - interval 7 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_value_in_a_window_function",+ "select last_value(event) over (partition by person_id order by timestamp) from events "+ "where timestamp > now() - interval 7 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_rows_by_position",+ "select timestamp, event from events where timestamp > now() - interval 7 day order by 1 desc limit 10",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_event_time_through_a_subquery",+ "select max(ts) from (select toStartOfMinute(timestamp) as ts from events "+ "where timestamp > now() - interval 7 day)",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"fresh_until": NOW + timedelta(minutes=5)},+ ),+ (+ "latest_event_time_in_a_fixed_past_range",+ "select max(timestamp) from events where timestamp >= '2026-10-01' and timestamp < '2026-10-15'",+ SqlWindowClass.FIXED_PAST,+ ),+ (+ "union_takes_the_freshest",+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "+ "union all select count() from events where timestamp > now() - interval 10 minute",+ SqlWindowClass.LIVE,+ ),+ (+ "unsettled_past_range",+ PAST_DAY,+ SqlWindowClass.UP_TO_NOW_LONG,+ {"now": datetime(2026, 10, 29, 0, 30, tzinfo=UTC)},+ ),+ (+ "age_counts_elapsed_time_across_a_dst_fall_back",+ "select count() from events where timestamp >= now() - interval 7 day",+ SqlWindowClass.UP_TO_NOW_LONG,+ {"timezone": AMSTERDAM, "now": datetime(2026, 10, 25, 0, 30, tzinfo=UTC)},+ ),+ (+ "hour_interval_counts_elapsed_time_across_a_dst_fall_back",+ "select count() from events where timestamp >= now() - interval 1 hour",+ SqlWindowClass.LIVE,+ {"timezone": AMSTERDAM, "now": datetime(2026, 10, 25, 1, 30, tzinfo=UTC)},+ ),+ ]+ )+ def test_grants_window_age(+ self, _name: str, sql: str, expected_window: SqlWindowClass, options: dict[str, Any] | None = None+ ) -> None:+ options = {"now": NOW, **(options or {})}+ expected_fresh_until = options.pop("fresh_until", options["now"] + SQL_WINDOW_MAX_AGE[expected_window])+ freshness = self._classify(sql, **options)+ assert isinstance(freshness, SqlFreshness)+ assert (freshness.window, freshness.fresh_until) == (expected_window, expected_fresh_until)++ @parameterized.expand(+ [+ ("no_time_filter", "select count() from events", FreshnessReason.UNBOUNDED),+ ("upper_bound_only", "select count() from events where timestamp < now()", FreshnessReason.UNBOUNDED),+ (+ "revenue_join",+ "select poe.revenue_analytics.revenue from events "+ "where timestamp >= '2025-01-01' and timestamp < '2025-02-01'",+ FreshnessReason.UNSUPPORTED_FIELD,+ ),+ (+ "clock_in_select_list",+ "select now(), count() from events where timestamp < '2025-01-01' and timestamp >= '2024-01-01'",+ FreshnessReason.CLOCK_OUTSIDE_BOUNDS,+ ),+ (+ "or_of_time_ranges",+ "select count() from events where timestamp > now() - interval 7 day or timestamp < '2020-01-01'",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "today",+ "select count() from events where toDate(timestamp) = today()",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "yesterday",+ "select count() from events where timestamp >= yesterday()",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "truncated_column",+ "select count() from events where toStartOfMonth(timestamp) < '2026-10-01' and timestamp >= '2026-09-01'",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "truncated_value",+ "select count() from events where timestamp >= toStartOfDay(now())",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "shift_function",+ "select count() from events where timestamp >= subtractDays(now(), 3)",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "integer_offset",+ "select count() from events where timestamp >= now() - 3600",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "date_value",+ "select count() from events where timestamp >= '2026-10-01' and timestamp < toDate('2026-10-27')",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "timezone_on_a_non_literal",+ "select count() from events where timestamp < toDateTime(now(), 'America/Los_Angeles') "+ "and timestamp >= '2026-01-01'",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "join_after_a_subquery",+ "select count() from (select uuid from events where timestamp >= '2026-01-01' "+ "and timestamp < '2026-02-01') e cross join persons p",+ FreshnessReason.JOIN,+ ),+ (+ "join",+ "select count() from events e join events f on e.uuid = f.uuid where e.timestamp > '2026-01-01'",+ FreshnessReason.JOIN,+ ),+ (+ "persons_table",+ "select count() from persons where created_at < '2025-01-01'",+ FreshnessReason.UNSUPPORTED_FIELD,+ ),+ (+ "cohort_membership",+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "+ "and person_id not in cohort 1",+ FreshnessReason.FOREIGN_SOURCE,+ ),+ (+ "string_plus_interval",+ "select count() from events where timestamp >= '2026-11-01' and timestamp < '2026-11-01' + interval 1 day",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "before_the_datetime_range",+ "select count() from events where timestamp > toDateTime(0) - interval 1 second",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "base_before_the_datetime_range",+ "select count() from events where timestamp >= toDateTime('1960-01-01 00:00:00') + interval 20 year",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "after_the_datetime_range",+ "select count() from events where timestamp >= toDateTime('2100-01-01 00:00:00') + interval 10 year",+ FreshnessReason.UNSUPPORTED_BOUND,+ ),+ (+ "literal_at_the_year_limit",+ "select count() from events where timestamp >= '9999-12-31 23:00:00'",+ FreshnessReason.UNSUPPORTED_BOUND,+ ZoneInfo("America/New_York"),+ ),+ (+ "wall_time_in_a_dst_gap",+ "select count() from events where timestamp >= '2026-03-29 02:30:00'",+ FreshnessReason.UNSUPPORTED_BOUND,+ AMSTERDAM,+ ),+ (+ "hour_interval_into_a_repeated_dst_hour",+ "select count() from events where timestamp >= toDateTime('2026-10-25 01:30:00') + interval 2 hour",+ FreshnessReason.UNSUPPORTED_BOUND,+ AMSTERDAM,+ ),+ (+ "calendar_shift_into_a_dst_gap",+ "select count() from events where timestamp >= toDateTime('2026-03-28 02:30:00') + interval 1 day",+ FreshnessReason.UNSUPPORTED_BOUND,+ AMSTERDAM,+ ),+ ]+ )+ def test_keeps_default_age(+ self, _name: str, sql: str, expected_reason: FreshnessReason, timezone: Any = None+ ) -> None:+ assert self._classify(sql, timezone=timezone) == Reject(reason=expected_reason)++ def test_record_without_a_timezone_is_missing(self) -> None:+ assert (+ freshness_from_record({"window": "live", "fresh_until": "2026-10-29T15:05:00", "reason": "window_age"})+ is None+ )+++class TestAcceptedBoundsMatchClickHouse(ClickhouseTestMixin, APIBaseTest):+ @parameterized.expand(+ [+ (f"{name}_{side}", value, timezone, side)+ for name, value, timezone in ACCEPTED_BOUNDS+ for side in ("start", "end")+ ]+ )+ def test_bound_matches_clickhouse(self, _name: str, value: str, timezone: str, side: str) -> None:+ self.team.timezone = timezone+ self.team.save()+ bound = f"{value} <= timestamp" if side == "start" else f"timestamp > toDateTime(0) and timestamp < {value}"+ sql = f"select count() from events where event = 'bound_check' and {bound}"+ window = extract_facts(+ _resolve(sql, self.team), now=datetime.now(UTC), timezone=self.team.timezone_info+ ).windows[0]+ at = window.start if side == "start" else window.end+ assert at is not None+ margin = timedelta(seconds=30 if "now()" in value else 1)+ for offset in (-margin, margin):+ _create_event(team=self.team, event="bound_check", distinct_id="d", timestamp=at + offset)+ flush_persons_and_events()+ assert execute_hogql_query(sql, team=self.team).results == [(1,)]diff --git a/posthog/hogql_queries/test/test_hogql_query_runner.py b/posthog/hogql_queries/test/test_hogql_query_runner.pyindex 54071e1f634d..9353bdc07552 100644--- a/posthog/hogql_queries/test/test_hogql_query_runner.py+++ b/posthog/hogql_queries/test/test_hogql_query_runner.py@@ -1,13 +1,25 @@-from datetime import UTC, datetime-from typing import cast--from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events+from datetime import UTC, datetime, timedelta+from typing import TYPE_CHECKING, cast++import time_machine+from posthog.test.base import (+ APIBaseTest,+ BaseTest,+ ClickhouseTestMixin,+ _create_event,+ _create_person,+ flush_persons_and_events,+)from unittest.mock import MagicMock, patch+if TYPE_CHECKING:+ from unittest.mock import _patch+from parameterized import parameterizedfrom posthog.schema import (CachedHogQLQueryResponse,+ DateRange,HogQLFilters,HogQLPropertyFilter,HogQLQuery,@@ -26,8 +38,11 @@ from posthog.hogql.user_query_validator import HOGQL_PERSONAL_API_KEY_OFFSET_ALLfrom posthog.hogql.visitor import clear_locationsfrom posthog.caching.utils import ThresholdMode, staleness_threshold_map-from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner-from posthog.hogql_queries.query_runner import ExecutionMode+from posthog.clickhouse.query_tagging import tags_context+from posthog.hogql_queries.hogql_query_runner import API_SQL_WINDOW_CACHE_AGE_FLAG, HogQLQueryRunner+from posthog.hogql_queries.query_runner import CACHE_ENTRY_FRESHNESS_KEY, ExecutionMode+from posthog.hogql_queries.sql_freshness.classifier import SqlFreshness, freshness_record+from posthog.hogql_queries.sql_freshness.rules import FreshnessReason, Reject, SqlWindowClassfrom posthog.models.utils import UUIDTfrom products.managed_warehouse.backend.facade.query_labels import MANAGED_WAREHOUSE_QUERY_STATUS_LABEL_PREFIX@@ -35,6 +50,15 @@ from products.product_analytics.backend.facade.models import InsightVariablefrom products.warehouse_sources.backend.facade.models import MANAGED_WAREHOUSE_SOURCE_PREFIX, ExternalDataSourcefrom products.warehouse_sources.backend.facade.types import ExternalDataSourceType+PAST_RANGE = "select count() from events where timestamp >= '2026-10-01' and timestamp < '2026-10-28'"+++def _freshness_flag(on: bool) -> "_patch[MagicMock]":+ return patch(+ "posthog.hogql_queries.query_runner.posthoganalytics.feature_enabled",+ side_effect=lambda key, *args, **kwargs: on and key == API_SQL_WINDOW_CACHE_AGE_FLAG,+ )+class TestHogQLQueryRunner(ClickhouseTestMixin, APIBaseTest):maxDiff = None@@ -573,3 +597,148 @@ class TestHogQLQueryRunner(ClickhouseTestMixin, APIBaseTest):response = runner.calculate()self.assertEqual(len(response.results), 5)++ def test_api_key_result_stays_cached_for_its_window(self) -> None:+ query = HogQLQuery(+ query=f"select count() from events where timestamp > now() - interval 7 day and event != '{self.random_uuid}'"+ )+ with (+ _freshness_flag(on=True),+ tags_context(access_method="personal_api_key"),+ patch("posthog.hogql_queries.query_runner.report_user_or_team_action") as report,+ ):+ with time_machine.travel("2026-10-29T15:00:00Z", tick=False):+ first = self._create_runner(query).run()+ first_event = report.call_args.args[1]+ with time_machine.travel("2026-10-29T15:10:00Z", tick=False):+ second = self._create_runner(query).run()+ second_event = report.call_args.args[1]+ assert (getattr(first, "is_cached", None), getattr(second, "is_cached", None)) == (False, True)+ expiry = datetime(2026, 10, 29, 16, 0, tzinfo=UTC)+ assert (getattr(first, "cache_target_age", None), getattr(second, "cache_target_age", None)) == (+ expiry,+ expiry,+ )+ expected_properties = {+ "sql_cache_window": "up_to_now_long",+ "sql_cache_fresh_until": expiry.isoformat(),+ "sql_cache_reason": "window_age",+ }+ for event in (first_event, second_event):+ assert {key: value for key, value in event.items() if key.startswith("sql_cache_")} == expected_properties++ def test_recompute_drops_window_inherited_from_the_cached_entry(self) -> None:+ runner = self._create_runner(+ HogQLQuery(query="select count() from events where timestamp > now() - interval 7 day")+ )+ runner._read_cache_entry_freshness(+ {+ "window": "up_to_now_long",+ "fresh_until": "2020-01-01T00:00:00+00:00",+ "reason": "window_age",+ }+ )+ runner.calculate()+ assert CACHE_ENTRY_FRESHNESS_KEY not in runner._cache_entry_with_freshness({})+++class TestHogQLQueryRunnerApiSqlFreshness(BaseTest):+ @parameterized.expand(+ [+ ("api_key_uses_its_window", "personal_api_key", True, True, timedelta(hours=1), False, False),+ ("expired_window_is_stale", "personal_api_key", True, True, timedelta(minutes=-1), False, True),+ ("lazy_read_keeps_the_longer_default", "personal_api_key", True, True, timedelta(minutes=-1), True, False),+ ("app_keeps_default", None, True, True, timedelta(hours=1), False, True),+ ("flag_off_keeps_default", "personal_api_key", False, True, timedelta(hours=1), False, True),+ ("entry_recomputed_without_window", "personal_api_key", True, False, timedelta(hours=1), False, True),+ ]+ )+ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)+ def test_entry_window_age_applies_only_to_flagged_api_key_reads(+ self,+ _name: str,+ access_method: str | None,+ flag_on: bool,+ entry_has_window: bool,+ fresh_for: timedelta,+ lazy: bool,+ expected_stale: bool,+ ) -> None:+ now = datetime(2026, 10, 29, 15, 0, tzinfo=UTC)+ window = SqlFreshness(+ window=SqlWindowClass.UP_TO_NOW_LONG, fresh_until=now + fresh_for, reason=FreshnessReason.WINDOW_AGE+ )+ with _freshness_flag(on=flag_on), tags_context(access_method=access_method):+ runner = HogQLQueryRunner(team=self.team, query=HogQLQuery(query="select 1"))+ runner._read_cache_entry_freshness(freshness_record(window) if entry_has_window else None)+ stale = runner._is_stale_for_request(last_refresh=now - timedelta(minutes=10), lazy=lazy)+ assert stale is expected_stale++ def _built_freshness(self, query: HogQLQuery) -> SqlFreshness | Reject | None:+ with _freshness_flag(on=True), tags_context(access_method="personal_api_key"):+ runner = HogQLQueryRunner(team=self.team, query=query)+ runner._build_query()+ return runner._sql_classification++ @parameterized.expand(+ [+ ("filters", f"{PAST_RANGE} and {{filters}}", False),+ (+ "filters_in_a_cte",+ f"with e as (select event from events where {{filters}}) {PAST_RANGE} and event in (select event from e)",+ False,+ ),+ ("variable", f"{PAST_RANGE} and event != {{variables.event_name}}", True),+ ("expression", f"{PAST_RANGE} and event != {{toString(now())}}", False),+ ]+ )+ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)+ def test_placeholders_keep_default_age(self, _name: str, query: str, uses_variable: bool) -> None:+ variable = InsightVariable.objects.create(team=self.team, name="Event", code_name="event_name", type="String")+ freshness = self._built_freshness(+ HogQLQuery(+ query=query,+ filters=HogQLFilters(dateRange=DateRange(date_from="-3d")),+ variables={+ str(variable.id): HogQLVariable(code_name="event_name", variableId=str(variable.id), value="x")+ }+ if uses_variable+ else None,+ )+ )+ assert freshness == Reject(reason=FreshnessReason.UNSUPPORTED_PLACEHOLDER)++ def test_classifier_error_keeps_default_age(self) -> None:+ with (+ patch("posthog.hogql_queries.sql_freshness.classifier.fresh_until", side_effect=RuntimeError),+ patch("posthog.hogql_queries.sql_freshness.classifier.capture_exception") as capture,+ ):+ freshness = self._built_freshness(HogQLQuery(query=PAST_RANGE))+ assert freshness == Reject(reason=FreshnessReason.UNPARSEABLE)+ assert capture.call_args.args[1] == {"context": "sql_freshness_classification"}++ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)+ def test_named_values_read_as_literals(self) -> None:+ freshness = self._built_freshness(+ HogQLQuery(+ query="select count() from events where timestamp >= {start} and timestamp < {end}",+ values={"start": "2026-10-01", "end": "2026-10-15"},+ )+ )+ assert isinstance(freshness, SqlFreshness)+ assert freshness.window == SqlWindowClass.FIXED_PAST++ @parameterized.expand([("project_timezone", None, "fixed_past"), ("utc", False, "up_to_now_long")])+ def test_bounds_follow_the_query_timezone(+ self, _name: str, convert_to_project_timezone: bool | None, expected_window: str+ ) -> None:+ self.team.timezone = "Asia/Tokyo"+ self.team.save()+ query = HogQLQuery(+ query="select count() from events where timestamp >= '2026-10-20' and timestamp < '2026-10-29'",+ modifiers=HogQLQueryModifiers(convertToProjectTimezone=convert_to_project_timezone),+ )+ with time_machine.travel(datetime(2026, 10, 28, 16, 0, tzinfo=UTC), tick=False):+ freshness = self._built_freshness(query)+ assert isinstance(freshness, SqlFreshness)+ assert freshness.window.value == expected_window