Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(query): cache api key sql results by the time window they read (#125610)

Vasco de Krijgerf4d91ccfc12c
diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.py
index 150d66308cda..50ebf373fcd1 100644
--- a/posthog/hogql_queries/hogql_query_runner.py
+++ b/posthog/hogql_queries/hogql_query_runner.py
@@ -1,5 +1,5 @@
from collections.abc import Callable
-from datetime import datetime
+from datetime import UTC, datetime
from functools import cached_property
from typing import Any, Optional, cast
@@ -27,15 +27,24 @@ from posthog.hogql.metadata import get_table_names
from posthog.hogql.parser import CacheOrigin, parse_select
from posthog.hogql.placeholders import find_placeholders, replace_placeholders
from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.resolver import resolve_types
from posthog.hogql.user_query_validator import validate_user_query
from posthog.hogql.variables import replace_variables
+from posthog.hogql.visitor import clone_expr
from posthog import settings as app_settings
from posthog.caching.utils import ThresholdMode, staleness_threshold_map
-from posthog.clickhouse.query_tagging import tag_contains_user_hogql
+from posthog.clickhouse.query_tagging import get_query_tag_value, is_api_key_access_method, tag_contains_user_hogql
from posthog.event_usage import AnalyticsProps
from posthog.hogql_queries.paginators import HogQLHasMorePaginator
-from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionMode
+from posthog.hogql_queries.query_runner import CACHE_ENTRY_FRESHNESS_KEY, AnalyticsQueryRunner, ExecutionMode
+from posthog.hogql_queries.sql_freshness.classifier import (
+ SqlFreshness,
+ classify,
+ freshness_from_record,
+ freshness_record,
+)
+from posthog.hogql_queries.sql_freshness.rules import FreshnessReason, Reject
from posthog.models import User
from posthog.models.activity_logging.retention import get_activity_log_lookback_restriction
@@ -49,12 +58,14 @@ from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLM
_INFORMATION_SCHEMA_PREFIX = "system.information_schema."
_ACTIVITY_LOGS_TABLE = "system.activity_logs"
+API_SQL_WINDOW_CACHE_AGE_FLAG = "api-sql-window-cache-age"
class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
query: HogQLQuery
cached_response: CachedHogQLQueryResponse
settings: Optional[HogQLGlobalSettings]
+ _sql_classification: SqlFreshness | Reject | None = None
# p95 duration of a query service HogQL query is 2.78sec
QUERY_SERVICE_MAX_EXECUTION_TIME = 10
@@ -75,12 +86,73 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
# A direct connection runs on another database, so the events route never applies to it.
return not self.query.connectionId
- # Treat SQL query caching like day insight
def cache_target_age(self, last_refresh: Optional[datetime], lazy: bool = False) -> Optional[datetime]:
if last_refresh is None:
return None
+ freshness = self._sql_classification
+ if freshness is not None and self._api_sql_freshness_enabled:
+ expiry = super().cache_target_age(last_refresh, lazy=lazy)
+ if isinstance(freshness, SqlFreshness) and expiry is not None:
+ return max(expiry, freshness.fresh_until)
+ return expiry
return last_refresh + staleness_threshold_map[ThresholdMode.LAZY if lazy else ThresholdMode.DEFAULT]["day"]
+ @cached_property
+ def _api_sql_freshness_enabled(self) -> bool:
+ return (
+ self._reads_events_route()
+ and is_api_key_access_method(get_query_tag_value("access_method"))
+ and self._team_flag_enabled_locally(API_SQL_WINDOW_CACHE_AGE_FLAG)
+ )
+
+ def _is_stale(self, last_refresh: Optional[datetime], lazy: bool = False) -> bool:
+ freshness = self._sql_classification
+ default_stale = super()._is_stale(last_refresh, lazy=lazy)
+ if isinstance(freshness, SqlFreshness) and last_refresh is not None and self._api_sql_freshness_enabled:
+ return default_stale and datetime.now(UTC) > freshness.fresh_until
+ return default_stale
+
+ def _cache_entry_with_freshness(self, response: dict[str, Any]) -> dict[str, Any]:
+ if not isinstance(self._sql_classification, SqlFreshness):
+ return response
+ return {**response, CACHE_ENTRY_FRESHNESS_KEY: freshness_record(self._sql_classification)}
+
+ def _read_cache_entry_freshness(self, record: Any) -> None:
+ self._sql_classification = freshness_from_record(record)
+
+ def _staleness_event_properties(self) -> dict[str, Any]:
+ classification = self._sql_classification
+ if (
+ classification is None
+ or getattr(self, "_cache_age_override", None) is not None
+ or not self._api_sql_freshness_enabled
+ ):
+ return {}
+ if isinstance(classification, Reject):
+ return {"sql_cache_reason": classification.reason.value}
+ return {f"sql_cache_{key}": value for key, value in freshness_record(classification).items()}
+
+ def _classify_sql_freshness(self, query: ast.SelectQuery | ast.SelectSetQuery) -> SqlFreshness | Reject:
+ if "events" not in {name.lower() for name in self._queried_table_names}:
+ return Reject(reason=FreshnessReason.NO_EVENTS)
+ try:
+ placeholders = find_placeholders(parse_select(self.query.query))
+ if (
+ placeholders.has_filters
+ or placeholders.placeholder_expressions
+ or any(chain[:1] == ["variables"] for chain in placeholders.placeholder_fields)
+ ):
+ return Reject(reason=FreshnessReason.UNSUPPORTED_PLACEHOLDER)
+ context = self.build_hogql_context(enable_select_queries=True, modifiers=self.modifiers)
+ resolved = resolve_types(clone_expr(query), context, dialect="hogql")
+ except Exception:
+ return Reject(reason=FreshnessReason.UNPARSEABLE)
+ return classify(
+ resolved,
+ now=datetime.now(UTC),
+ timezone=UTC if self.modifiers.convertToProjectTimezone is False else self.team.timezone_info,
+ )
+
def _validate_direct_connection(self, *, user: Optional[User] = None, force: bool = False) -> None:
if not force and self._direct_connection_validated:
return
@@ -280,8 +352,16 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
and self.team.pk not in app_settings.API_QUERIES_LEGACY_TEAM_LIST
)
+ def _build_query(self) -> ast.SelectQuery | ast.SelectSetQuery:
+ query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()
+ if self._api_sql_freshness_enabled:
+ with self.timings.measure("sql_freshness"):
+ self._sql_classification = self._classify_sql_freshness(query)
+ return query
+
def _calculate(self) -> HogQLQueryResponse:
tag_contains_user_hogql()
+ self._sql_classification = None
if self._capped_for_query_service():
assert self.settings is not None
# p95 threads is 102, limiting to 60 (below global max_threads of 64)
@@ -325,7 +405,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
trino_canary=self._trino_canary,
)
- query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()
+ query = self._build_query()
if self.is_query_service:
validate_user_query(query, team=self.team)
diff --git a/posthog/hogql_queries/query_runner.py b/posthog/hogql_queries/query_runner.py
index bb7c9d77eb5c..3e3ad93e80b9 100644
--- a/posthog/hogql_queries/query_runner.py
+++ b/posthog/hogql_queries/query_runner.py
@@ -304,6 +304,7 @@ def _contains_user_hogql_label() -> str:
EXTENDED_CACHE_AGE = timedelta(days=1)
+CACHE_ENTRY_FRESHNESS_KEY = "_freshness"
QUERY_HASH_VERSION = 1
@@ -2292,6 +2293,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
if raw_results is not None:
cached_response_candidate["results"] = []
cached_response_candidate["is_cached"] = True
+ self._read_cache_entry_freshness(cached_response_candidate.pop(CACHE_ENTRY_FRESHNESS_KEY, None))
# When rolling out schema changes, cached responses may not match the new schema.
# Trigger recomputation in this case.
try:
@@ -2320,7 +2322,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
if custom_names_modified:
# Update cache with patched response so subsequent requests get the updated names
cache_manager.store_result(
- response=cached_response.model_dump(),
+ response=self._cache_entry_with_freshness(cached_response.model_dump()),
target_age=cached_response.cache_target_age,
)
@@ -2837,6 +2839,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
**query_run.event_properties(self.modifiers),
"cache_hit": isinstance(results, self.cached_response_type),
"cache_age_override": self._cache_age_override,
+ **self._staleness_event_properties(),
"response_time_ms": query_run.elapsed_ms(),
"clickhouse_rows_read": cached_query_scan.rows_read if cached_query_scan else None,
"clickhouse_duration_ms": cached_query_scan.duration_ms if cached_query_scan else None,
@@ -2899,7 +2902,6 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
CachedResponse: type[CR] = self.cached_response_type
last_refresh = datetime.now(UTC)
- target_age = self.cache_target_age(last_refresh=last_refresh)
# Avoid affecting cache key
# Add user based modifiers here, primarily for user specific feature flagging
@@ -2967,6 +2969,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
SURVEY_QUERY_EXECUTION_DURATION.labels(**survey_query_metric_labels).observe(query_duration_seconds)
self._phase = RunPhase(name="finish", stats=query_stats, query_duration_ms=query_duration_ms)
+ target_age = self.cache_target_age(last_refresh=last_refresh)
fresh_response_dict: dict[str, Any] = {
**query_result.model_dump(),
"is_cached": False,
@@ -3065,7 +3068,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
if cacheable:
with self.timings.measure("cache_write"):
stored = cache_manager.store_result(
- response=fresh_response_dict,
+ response=self._cache_entry_with_freshness(fresh_response_dict),
# This would be a possible place to decide to not ever keep this cache warm
# Example: Not for super quickly calculated insights
# Set target_age to None in that case
@@ -3091,6 +3094,7 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
"last_refresh": last_refresh.isoformat(),
"cache_write_success": stored,
"cache_age_override": getattr(self, "_cache_age_override", None),
+ **self._staleness_event_properties(),
"calculation_trigger": query_run.trigger,
"response_time_ms": query_run.elapsed_ms(),
"query_duration_ms": query_duration_ms,
@@ -3453,6 +3457,21 @@ class QueryRunner(ABC, Generic[Q, R, CR]):
return True
return datetime.now(UTC) > last_refresh + timedelta(seconds=override)
+ def _staleness_event_properties(self) -> dict[str, Any]:
+ return {}
+
+ def _cache_entry_with_freshness(self, response: dict[str, Any]) -> dict[str, Any]:
+ """Adds the runner's freshness decision to a cache entry before it is stored.
+
+ A cache hit returns before the runner compiles its query, so the runner cannot work out
+ freshness again on that request. A runner that decides freshness from the compiled query
+ stores the decision under CACHE_ENTRY_FRESHNESS_KEY and reads it back on the hit.
+ """
+ return response
+
+ def _read_cache_entry_freshness(self, record: Any) -> None:
+ """Restores the decision that `_cache_entry_with_freshness` stored, before the response is parsed."""
+
def _is_stale(self, last_refresh: Optional[datetime], lazy: bool = False) -> bool:
query_date_range = getattr(self, "query_date_range", None)
date_to = query_date_range.date_to() if query_date_range else None
diff --git a/posthog/hogql_queries/sql_freshness/__init__.py b/posthog/hogql_queries/sql_freshness/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/posthog/hogql_queries/sql_freshness/classifier.py b/posthog/hogql_queries/sql_freshness/classifier.py
new file mode 100644
index 000000000000..246b5cf763dd
--- /dev/null
+++ b/posthog/hogql_queries/sql_freshness/classifier.py
@@ -0,0 +1,63 @@
+from datetime import UTC, datetime, tzinfo
+from typing import Any, Optional
+
+from dateutil.parser import isoparse
+
+from posthog.hogql import ast
+
+from posthog.dataclasses import frozen
+from posthog.exceptions_capture import capture_exception
+from posthog.hogql_queries.sql_freshness.facts import extract_facts
+from posthog.hogql_queries.sql_freshness.rules import (
+ FreshnessReason,
+ Reject,
+ SqlWindowClass,
+ fresh_until,
+ reject_reason,
+ shortest_lived_window,
+)
+
+
+@frozen
+class SqlFreshness:
+ window: SqlWindowClass
+ fresh_until: datetime
+ reason: FreshnessReason
+
+
+def classify(
+ resolved_query: ast.SelectQuery | ast.SelectSetQuery, *, now: datetime, timezone: tzinfo
+) -> SqlFreshness | Reject:
+ try:
+ facts = extract_facts(resolved_query, now=now, timezone=timezone)
+ reason = reject_reason(facts)
+ if reason is not None:
+ return Reject(reason=reason)
+ until, binding_reason = fresh_until(facts)
+ window = shortest_lived_window(facts)
+ except Exception as error:
+ capture_exception(error, {"context": "sql_freshness_classification"})
+ return Reject(reason=FreshnessReason.UNPARSEABLE)
+ return SqlFreshness(window=window, fresh_until=until.astimezone(UTC), reason=binding_reason)
+
+
+def freshness_record(freshness: SqlFreshness) -> dict[str, str]:
+ return {
+ "window": freshness.window.value,
+ "fresh_until": freshness.fresh_until.isoformat(),
+ "reason": freshness.reason.value,
+ }
+
+
+def freshness_from_record(record: Any) -> Optional[SqlFreshness]:
+ try:
+ fresh_until = isoparse(record["fresh_until"])
+ if fresh_until.tzinfo is None:
+ return None
+ return SqlFreshness(
+ window=SqlWindowClass(record["window"]),
+ fresh_until=fresh_until,
+ reason=FreshnessReason(record["reason"]),
+ )
+ except (AttributeError, KeyError, OverflowError, TypeError, ValueError):
+ return None
diff --git a/posthog/hogql_queries/sql_freshness/facts.py b/posthog/hogql_queries/sql_freshness/facts.py
new file mode 100644
index 000000000000..6464964658ed
--- /dev/null
+++ b/posthog/hogql_queries/sql_freshness/facts.py
@@ -0,0 +1,432 @@
+from datetime import UTC, datetime, tzinfo
+from typing import Any, Optional
+from zoneinfo import ZoneInfo
+
+from dateutil.parser import isoparse
+
+from posthog.hogql import ast
+from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.database.schema.groups import GroupsTable
+from posthog.hogql.database.schema.person_distinct_ids import PersonDistinctIdsTable
+from posthog.hogql.database.schema.persons import PersonsTable
+from posthog.hogql.database.schema.sessions_v1 import SessionsTableV1
+from posthog.hogql.database.schema.sessions_v2 import SessionsTableV2
+from posthog.hogql.database.schema.sessions_v3 import SessionsTableV3
+from posthog.hogql.database.schema.util.where_clause_extractor import (
+ contains_in_cohort,
+ top_level_conjuncts,
+ unwrap_table_alias_type,
+)
+from posthog.hogql.functions.clickhouse.datetime import DATE_GENERATOR_FUNCTIONS
+from posthog.hogql.functions.mapping import find_hogql_function
+from posthog.hogql.transforms.order_by_pushdown import unwrap_alias
+from posthog.hogql.visitor import TraversingVisitor
+
+from posthog.dataclasses import frozen
+from posthog.interval_specs import INTERVAL_SPECS, PERIOD_MAP
+
+_SUB_DAY_UNITS = frozenset({"second", "minute", "hour"})
+_JOINABLE_TABLES = (
+ PersonsTable,
+ PersonDistinctIdsTable,
+ GroupsTable,
+ SessionsTableV1,
+ SessionsTableV2,
+ SessionsTableV3,
+)
+_EXPRESSION_FIELDS = frozenset({"person_id"})
+_TIMESTAMP_COLUMN = "timestamp"
+_LATEST_EVENT_FUNCTION_PREFIXES = ("max", "argmax", "anylast", "last_value")
+_ZONE_ARGUMENT_POSITION = {"toDateTime": 1, "toDateTime64": 2, "parseDateTimeBestEffort": 1}
+_NOW_FUNCTIONS = frozenset({"now", "now64"})
+_EARLIEST_DATETIME = datetime(1970, 1, 1, tzinfo=UTC)
+_LATEST_DATETIME = datetime(2106, 2, 7, 6, 28, 15, tzinfo=UTC)
+_CLOCK_FUNCTIONS = {id(meta): meta.clickhouse_name for meta in DATE_GENERATOR_FUNCTIONS.values()} | {
+ id(find_hogql_function("nowInBlock")): "nowInBlock"
+}
+_LOWER_BOUND_OPS = frozenset({ast.CompareOperationOp.Gt, ast.CompareOperationOp.GtEq})
+_FLIPPED_OPS = {
+ ast.CompareOperationOp.Gt: ast.CompareOperationOp.Lt,
+ ast.CompareOperationOp.GtEq: ast.CompareOperationOp.LtEq,
+ ast.CompareOperationOp.Lt: ast.CompareOperationOp.Gt,
+ ast.CompareOperationOp.LtEq: ast.CompareOperationOp.GtEq,
+ ast.CompareOperationOp.Eq: ast.CompareOperationOp.Eq,
+}
+
+
+@frozen
+class Window:
+ start: datetime
+ end: Optional[datetime]
+
+
+@frozen
+class _Bound:
+ start: Optional[datetime]
+ end: Optional[datetime]
+
+
+@frozen
+class QueryFacts:
+ now: datetime
+ start_of_today: datetime
+ windows: tuple[Window, ...]
+ unsupported_field: bool
+ join: bool
+ foreign_source: bool
+ unbounded_select: bool
+ unsupported_bound: bool
+ clock_outside_bounds: bool
+ reads_now: bool
+ reads_latest_event: bool
+
+
+class _UnsupportedBound(Exception):
+ pass
+
+
+def _clock_name(call: ast.Call) -> Optional[str]:
+ meta = find_hogql_function(call.name)
+ return _CLOCK_FUNCTIONS.get(id(meta)) if meta is not None else None
+
+
+def _is_events_table(table_type: Any) -> bool:
+ table_type = unwrap_table_alias_type(table_type)
+ return isinstance(table_type, ast.TableType) and isinstance(table_type.table, EventsTable)
+
+
+def _is_subquery_type(table_type: Any) -> bool:
+ return isinstance(
+ table_type, ast.SelectQueryType | ast.SelectSetQueryType | ast.SelectQueryAliasType | ast.CTETableType
+ )
+
+
+def _reads_supported_table(table_type: Any) -> bool:
+ if isinstance(table_type, ast.VirtualTableType):
+ return _reads_supported_table(table_type.table_type)
+ if isinstance(table_type, ast.LazyJoinType):
+ return isinstance(table_type.lazy_join.join_table, _JOINABLE_TABLES) and _reads_supported_table(
+ table_type.table_type
+ )
+ return _is_events_table(table_type)
+
+
+def _is_supported_field_type(field_type: Any) -> bool:
+ if isinstance(field_type, ast.FieldAliasType):
+ return True
+ if isinstance(field_type, ast.PropertyType):
+ return _is_supported_field_type(field_type.field_type)
+ if isinstance(field_type, ast.ExpressionFieldType):
+ return field_type.name in _EXPRESSION_FIELDS and _reads_supported_table(field_type.table_type)
+ if not isinstance(field_type, ast.FieldType):
+ return False
+ return _is_subquery_type(field_type.table_type) or _reads_supported_table(field_type.table_type)
+
+
+def _is_events_timestamp(expr: ast.Expr) -> bool:
+ expr = unwrap_alias(expr)
+ field_type = expr.type if isinstance(expr, ast.Field) else None
+ return (
+ isinstance(field_type, ast.FieldType)
+ and field_type.name == _TIMESTAMP_COLUMN
+ and _is_events_table(field_type.table_type)
+ )
+
+
+def _int_constant(expr: ast.Expr) -> Optional[int]:
+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):
+ return expr.value
+ return None
+
+
+def _string_constant(expr: ast.Expr) -> Optional[str]:
+ return expr.value if isinstance(expr, ast.Constant) and isinstance(expr.value, str) else None
+
+
+def _parse_instant(value: Optional[str], timezone: tzinfo) -> Optional[datetime]:
+ if value is None or not (len(value) >= 10 and value[4] == "-" and value[7] == "-" and value[:4].isdigit()):
+ return None
+ try:
+ parsed = isoparse(value)
+ if parsed.tzinfo:
+ return parsed.astimezone(timezone)
+ local = parsed.replace(tzinfo=timezone)
+ return local if _wall_time_exists(local) else None
+ except (OverflowError, ValueError):
+ return None
+
+
+def _wall_time_exists(at: datetime) -> bool:
+ return at.astimezone(UTC).astimezone(at.tzinfo).replace(tzinfo=None) == at.replace(tzinfo=None)
+
+
+def _repeats_wall_time(at: datetime, timezone: tzinfo) -> bool:
+ return any(at.astimezone(zone).fold == 1 for zone in (timezone, at.tzinfo))
+
+
+def _shifted(at: datetime, unit: str, amount: int) -> Optional[datetime]:
+ if unit in _SUB_DAY_UNITS:
+ return (at.astimezone(UTC) + PERIOD_MAP[unit] * amount).astimezone(at.tzinfo)
+ moved = at + PERIOD_MAP[unit] * amount
+ return moved if _wall_time_exists(moved) else None
+
+
+class _TimestampFinder(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.found = False
+
+ def visit_field(self, node: ast.Field) -> None:
+ if _is_events_timestamp(node):
+ self.found = True
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ pass
+
+
+def _mentions_events_timestamp(expr: ast.Expr) -> bool:
+ finder = _TimestampFinder()
+ finder.visit(expr)
+ return finder.found
+
+
+def _subquery_column_types(table_type: Any, name: str) -> list[ast.Type]:
+ if isinstance(table_type, ast.SelectQueryAliasType | ast.CTETableType | ast.SelectViewType):
+ return _subquery_column_types(table_type.select_query_type, name)
+ if isinstance(table_type, ast.SelectSetQueryType):
+ return [column for query_type in table_type.types for column in _subquery_column_types(query_type, name)]
+ if isinstance(table_type, ast.SelectQueryType) and name in table_type.columns:
+ return [table_type.columns[name]]
+ return []
+
+
+class _DerivedTimestampFinder(TraversingVisitor):
+ def __init__(self, alias_exprs: dict[int, ast.Expr]) -> None:
+ super().__init__()
+ self.alias_exprs = alias_exprs
+ self.followed: set[int] = set()
+ self.found = False
+
+ def visit_field(self, node: ast.Field) -> None:
+ self.found = self.found or self._derives_from_events_timestamp(node.type)
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ pass
+
+ def _derives_from_events_timestamp(self, field_type: Any) -> bool:
+ if id(field_type) in self.followed:
+ return False
+ self.followed.add(id(field_type))
+ if isinstance(field_type, ast.FieldAliasType):
+ alias_expr = self.alias_exprs.get(id(field_type))
+ if alias_expr is not None:
+ self.visit(alias_expr)
+ return self.found
+ return self._derives_from_events_timestamp(field_type.type)
+ if not isinstance(field_type, ast.FieldType):
+ return False
+ if field_type.name == _TIMESTAMP_COLUMN and _is_events_table(field_type.table_type):
+ return True
+ return any(
+ self._derives_from_events_timestamp(column)
+ for column in _subquery_column_types(field_type.table_type, field_type.name)
+ )
+
+
+class _QueryScan(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.selects: list[ast.SelectQuery] = []
+ self.clock_calls: set[int] = set()
+ self.unsupported_field = False
+ self.alias_exprs: dict[int, ast.Expr] = {}
+ self.latest_event_candidates: list[ast.Expr] = []
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ self.selects.append(node)
+ for order in node.order_by or []:
+ position = _int_constant(order.expr)
+ if order.order == "DESC" and position is not None and 0 < position <= len(node.select):
+ self.latest_event_candidates.append(node.select[position - 1])
+ super().visit_select_query(node)
+
+ def visit_field(self, node: ast.Field) -> None:
+ if not isinstance(node.type, ast.BaseTableType) and not _is_supported_field_type(node.type):
+ self.unsupported_field = True
+
+ def visit_call(self, node: ast.Call) -> None:
+ if _clock_name(node) is not None:
+ self.clock_calls.add(id(node))
+ if node.name.lower().startswith(_LATEST_EVENT_FUNCTION_PREFIXES):
+ self.latest_event_candidates.extend(node.args)
+ super().visit_call(node)
+
+ def visit_window_function(self, node: ast.WindowFunction) -> None:
+ if node.name.lower().startswith(_LATEST_EVENT_FUNCTION_PREFIXES):
+ window_order = node.over_expr.order_by if node.over_expr is not None else None
+ self.latest_event_candidates.extend(
+ [*(node.exprs or []), *(node.args or []), *(order.expr for order in window_order or [])]
+ )
+ super().visit_window_function(node)
+
+ def visit_order_expr(self, node: ast.OrderExpr) -> None:
+ if node.order == "DESC":
+ self.latest_event_candidates.append(node.expr)
+ super().visit_order_expr(node)
+
+ def visit_alias(self, node: ast.Alias) -> None:
+ self.alias_exprs[id(node.type)] = node.expr
+ super().visit_alias(node)
+
+ def reads_latest_event(self) -> bool:
+ finder = _DerivedTimestampFinder(self.alias_exprs)
+ for candidate in self.latest_event_candidates:
+ finder.visit(candidate)
+ return finder.found
+
+
+class _Evaluator:
+ def __init__(self, *, now: datetime, timezone: tzinfo) -> None:
+ self.now = now
+ self.timezone = timezone
+ self.consumed_clock_calls: set[int] = set()
+ self.reads_now = False
+
+ def evaluate(self, expr: ast.Expr) -> Optional[datetime]:
+ at = self._value(unwrap_alias(expr))
+ return at if at is not None and _EARLIEST_DATETIME <= at <= _LATEST_DATETIME else None
+
+ def _value(self, expr: ast.Expr) -> Optional[datetime]:
+ if isinstance(expr, ast.Constant):
+ return _parse_instant(_string_constant(expr), self.timezone)
+ if isinstance(expr, ast.Call):
+ return self._call(expr)
+ if isinstance(expr, ast.ArithmeticOperation) and expr.op in (
+ ast.ArithmeticOperationOp.Add,
+ ast.ArithmeticOperationOp.Sub,
+ ):
+ if isinstance(unwrap_alias(expr.left), ast.Constant):
+ return None
+ base = self.evaluate(expr.left)
+ step = _interval(expr.right)
+ if base is None or step is None:
+ return None
+ unit, amount = step
+ try:
+ return _shifted(base, unit, -amount if expr.op == ast.ArithmeticOperationOp.Sub else amount)
+ except (OverflowError, ValueError):
+ return None
+ return None
+
+ def _call(self, call: ast.Call) -> Optional[datetime]:
+ if _clock_name(call) in _NOW_FUNCTIONS and not call.args:
+ self.consumed_clock_calls.add(id(call))
+ self.reads_now = True
+ return self.now
+ epoch = _int_constant(call.args[0]) if len(call.args) == 1 else None
+ if epoch is not None and call.name in ("toDateTime", "fromUnixTimestamp"):
+ timezone = self.timezone if call.name == "toDateTime" else UTC
+ try:
+ return datetime.fromtimestamp(epoch, timezone)
+ except (OverflowError, ValueError, OSError):
+ return None
+ zone_position = _ZONE_ARGUMENT_POSITION.get(call.name)
+ if zone_position is None or not call.args or len(call.args) > zone_position + 1:
+ return None
+ if call.name == "toDateTime64" and len(call.args) > 1 and _int_constant(call.args[1]) is None:
+ return None
+ zone_name = _string_constant(call.args[zone_position]) if len(call.args) > zone_position else None
+ if len(call.args) > zone_position and not zone_name:
+ return None
+ try:
+ timezone = ZoneInfo(zone_name) if zone_name else self.timezone
+ except (KeyError, ValueError):
+ return None
+ return _parse_instant(_string_constant(call.args[0]), timezone)
+
+
+def _interval(expr: ast.Expr) -> Optional[tuple[str, int]]:
+ if isinstance(expr, ast.Call) and expr.name.startswith("toInterval") and len(expr.args) == 1:
+ unit = expr.name.removeprefix("toInterval").lower()
+ amount = _int_constant(expr.args[0])
+ if unit in PERIOD_MAP and amount is not None:
+ return unit, amount
+ return None
+
+
+def _bound(
+ evaluator: _Evaluator, column: ast.Expr, op: ast.CompareOperationOp, value_expr: ast.Expr
+) -> Optional[_Bound]:
+ if not _is_events_timestamp(column):
+ return None
+ at = evaluator.evaluate(value_expr)
+ if at is None or _repeats_wall_time(at, evaluator.timezone):
+ raise _UnsupportedBound()
+ if op == ast.CompareOperationOp.Eq:
+ return _Bound(start=at, end=at)
+ return _Bound(start=at, end=None) if op in _LOWER_BOUND_OPS else _Bound(start=None, end=at)
+
+
+def _bounds_from_conjunct(evaluator: _Evaluator, conjunct: ast.Expr) -> _Bound:
+ if isinstance(conjunct, ast.CompareOperation) and conjunct.op in _FLIPPED_OPS:
+ bound = _bound(evaluator, conjunct.left, conjunct.op, conjunct.right)
+ if bound is None:
+ bound = _bound(evaluator, conjunct.right, _FLIPPED_OPS[conjunct.op], conjunct.left)
+ if bound is not None:
+ return bound
+ raise _UnsupportedBound()
+
+
+def _select_bounds(evaluator: _Evaluator, select: ast.SelectQuery) -> list[_Bound]:
+ return [
+ _bounds_from_conjunct(evaluator, conjunct)
+ for clause in (select.where, select.prewhere)
+ if clause is not None
+ for conjunct in top_level_conjuncts(clause)
+ if _mentions_events_timestamp(conjunct)
+ ]
+
+
+def extract_facts(
+ resolved_query: ast.SelectQuery | ast.SelectSetQuery, *, now: datetime, timezone: tzinfo
+) -> QueryFacts:
+ now = now.astimezone(timezone)
+ scan = _QueryScan()
+ scan.visit(resolved_query)
+ evaluator = _Evaluator(now=now, timezone=timezone)
+ windows: list[Window] = []
+ has_join = foreign_source = unbounded_select = unsupported_bound = False
+ for select in scan.selects:
+ join = select.select_from
+ if join is None:
+ continue
+ has_join = has_join or join.next_join is not None
+ if _is_subquery_type(join.type):
+ continue
+ if not _is_events_table(join.type):
+ foreign_source = True
+ continue
+ try:
+ bounds = _select_bounds(evaluator, select)
+ except _UnsupportedBound:
+ unsupported_bound = True
+ continue
+ starts = [bound.start.astimezone(UTC) for bound in bounds if bound.start is not None]
+ ends = [bound.end.astimezone(UTC) for bound in bounds if bound.end is not None]
+ if not starts:
+ unbounded_select = True
+ continue
+ windows.append(Window(start=max(starts), end=min(ends) if ends else None))
+ return QueryFacts(
+ now=now.astimezone(UTC),
+ start_of_today=INTERVAL_SPECS["day"].align(now, None),
+ windows=tuple(windows),
+ unsupported_field=scan.unsupported_field,
+ join=has_join,
+ foreign_source=foreign_source or contains_in_cohort(resolved_query),
+ unbounded_select=unbounded_select or not windows,
+ unsupported_bound=unsupported_bound,
+ clock_outside_bounds=bool(scan.clock_calls - evaluator.consumed_clock_calls),
+ reads_now=evaluator.reads_now,
+ reads_latest_event=scan.reads_latest_event(),
+ )
diff --git a/posthog/hogql_queries/sql_freshness/rules.py b/posthog/hogql_queries/sql_freshness/rules.py
new file mode 100644
index 000000000000..14ef0e7ac984
--- /dev/null
+++ b/posthog/hogql_queries/sql_freshness/rules.py
@@ -0,0 +1,95 @@
+from datetime import datetime, timedelta
+from enum import StrEnum
+from typing import Optional
+
+from posthog.dataclasses import frozen
+from posthog.hogql_queries.sql_freshness.facts import QueryFacts, Window
+
+
+class SqlWindowClass(StrEnum):
+ FIXED_PAST = "fixed_past"
+ LIVE = "live"
+ UP_TO_NOW_SHORT = "up_to_now_short"
+ UP_TO_NOW_LONG = "up_to_now_long"
+
+
+SQL_WINDOW_MAX_AGE: dict[SqlWindowClass, timedelta] = {
+ SqlWindowClass.FIXED_PAST: timedelta(hours=6),
+ SqlWindowClass.LIVE: timedelta(minutes=5),
+ SqlWindowClass.UP_TO_NOW_SHORT: timedelta(minutes=30),
+ SqlWindowClass.UP_TO_NOW_LONG: timedelta(hours=1),
+}
+
+LIVE_WINDOW_MAX_LENGTH = timedelta(hours=1)
+SHORT_WINDOW_MAX_LENGTH = timedelta(days=1)
+FIXED_PAST_SETTLE_TIME = timedelta(hours=1)
+
+
+class FreshnessReason(StrEnum):
+ WINDOW_AGE = "window_age"
+ NOW_ANCHORED = "now_anchored"
+ LATEST_EVENT = "latest_event"
+ NO_EVENTS = "no_events"
+ UNSUPPORTED_FIELD = "unsupported_field"
+ JOIN = "join"
+ FOREIGN_SOURCE = "foreign_source"
+ UNBOUNDED = "unbounded"
+ UNSUPPORTED_BOUND = "unsupported_bound"
+ CLOCK_OUTSIDE_BOUNDS = "clock_outside_bounds"
+ UNPARSEABLE = "unparseable"
+ UNSUPPORTED_PLACEHOLDER = "unsupported_placeholder"
+
+
+@frozen
+class Reject:
+ reason: FreshnessReason
+
+
+def window_class(window: Window, facts: QueryFacts) -> SqlWindowClass:
+ if (
+ window.end is not None
+ and window.end <= facts.start_of_today
+ and window.end <= facts.now - FIXED_PAST_SETTLE_TIME
+ ):
+ return SqlWindowClass.FIXED_PAST
+ return _class_for_length(facts.now - window.start)
+
+
+def _class_for_length(length: timedelta) -> SqlWindowClass:
+ if length <= LIVE_WINDOW_MAX_LENGTH:
+ return SqlWindowClass.LIVE
+ if length <= SHORT_WINDOW_MAX_LENGTH:
+ return SqlWindowClass.UP_TO_NOW_SHORT
+ return SqlWindowClass.UP_TO_NOW_LONG
+
+
+def shortest_lived_window(facts: QueryFacts) -> SqlWindowClass:
+ return min((window_class(window, facts) for window in facts.windows), key=SQL_WINDOW_MAX_AGE.__getitem__)
+
+
+def reject_reason(facts: QueryFacts) -> Optional[FreshnessReason]:
+ checks = (
+ (facts.unsupported_field, FreshnessReason.UNSUPPORTED_FIELD),
+ (facts.join, FreshnessReason.JOIN),
+ (facts.foreign_source, FreshnessReason.FOREIGN_SOURCE),
+ (facts.unsupported_bound, FreshnessReason.UNSUPPORTED_BOUND),
+ (facts.unbounded_select, FreshnessReason.UNBOUNDED),
+ (facts.clock_outside_bounds, FreshnessReason.CLOCK_OUTSIDE_BOUNDS),
+ )
+ return next((reason for failed, reason in checks if failed), None)
+
+
+def fresh_until(facts: QueryFacts) -> tuple[datetime, FreshnessReason]:
+ """Every limit that applies holds at once, so the earliest one sets the expiry."""
+ limits = [(facts.now + SQL_WINDOW_MAX_AGE[shortest_lived_window(facts)], FreshnessReason.WINDOW_AGE)]
+ if facts.reads_now:
+ limits += [
+ (facts.now + SQL_WINDOW_MAX_AGE[_class_for_length(window.end - window.start)], FreshnessReason.NOW_ANCHORED)
+ for window in facts.windows
+ if window.end is not None and window_class(window, facts) == SqlWindowClass.FIXED_PAST
+ ]
+ if facts.reads_latest_event and any(
+ window_class(window, facts) != SqlWindowClass.FIXED_PAST for window in facts.windows
+ ):
+ limits.append((facts.now + SQL_WINDOW_MAX_AGE[SqlWindowClass.LIVE], FreshnessReason.LATEST_EVENT))
+ return min(limits, key=lambda limit: limit[0])
diff --git a/posthog/hogql_queries/sql_freshness/test/test_classifier.py b/posthog/hogql_queries/sql_freshness/test/test_classifier.py
new file mode 100644
index 000000000000..5fa9f9e29d4f
--- /dev/null
+++ b/posthog/hogql_queries/sql_freshness/test/test_classifier.py
@@ -0,0 +1,395 @@
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from posthog.test.base import APIBaseTest, BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from parameterized import parameterized
+
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.parser import parse_select
+from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.resolver import resolve_types
+
+from posthog.hogql_queries.sql_freshness.classifier import SqlFreshness, classify, freshness_from_record
+from posthog.hogql_queries.sql_freshness.facts import extract_facts
+from posthog.hogql_queries.sql_freshness.rules import SQL_WINDOW_MAX_AGE, FreshnessReason, Reject, SqlWindowClass
+
+NOW = datetime(2026, 10, 29, 15, 0, tzinfo=UTC)
+AMSTERDAM = ZoneInfo("Europe/Amsterdam")
+PAST_DAY = "select count() from events where timestamp >= '2026-10-28' and timestamp < '2026-10-29'"
+ACCEPTED_BOUNDS = [
+ ("date_literal", "'2026-03-29'", "Europe/Amsterdam"),
+ ("datetime_literal", "'2026-03-29 03:30:00'", "Europe/Amsterdam"),
+ ("utc_literal", "'2026-10-29T08:00:00Z'", "Europe/Amsterdam"),
+ ("zoned_literal", "toDateTime('2026-10-29 08:00:00', 'Asia/Tokyo')", "UTC"),
+ ("zoned_datetime64", "toDateTime64('2026-10-29 08:00:00', 6, 'Asia/Tokyo')", "UTC"),
+ ("datetime64_with_precision_only", "toDateTime64('2026-10-29 08:00:00', 3)", "Europe/Amsterdam"),
+ ("best_effort_with_offset", "parseDateTimeBestEffort('2026-10-29T08:00:00Z')", "Europe/Amsterdam"),
+ ("best_effort_with_zone", "parseDateTimeBestEffort('2026-10-29 08:00:00', 'Asia/Tokyo')", "UTC"),
+ ("epoch", "toDateTime(1761955200)", "Europe/Amsterdam"),
+ ("from_unix_timestamp", "fromUnixTimestamp(1761955200)", "Europe/Amsterdam"),
+ ("epoch_plus_a_day_across_a_fall_back", "toDateTime(1792836000) + interval 1 day", "Europe/Amsterdam"),
+ ("from_unix_timestamp_plus_a_month", "fromUnixTimestamp(1769815800) + interval 1 month", "Europe/Amsterdam"),
+ ("day_interval_across_a_fall_back", "toDateTime('2026-10-24 12:00:00') + interval 1 day", "Europe/Amsterdam"),
+ ("hour_interval_across_a_fall_back", "toDateTime('2026-10-25 01:30:00') + interval 3 hour", "Europe/Amsterdam"),
+ ("month_interval_at_a_month_end", "toDateTime('2026-01-31 12:00:00') + interval 1 month", "Europe/Amsterdam"),
+ ("wall_time_in_a_dst_fold", "'2026-10-25 02:30:00'", "Europe/Amsterdam"),
+ (
+ "hour_interval_across_a_spring_forward",
+ "toDateTime('2026-03-29 01:30:00') + interval 1 hour",
+ "Europe/Amsterdam",
+ ),
+ ("now_minus_hours", "now() - interval 6 hour", "Europe/Amsterdam"),
+ ("now_minus_days", "now() - interval 7 day", "Europe/Amsterdam"),
+]
+
+
+def _resolve(sql: str, team: Any) -> Any:
+ context = HogQLContext(team_id=team.pk, database=Database.create_for(team=team), enable_select_queries=True)
+ return resolve_types(parse_select(sql), context, dialect="hogql")
+
+
+class TestClassify(BaseTest):
+ def _classify(self, sql: str, now: datetime = NOW, timezone: Any = None) -> SqlFreshness | Reject:
+ return classify(_resolve(sql, self.team), now=now, timezone=timezone or self.team.timezone_info)
+
+ @parameterized.expand(
+ [
+ (
+ "last_30_minutes",
+ "select count() from events where timestamp > now() - interval 30 minute",
+ SqlWindowClass.LIVE,
+ ),
+ (
+ "last_6_hours_with_alias",
+ "select count() as c from events where timestamp >= now() - interval 6 hour order by c",
+ SqlWindowClass.UP_TO_NOW_SHORT,
+ ),
+ (
+ "last_7_days",
+ "select event, count() from events where timestamp > now() - interval 7 day group by event",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "cte_of_events",
+ "with recent as (select event from events where timestamp > now() - interval 7 day) "
+ "select count() from recent",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "or_without_time",
+ "select count() from events where (event = 'a' or event = 'b') and timestamp > now() - interval 2 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "person_id",
+ "select uniq(person_id) from events where timestamp > now() - interval 7 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "person_properties",
+ "select count() from events where timestamp > now() - interval 7 day and person.properties.email = 'a'",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "group_properties",
+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "
+ "and group_0.properties.plan = 'x'",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "session_properties",
+ "select count() from events where timestamp > now() - interval 2 day and session.$session_duration > 10",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ ),
+ (
+ "absolute_past_range",
+ "select count() from events where timestamp >= '2026-10-01' and timestamp <= '2026-10-15'",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "zoned_literal_uses_its_timezone",
+ "select count() from events where timestamp >= '2026-10-01' "
+ "and timestamp < toDateTime('2026-10-29 08:00:00', 'Asia/Tokyo')",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "epoch_seconds",
+ "select count() from events where timestamp >= toDateTime(1759276800) "
+ "and timestamp < fromUnixTimestamp(1760486400)",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "literal_plus_days",
+ "select count() from events where timestamp >= toDateTime('2026-10-01 00:00:00') "
+ "and timestamp < toDateTime('2026-10-01 00:00:00') + interval 14 day",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "now_anchored_past_window_moves",
+ "select count() from events where timestamp > now() - interval 9 day and timestamp < now() - interval 2 day",
+ SqlWindowClass.FIXED_PAST,
+ {"fresh_until": NOW + timedelta(hours=1)},
+ ),
+ (
+ "narrow_now_anchored_window_expires_within_its_width",
+ "select count() from events where timestamp >= now() - interval 25 hour and timestamp < now() - interval 24 hour",
+ SqlWindowClass.FIXED_PAST,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_event_time_up_to_now",
+ "select max(timestamp) from events where timestamp > now() - interval 2 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_rows_up_to_now",
+ "select event from events where timestamp > now() - interval 7 day order by timestamp desc limit 10",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_rows_by_an_alias",
+ "select timestamp as ts from events where timestamp > now() - interval 7 day order by ts desc limit 10",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_value_by_time",
+ "select argMax(event, timestamp) from events where timestamp > now() - interval 7 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_event_time_with_a_condition",
+ "select maxIf(timestamp, event = 'signup') from events where timestamp > now() - interval 7 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_value_in_a_window_function",
+ "select last_value(event) over (partition by person_id order by timestamp) from events "
+ "where timestamp > now() - interval 7 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_rows_by_position",
+ "select timestamp, event from events where timestamp > now() - interval 7 day order by 1 desc limit 10",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_event_time_through_a_subquery",
+ "select max(ts) from (select toStartOfMinute(timestamp) as ts from events "
+ "where timestamp > now() - interval 7 day)",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"fresh_until": NOW + timedelta(minutes=5)},
+ ),
+ (
+ "latest_event_time_in_a_fixed_past_range",
+ "select max(timestamp) from events where timestamp >= '2026-10-01' and timestamp < '2026-10-15'",
+ SqlWindowClass.FIXED_PAST,
+ ),
+ (
+ "union_takes_the_freshest",
+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "
+ "union all select count() from events where timestamp > now() - interval 10 minute",
+ SqlWindowClass.LIVE,
+ ),
+ (
+ "unsettled_past_range",
+ PAST_DAY,
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"now": datetime(2026, 10, 29, 0, 30, tzinfo=UTC)},
+ ),
+ (
+ "age_counts_elapsed_time_across_a_dst_fall_back",
+ "select count() from events where timestamp >= now() - interval 7 day",
+ SqlWindowClass.UP_TO_NOW_LONG,
+ {"timezone": AMSTERDAM, "now": datetime(2026, 10, 25, 0, 30, tzinfo=UTC)},
+ ),
+ (
+ "hour_interval_counts_elapsed_time_across_a_dst_fall_back",
+ "select count() from events where timestamp >= now() - interval 1 hour",
+ SqlWindowClass.LIVE,
+ {"timezone": AMSTERDAM, "now": datetime(2026, 10, 25, 1, 30, tzinfo=UTC)},
+ ),
+ ]
+ )
+ def test_grants_window_age(
+ self, _name: str, sql: str, expected_window: SqlWindowClass, options: dict[str, Any] | None = None
+ ) -> None:
+ options = {"now": NOW, **(options or {})}
+ expected_fresh_until = options.pop("fresh_until", options["now"] + SQL_WINDOW_MAX_AGE[expected_window])
+ freshness = self._classify(sql, **options)
+ assert isinstance(freshness, SqlFreshness)
+ assert (freshness.window, freshness.fresh_until) == (expected_window, expected_fresh_until)
+
+ @parameterized.expand(
+ [
+ ("no_time_filter", "select count() from events", FreshnessReason.UNBOUNDED),
+ ("upper_bound_only", "select count() from events where timestamp < now()", FreshnessReason.UNBOUNDED),
+ (
+ "revenue_join",
+ "select poe.revenue_analytics.revenue from events "
+ "where timestamp >= '2025-01-01' and timestamp < '2025-02-01'",
+ FreshnessReason.UNSUPPORTED_FIELD,
+ ),
+ (
+ "clock_in_select_list",
+ "select now(), count() from events where timestamp < '2025-01-01' and timestamp >= '2024-01-01'",
+ FreshnessReason.CLOCK_OUTSIDE_BOUNDS,
+ ),
+ (
+ "or_of_time_ranges",
+ "select count() from events where timestamp > now() - interval 7 day or timestamp < '2020-01-01'",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "today",
+ "select count() from events where toDate(timestamp) = today()",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "yesterday",
+ "select count() from events where timestamp >= yesterday()",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "truncated_column",
+ "select count() from events where toStartOfMonth(timestamp) < '2026-10-01' and timestamp >= '2026-09-01'",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "truncated_value",
+ "select count() from events where timestamp >= toStartOfDay(now())",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "shift_function",
+ "select count() from events where timestamp >= subtractDays(now(), 3)",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "integer_offset",
+ "select count() from events where timestamp >= now() - 3600",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "date_value",
+ "select count() from events where timestamp >= '2026-10-01' and timestamp < toDate('2026-10-27')",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "timezone_on_a_non_literal",
+ "select count() from events where timestamp < toDateTime(now(), 'America/Los_Angeles') "
+ "and timestamp >= '2026-01-01'",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "join_after_a_subquery",
+ "select count() from (select uuid from events where timestamp >= '2026-01-01' "
+ "and timestamp < '2026-02-01') e cross join persons p",
+ FreshnessReason.JOIN,
+ ),
+ (
+ "join",
+ "select count() from events e join events f on e.uuid = f.uuid where e.timestamp > '2026-01-01'",
+ FreshnessReason.JOIN,
+ ),
+ (
+ "persons_table",
+ "select count() from persons where created_at < '2025-01-01'",
+ FreshnessReason.UNSUPPORTED_FIELD,
+ ),
+ (
+ "cohort_membership",
+ "select count() from events where timestamp < '2026-10-01' and timestamp >= '2026-09-01' "
+ "and person_id not in cohort 1",
+ FreshnessReason.FOREIGN_SOURCE,
+ ),
+ (
+ "string_plus_interval",
+ "select count() from events where timestamp >= '2026-11-01' and timestamp < '2026-11-01' + interval 1 day",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "before_the_datetime_range",
+ "select count() from events where timestamp > toDateTime(0) - interval 1 second",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "base_before_the_datetime_range",
+ "select count() from events where timestamp >= toDateTime('1960-01-01 00:00:00') + interval 20 year",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "after_the_datetime_range",
+ "select count() from events where timestamp >= toDateTime('2100-01-01 00:00:00') + interval 10 year",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ),
+ (
+ "literal_at_the_year_limit",
+ "select count() from events where timestamp >= '9999-12-31 23:00:00'",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ ZoneInfo("America/New_York"),
+ ),
+ (
+ "wall_time_in_a_dst_gap",
+ "select count() from events where timestamp >= '2026-03-29 02:30:00'",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ AMSTERDAM,
+ ),
+ (
+ "hour_interval_into_a_repeated_dst_hour",
+ "select count() from events where timestamp >= toDateTime('2026-10-25 01:30:00') + interval 2 hour",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ AMSTERDAM,
+ ),
+ (
+ "calendar_shift_into_a_dst_gap",
+ "select count() from events where timestamp >= toDateTime('2026-03-28 02:30:00') + interval 1 day",
+ FreshnessReason.UNSUPPORTED_BOUND,
+ AMSTERDAM,
+ ),
+ ]
+ )
+ def test_keeps_default_age(
+ self, _name: str, sql: str, expected_reason: FreshnessReason, timezone: Any = None
+ ) -> None:
+ assert self._classify(sql, timezone=timezone) == Reject(reason=expected_reason)
+
+ def test_record_without_a_timezone_is_missing(self) -> None:
+ assert (
+ freshness_from_record({"window": "live", "fresh_until": "2026-10-29T15:05:00", "reason": "window_age"})
+ is None
+ )
+
+
+class TestAcceptedBoundsMatchClickHouse(ClickhouseTestMixin, APIBaseTest):
+ @parameterized.expand(
+ [
+ (f"{name}_{side}", value, timezone, side)
+ for name, value, timezone in ACCEPTED_BOUNDS
+ for side in ("start", "end")
+ ]
+ )
+ def test_bound_matches_clickhouse(self, _name: str, value: str, timezone: str, side: str) -> None:
+ self.team.timezone = timezone
+ self.team.save()
+ bound = f"{value} <= timestamp" if side == "start" else f"timestamp > toDateTime(0) and timestamp < {value}"
+ sql = f"select count() from events where event = 'bound_check' and {bound}"
+ window = extract_facts(
+ _resolve(sql, self.team), now=datetime.now(UTC), timezone=self.team.timezone_info
+ ).windows[0]
+ at = window.start if side == "start" else window.end
+ assert at is not None
+ margin = timedelta(seconds=30 if "now()" in value else 1)
+ for offset in (-margin, margin):
+ _create_event(team=self.team, event="bound_check", distinct_id="d", timestamp=at + offset)
+ flush_persons_and_events()
+ assert execute_hogql_query(sql, team=self.team).results == [(1,)]
diff --git a/posthog/hogql_queries/test/test_hogql_query_runner.py b/posthog/hogql_queries/test/test_hogql_query_runner.py
index 54071e1f634d..9353bdc07552 100644
--- a/posthog/hogql_queries/test/test_hogql_query_runner.py
+++ b/posthog/hogql_queries/test/test_hogql_query_runner.py
@@ -1,13 +1,25 @@
-from datetime import UTC, datetime
-from typing import cast
-
-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events
+from datetime import UTC, datetime, timedelta
+from typing import TYPE_CHECKING, cast
+
+import time_machine
+from posthog.test.base import (
+ APIBaseTest,
+ BaseTest,
+ ClickhouseTestMixin,
+ _create_event,
+ _create_person,
+ flush_persons_and_events,
+)
from unittest.mock import MagicMock, patch
+if TYPE_CHECKING:
+ from unittest.mock import _patch
+
from parameterized import parameterized
from posthog.schema import (
CachedHogQLQueryResponse,
+ DateRange,
HogQLFilters,
HogQLPropertyFilter,
HogQLQuery,
@@ -26,8 +38,11 @@ from posthog.hogql.user_query_validator import HOGQL_PERSONAL_API_KEY_OFFSET_ALL
from posthog.hogql.visitor import clear_locations
from posthog.caching.utils import ThresholdMode, staleness_threshold_map
-from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner
-from posthog.hogql_queries.query_runner import ExecutionMode
+from posthog.clickhouse.query_tagging import tags_context
+from posthog.hogql_queries.hogql_query_runner import API_SQL_WINDOW_CACHE_AGE_FLAG, HogQLQueryRunner
+from posthog.hogql_queries.query_runner import CACHE_ENTRY_FRESHNESS_KEY, ExecutionMode
+from posthog.hogql_queries.sql_freshness.classifier import SqlFreshness, freshness_record
+from posthog.hogql_queries.sql_freshness.rules import FreshnessReason, Reject, SqlWindowClass
from posthog.models.utils import UUIDT
from products.managed_warehouse.backend.facade.query_labels import MANAGED_WAREHOUSE_QUERY_STATUS_LABEL_PREFIX
@@ -35,6 +50,15 @@ from products.product_analytics.backend.facade.models import InsightVariable
from products.warehouse_sources.backend.facade.models import MANAGED_WAREHOUSE_SOURCE_PREFIX, ExternalDataSource
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
+PAST_RANGE = "select count() from events where timestamp >= '2026-10-01' and timestamp < '2026-10-28'"
+
+
+def _freshness_flag(on: bool) -> "_patch[MagicMock]":
+ return patch(
+ "posthog.hogql_queries.query_runner.posthoganalytics.feature_enabled",
+ side_effect=lambda key, *args, **kwargs: on and key == API_SQL_WINDOW_CACHE_AGE_FLAG,
+ )
+
class TestHogQLQueryRunner(ClickhouseTestMixin, APIBaseTest):
maxDiff = None
@@ -573,3 +597,148 @@ class TestHogQLQueryRunner(ClickhouseTestMixin, APIBaseTest):
response = runner.calculate()
self.assertEqual(len(response.results), 5)
+
+ def test_api_key_result_stays_cached_for_its_window(self) -> None:
+ query = HogQLQuery(
+ query=f"select count() from events where timestamp > now() - interval 7 day and event != '{self.random_uuid}'"
+ )
+ with (
+ _freshness_flag(on=True),
+ tags_context(access_method="personal_api_key"),
+ patch("posthog.hogql_queries.query_runner.report_user_or_team_action") as report,
+ ):
+ with time_machine.travel("2026-10-29T15:00:00Z", tick=False):
+ first = self._create_runner(query).run()
+ first_event = report.call_args.args[1]
+ with time_machine.travel("2026-10-29T15:10:00Z", tick=False):
+ second = self._create_runner(query).run()
+ second_event = report.call_args.args[1]
+ assert (getattr(first, "is_cached", None), getattr(second, "is_cached", None)) == (False, True)
+ expiry = datetime(2026, 10, 29, 16, 0, tzinfo=UTC)
+ assert (getattr(first, "cache_target_age", None), getattr(second, "cache_target_age", None)) == (
+ expiry,
+ expiry,
+ )
+ expected_properties = {
+ "sql_cache_window": "up_to_now_long",
+ "sql_cache_fresh_until": expiry.isoformat(),
+ "sql_cache_reason": "window_age",
+ }
+ for event in (first_event, second_event):
+ assert {key: value for key, value in event.items() if key.startswith("sql_cache_")} == expected_properties
+
+ def test_recompute_drops_window_inherited_from_the_cached_entry(self) -> None:
+ runner = self._create_runner(
+ HogQLQuery(query="select count() from events where timestamp > now() - interval 7 day")
+ )
+ runner._read_cache_entry_freshness(
+ {
+ "window": "up_to_now_long",
+ "fresh_until": "2020-01-01T00:00:00+00:00",
+ "reason": "window_age",
+ }
+ )
+ runner.calculate()
+ assert CACHE_ENTRY_FRESHNESS_KEY not in runner._cache_entry_with_freshness({})
+
+
+class TestHogQLQueryRunnerApiSqlFreshness(BaseTest):
+ @parameterized.expand(
+ [
+ ("api_key_uses_its_window", "personal_api_key", True, True, timedelta(hours=1), False, False),
+ ("expired_window_is_stale", "personal_api_key", True, True, timedelta(minutes=-1), False, True),
+ ("lazy_read_keeps_the_longer_default", "personal_api_key", True, True, timedelta(minutes=-1), True, False),
+ ("app_keeps_default", None, True, True, timedelta(hours=1), False, True),
+ ("flag_off_keeps_default", "personal_api_key", False, True, timedelta(hours=1), False, True),
+ ("entry_recomputed_without_window", "personal_api_key", True, False, timedelta(hours=1), False, True),
+ ]
+ )
+ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)
+ def test_entry_window_age_applies_only_to_flagged_api_key_reads(
+ self,
+ _name: str,
+ access_method: str | None,
+ flag_on: bool,
+ entry_has_window: bool,
+ fresh_for: timedelta,
+ lazy: bool,
+ expected_stale: bool,
+ ) -> None:
+ now = datetime(2026, 10, 29, 15, 0, tzinfo=UTC)
+ window = SqlFreshness(
+ window=SqlWindowClass.UP_TO_NOW_LONG, fresh_until=now + fresh_for, reason=FreshnessReason.WINDOW_AGE
+ )
+ with _freshness_flag(on=flag_on), tags_context(access_method=access_method):
+ runner = HogQLQueryRunner(team=self.team, query=HogQLQuery(query="select 1"))
+ runner._read_cache_entry_freshness(freshness_record(window) if entry_has_window else None)
+ stale = runner._is_stale_for_request(last_refresh=now - timedelta(minutes=10), lazy=lazy)
+ assert stale is expected_stale
+
+ def _built_freshness(self, query: HogQLQuery) -> SqlFreshness | Reject | None:
+ with _freshness_flag(on=True), tags_context(access_method="personal_api_key"):
+ runner = HogQLQueryRunner(team=self.team, query=query)
+ runner._build_query()
+ return runner._sql_classification
+
+ @parameterized.expand(
+ [
+ ("filters", f"{PAST_RANGE} and {{filters}}", False),
+ (
+ "filters_in_a_cte",
+ f"with e as (select event from events where {{filters}}) {PAST_RANGE} and event in (select event from e)",
+ False,
+ ),
+ ("variable", f"{PAST_RANGE} and event != {{variables.event_name}}", True),
+ ("expression", f"{PAST_RANGE} and event != {{toString(now())}}", False),
+ ]
+ )
+ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)
+ def test_placeholders_keep_default_age(self, _name: str, query: str, uses_variable: bool) -> None:
+ variable = InsightVariable.objects.create(team=self.team, name="Event", code_name="event_name", type="String")
+ freshness = self._built_freshness(
+ HogQLQuery(
+ query=query,
+ filters=HogQLFilters(dateRange=DateRange(date_from="-3d")),
+ variables={
+ str(variable.id): HogQLVariable(code_name="event_name", variableId=str(variable.id), value="x")
+ }
+ if uses_variable
+ else None,
+ )
+ )
+ assert freshness == Reject(reason=FreshnessReason.UNSUPPORTED_PLACEHOLDER)
+
+ def test_classifier_error_keeps_default_age(self) -> None:
+ with (
+ patch("posthog.hogql_queries.sql_freshness.classifier.fresh_until", side_effect=RuntimeError),
+ patch("posthog.hogql_queries.sql_freshness.classifier.capture_exception") as capture,
+ ):
+ freshness = self._built_freshness(HogQLQuery(query=PAST_RANGE))
+ assert freshness == Reject(reason=FreshnessReason.UNPARSEABLE)
+ assert capture.call_args.args[1] == {"context": "sql_freshness_classification"}
+
+ @time_machine.travel("2026-10-29T15:00:00Z", tick=False)
+ def test_named_values_read_as_literals(self) -> None:
+ freshness = self._built_freshness(
+ HogQLQuery(
+ query="select count() from events where timestamp >= {start} and timestamp < {end}",
+ values={"start": "2026-10-01", "end": "2026-10-15"},
+ )
+ )
+ assert isinstance(freshness, SqlFreshness)
+ assert freshness.window == SqlWindowClass.FIXED_PAST
+
+ @parameterized.expand([("project_timezone", None, "fixed_past"), ("utc", False, "up_to_now_long")])
+ def test_bounds_follow_the_query_timezone(
+ self, _name: str, convert_to_project_timezone: bool | None, expected_window: str
+ ) -> None:
+ self.team.timezone = "Asia/Tokyo"
+ self.team.save()
+ query = HogQLQuery(
+ query="select count() from events where timestamp >= '2026-10-20' and timestamp < '2026-10-29'",
+ modifiers=HogQLQueryModifiers(convertToProjectTimezone=convert_to_project_timezone),
+ )
+ with time_machine.travel(datetime(2026, 10, 28, 16, 0, tzinfo=UTC), tick=False):
+ freshness = self._built_freshness(query)
+ assert isinstance(freshness, SqlFreshness)
+ assert freshness.window.value == expected_window