Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(events): apply the documented 24h default for a missing after (#128889)

Vasco de Krijgerbfcaa1509d9b
diff --git a/posthog/api/event.py b/posthog/api/event.py
index 3005d43d813e..48c382dc2a82 100644
--- a/posthog/api/event.py
+++ b/posthog/api/event.py
@@ -39,7 +39,12 @@ from posthog.errors import ExposedCHQueryError
from posthog.event_usage import get_request_analytics_properties
from posthog.exceptions_capture import capture_exception
from posthog.models import Element, Person, PropertyDefinition, User
-from posthog.models.event.legacy_events_query import LegacyEventsListQuery, get_one_event
+from posthog.models.event.legacy_events_query import (
+ EVENT_LIST_DEFAULT_AFTER_WINDOW,
+ LegacyEventsListQuery,
+ get_one_event,
+ resolve_before,
+)
from posthog.models.event.util import ClickhouseEventSerializer
from posthog.models.person.util import get_persons_mapped_by_distinct_id
from posthog.models.team import Team
@@ -66,6 +71,7 @@ EVENT_VALUES_COUNTER = Counter(
QUERY_DEFAULT_EXPORT_LIMIT = 1_000
EVENT_LIST_MAX_LIMIT = 1_000
+DEFAULT_AFTER_HEADER = "X-PostHog-Events-Default-After"
# Legacy property-filter keys the frontend still appends but EventsQuery's schema forbids.
@@ -203,8 +209,11 @@ class EventViewSet(
request: request.Request,
last_event_timestamp: datetime,
order_by: list[str],
+ after: Optional[str],
) -> str:
params = request.GET.dict()
+ if after:
+ params["after"] = after
reverse = "-timestamp" in order_by
timestamp = last_event_timestamp.replace(tzinfo=UTC).isoformat()
if reverse:
@@ -309,13 +318,21 @@ class EventViewSet(
restricted_context = self._get_restricted_properties_context(request, team)
self._reject_restricted_property_references(properties, order_by, restricted_context)
+ before = request.GET.get("before")
+ after = request.GET.get("after")
+ default_after: Optional[str] = None
+ if not after:
+ before_dt = resolve_before(before, team.timezone_info)
+ before = before_dt.isoformat()
+ default_after = after = (before_dt.astimezone(UTC) - EVENT_LIST_DEFAULT_AFTER_WINDOW).isoformat()
+
request_user = cast(Optional[User], request.user if request.user.is_authenticated else None)
query_result, has_more = LegacyEventsListQuery(team, request_user).run(
limit=limit,
offset=offset,
order=order,
- before=request.GET.get("before"),
- after=request.GET.get("after"),
+ before=before,
+ after=after,
event=request.GET.get("event"),
person_id=request.GET.get("person_id"),
distinct_id=request.GET.get("distinct_id"),
@@ -335,17 +352,17 @@ class EventViewSet(
next_url: Optional[str] = None
if not is_csv_request and has_more and query_result:
- next_url = self._build_next_url(request, query_result[-1]["timestamp"], order_by)
- headers = None
+ next_url = self._build_next_url(request, query_result[-1]["timestamp"], order_by, after)
+ headers: dict[str, str] = {}
if settings.PATCH_EVENT_LIST_MAX_OFFSET > 0:
- headers = {"X-PostHog-Warn": "https://posthog.com/docs/api/events"}
+ headers["X-PostHog-Warn"] = "https://posthog.com/docs/api/events"
elif deprecate_offset and offset:
- headers = {
- "X-PostHog-Warn": (
- "offset is deprecated. "
- "Use: https://posthog.com/docs/api/queries#5-use-timestamp-based-pagination-instead-of-offset"
- )
- }
+ headers["X-PostHog-Warn"] = (
+ "offset is deprecated. "
+ "Use: https://posthog.com/docs/api/queries#5-use-timestamp-based-pagination-instead-of-offset"
+ )
+ if default_after:
+ headers[DEFAULT_AFTER_HEADER] = default_after
return response.Response({"next": next_url, "results": result}, headers=headers)
except Exception as ex:
diff --git a/posthog/api/test/test_event.py b/posthog/api/test/test_event.py
index d09178e6e4ce..a385c1c366b7 100644
--- a/posthog/api/test/test_event.py
+++ b/posthog/api/test/test_event.py
@@ -1,6 +1,6 @@
import json
from datetime import datetime
-from urllib.parse import unquote, urlencode
+from urllib.parse import parse_qs, unquote, urlencode, urlparse
from zoneinfo import ZoneInfo
import time_machine
@@ -155,7 +155,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# +1 for the saved-expressions fetch in the HogQL database build. +1 each for the flag-cache
# TTL, projection kill-switch and properties-narrowing instance settings, cold-cache here but
# TTL-cached per worker in production. +1 for the organization's flag_evaluations mode lookup in the HogQL database build.
- expected_queries = 25 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 26
+ expected_queries = 24 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 25
with self.assertNumQueries(expected_queries):
response = self.client.get(
@@ -233,7 +233,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
with self.settings(USE_PRECALCULATED_CH_COHORT_PEOPLE=True): # Normally this is False in tests
with time_machine.travel("2020-01-04T13:01:01Z", tick=False):
response = self.client.get(
- f"/api/projects/{self.team.id}/events/?properties=%s"
+ f"/api/projects/{self.team.id}/events/?after=2020-01-01&properties=%s"
% (json.dumps([{"key": "id", "value": cohort1.id, "type": "cohort"}]))
).json()
@@ -734,7 +734,8 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
assert response["results"][0]["id"] == event1_uuid
response = self.client.get(
- f"/api/projects/{self.team.id}/events/?before=2020-01-09T00:00:00.000Z&action_id=%s" % action.pk
+ f"/api/projects/{self.team.id}/events/?after=2020-01-01T00:00:00.000Z&before=2020-01-09T00:00:00.000Z&action_id=%s"
+ % action.pk
).json()
assert len(response["results"]) == 1
assert response["results"][0]["id"] == event2_uuid
@@ -744,7 +745,9 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
assert len(response["results"]) == 1
assert response["results"][0]["id"] == event1_uuid
- response = self.client.get(f"/api/projects/{self.team.id}/events/?before=2020-01-09T00:00:00.000Z").json()
+ response = self.client.get(
+ f"/api/projects/{self.team.id}/events/?after=2020-01-01T00:00:00.000Z&before=2020-01-09T00:00:00.000Z"
+ ).json()
assert len(response["results"]) == 2
assert response["results"][0]["id"] == event2_uuid
assert response["results"][1]["id"] == event3_uuid
@@ -759,15 +762,17 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
distinct_id="1",
timestamp=timezone.now() - relativedelta(months=11) + relativedelta(days=idx, seconds=idx),
)
- response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1").json()
+ response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1&after=2020-01-01").json()
assert len(response["results"]) == 100
- assert f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&before=" in unquote(
- response["next"]
+ assert (
+ f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&after=2020-01-01&before="
+ in unquote(response["next"])
)
- response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1").json()
+ response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1&after=2020-01-01").json()
assert len(response["results"]) == 100
- assert f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&before=" in unquote(
- response["next"]
+ assert (
+ f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&after=2020-01-01&before="
+ in unquote(response["next"])
)
page2 = self.client.get(response["next"]).json()
@@ -785,7 +790,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
assert len(page2["results"]) == 100
assert (
unquote(page2["next"])
- == f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&before=2020-12-30T12:03:53.829294+00:00"
+ == f"http://testserver/api/projects/{self.team.id}/events/?distinct_id=1&after=2020-01-01&before=2020-12-30T12:03:53.829294+00:00"
)
page3 = self.client.get(page2["next"]).json()
@@ -850,7 +855,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
)
response = self.client.get(
- f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10&orderBy={json.dumps(['timestamp'])}"
+ f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10&after=-360d&orderBy={json.dumps(['timestamp'])}"
).json()
assert len(response["results"]) == 10
assert parser.parse(response["results"][0]["timestamp"]) < parser.parse(response["results"][-1]["timestamp"])
@@ -865,7 +870,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
timestamp=timezone.now() - relativedelta(months=11) + relativedelta(days=idx, seconds=idx),
)
- response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10").json()
+ response = self.client.get(f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10&after=-360d").json()
assert len(response["results"]) == 10
assert parser.parse(response["results"][0]["timestamp"]) > parser.parse(response["results"][-1]["timestamp"])
assert "before=" in response["next"]
@@ -880,7 +885,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
)
response = self.client.get(
- f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10&orderBy={json.dumps(['-timestamp'])}"
+ f"/api/projects/{self.team.id}/events/?distinct_id=1&limit=10&after=-360d&orderBy={json.dumps(['-timestamp'])}"
).json()
assert len(response["results"]) == 10
assert parser.parse(response["results"][0]["timestamp"]) > parser.parse(response["results"][-1]["timestamp"])
@@ -1128,7 +1133,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# until finding one with >= half_limit results, then falls back to full range
# Windows: [60, 300, 900, 3600, 21600, 86400] seconds
- # With only 1 result (< half of default limit 100), tries all 6 windows + fallback = 7 calls
+ # With only 1 result (< half of default limit 100), tries the 5 shorter windows + the 24h default = 6 calls
patch_execute_query.return_value = (
[
{
@@ -1145,7 +1150,7 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
)
response = self.client.get(f"/api/projects/{self.team.id}/events/").json()
assert len(response["results"]) == 1
- assert patch_execute_query.call_count == 7 # 6 windows + 1 fallback
+ assert patch_execute_query.call_count == 6
# With 50+ results (>= half of limit 100), succeeds on first window = 1 call
patch_execute_query.reset_mock()
@@ -1168,9 +1173,8 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
assert patch_execute_query.call_count == 1
@patch("posthog.models.event.legacy_events_query._execute_events_list_query", wraps=_execute_events_list_query)
- @override_settings(PATCH_EVENT_LIST_MAX_OFFSET=2)
def test_default_after(self, patch_execute_query):
- # With PATCH_EVENT_LIST_MAX_OFFSET=2, default after = before - 24h
+ # A missing after defaults to before - 24h
# Progressive windows to try (< 86400s): [60, 300, 900, 3600, 21600] = 5 windows
# Events at: 01-01T01:01, 01-02T02:02, ..., 01-09T09:09
[
@@ -1202,11 +1206,65 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
assert len(response["results"]) == 1
assert patch_execute_query.call_count == 14
- # No events after 01-09T09:09, all windows fail including fallback
- # 5 windows + fallback = 6 calls (cumulative: 20)
+ # No events after 01-09T09:09. The previous request cached the full 24h range, so no probes run
+ # 1 call (cumulative: 15)
response = self.client.get(f"/api/projects/{self.team.id}/events/?before=2024-01-10T10:20:02Z&limit=1").json()
assert len(response["results"]) == 0
- assert patch_execute_query.call_count == 20
+ assert patch_execute_query.call_count == 15
+
+ def test_missing_after_defaults_to_24h(self):
+ with time_machine.travel("2024-03-10T12:00:00Z", tick=False):
+ old_event = _create_event(
+ team=self.team, event="old", distinct_id="1", timestamp=timezone.now() - relativedelta(days=3)
+ )
+ recent_events = [
+ _create_event(
+ team=self.team,
+ event="recent",
+ distinct_id="1",
+ timestamp=timezone.now() - relativedelta(seconds=seconds),
+ )
+ for seconds in (10, 20, 30)
+ ]
+ flush_persons_and_events()
+
+ first_page = self.client.get(
+ f"/api/projects/{self.team.id}/events/?limit=2", HTTP_ORIGIN="https://example.com"
+ )
+ full_page = self.client.get(f"/api/projects/{self.team.id}/events/")
+ explicit_after = self.client.get(
+ f"/api/projects/{self.team.id}/events/",
+ data={"after": (timezone.now() - relativedelta(days=7)).isoformat()},
+ )
+
+ assert first_page.headers["X-PostHog-Events-Default-After"] == "2024-03-09T12:00:05+00:00"
+ assert "X-PostHog-Events-Default-After" in first_page.headers["Access-Control-Expose-Headers"]
+ assert parse_qs(urlparse(first_page.json()["next"]).query)["after"] == ["2024-03-09T12:00:05+00:00"]
+ assert [event["id"] for event in full_page.json()["results"]] == recent_events
+ assert "X-PostHog-Events-Default-After" not in explicit_after.headers
+ assert [event["id"] for event in explicit_after.json()["results"]] == [*recent_events, old_event]
+
+ @parameterized.expand(
+ [
+ ("clocks_go_forward", "2024-03-10T12:00:00Z", "2024-03-09T12:00:05+00:00"),
+ ("clocks_go_back", "2024-11-03T12:00:00Z", "2024-11-02T12:00:05+00:00"),
+ ]
+ )
+ def test_missing_after_spans_24_hours_across_a_clock_change(self, _name, now, expected_after):
+ self.team.timezone = "America/New_York"
+ self.team.save()
+
+ with (
+ time_machine.travel(now, tick=False),
+ patch(
+ "posthog.models.event.legacy_events_query._execute_events_list_query",
+ wraps=_execute_events_list_query,
+ ) as patch_execute_query,
+ ):
+ response = self.client.get(f"/api/projects/{self.team.id}/events/")
+
+ assert response.headers["X-PostHog-Events-Default-After"] == expected_after
+ assert patch_execute_query.call_count == 6
def test_optimize_query_with_bounded_dates(self):
# Test that bounded date ranges return correct results
@@ -1613,6 +1671,37 @@ class TestEventListTimeWindowOptimization(ClickhouseTestMixin, APIBaseTest):
# Should not cache anything when fallback is used
mock_cache.set.assert_not_called()
+ @parameterized.expand(
+ [
+ ("first_poll_probes_then_caches_the_range", None, 6, {"window": 86400, "result_count": 60}),
+ ("later_poll_skips_the_probes", {"window": 86400, "result_count": 60}, 1, None),
+ ]
+ )
+ @patch("posthog.models.event.legacy_events_query.cache")
+ @patch("posthog.models.event.legacy_events_query._execute_events_list_query")
+ def test_range_as_wide_as_a_window_reuses_it(
+ self, _name, cached, expected_queries, expected_cached, patch_execute_query, mock_cache
+ ):
+ mock_cache.get.return_value = cached
+ row = {
+ "uuid": "event",
+ "event": "test",
+ "properties": "{}",
+ "timestamp": timezone.now(),
+ "team_id": str(self.team.pk),
+ "distinct_id": "1",
+ "elements_chain": "",
+ }
+ patch_execute_query.side_effect = [([], False)] * (expected_queries - 1) + [([row] * 60, False)]
+
+ self.client.get(f"/api/projects/{self.team.id}/events/?after=2024-01-01T00:00:00Z&before=2024-01-02T00:00:00Z")
+
+ assert patch_execute_query.call_count == expected_queries
+ if expected_cached:
+ assert mock_cache.set.call_args[0][1] == expected_cached
+ else:
+ mock_cache.set.assert_not_called()
+
@patch("posthog.models.event.legacy_events_query._execute_events_list_query")
def test_only_tries_windows_shorter_than_request(self, patch_execute_query):
# Request a 10-minute window (600 seconds)
diff --git a/posthog/models/event/legacy_events_query.py b/posthog/models/event/legacy_events_query.py
index 6f7e258d7d80..494ad3d459fd 100644
--- a/posthog/models/event/legacy_events_query.py
+++ b/posthog/models/event/legacy_events_query.py
@@ -1,8 +1,9 @@
import uuid
import random
import builtins
-from datetime import datetime, timedelta
+from datetime import UTC, datetime, timedelta
from typing import Optional, cast
+from zoneinfo import ZoneInfo
from django.conf import settings
from django.core.cache import cache
@@ -35,6 +36,7 @@ EVENT_LIST_SELECT_COLUMNS = ["uuid", "event", "properties", "timestamp", "distin
EVENT_LIST_TIME_WINDOWS = [60, 300, 900, 3600, 21600, 86400]
EVENT_LIST_CACHE_TTL = 86400 # 24 hours
EVENT_LIST_CACHE_KEY_PREFIX = "event_list_good_period"
+EVENT_LIST_DEFAULT_AFTER_WINDOW = timedelta(hours=24)
def _get_limit_size_category(limit: int) -> str:
@@ -81,6 +83,11 @@ def _execute_events_list_query(runner: EventsQueryRunner, database: Database) ->
return rows, runner.paginator.has_more()
+def resolve_before(before: Optional[str], tzinfo: ZoneInfo) -> datetime:
+ # Just past now, so events ingested a moment ago aren't excluded.
+ return relative_date_parse(before, tzinfo) if before else datetime.now(tzinfo) + timedelta(seconds=5)
+
+
def get_one_event(team: Team, pk: str) -> Optional[dict]:
"""Fetch a single event by uuid for the events retrieve endpoint, or None if it doesn't exist."""
query = ast.SelectQuery(
@@ -146,9 +153,9 @@ class LegacyEventsListQuery:
if before and after:
try:
tzinfo = self.team.timezone_info
- request_window_seconds = int(
- (relative_date_parse(before, tzinfo) - relative_date_parse(after, tzinfo)).total_seconds()
- )
+ before_utc = relative_date_parse(before, tzinfo).astimezone(UTC)
+ after_utc = relative_date_parse(after, tzinfo).astimezone(UTC)
+ request_window_seconds = int((before_utc - after_utc).total_seconds())
except (ValueError, TypeError):
pass
@@ -165,7 +172,9 @@ class LegacyEventsListQuery:
cached_window = cached_data.get("window")
elif cached_data and isinstance(cached_data, int):
cached_window = cached_data # legacy cache format: bare window integer
- if cached_window and cached_window in windows_to_try:
+ if cached_window and request_window_seconds is not None and cached_window >= request_window_seconds:
+ windows_to_try = []
+ elif cached_window and cached_window in windows_to_try:
windows_to_try.remove(cached_window)
windows_to_try.insert(0, cached_window)
@@ -196,13 +205,16 @@ class LegacyEventsListQuery:
successful_window = window
break
+ if not successful_window and (applied_window is not None or not windows_to_try):
+ # Windows were applied but came up short, or there were none — run the full range.
+ rows, has_more, applied_window = self.run_page(**page_kwargs)
+ if request_window_seconds in EVENT_LIST_TIME_WINDOWS and len(rows) >= half_limit:
+ successful_window = request_window_seconds
+
if successful_window:
new_cache_data = {"window": successful_window, "result_count": len(rows)}
if new_cache_data != cached_data:
cache.set(cache_key, new_cache_data, EVENT_LIST_CACHE_TTL)
- elif applied_window is not None or not windows_to_try:
- # Windows were applied but came up short, or there were none — run the full range.
- rows, has_more, applied_window = self.run_page(**page_kwargs)
return rows, has_more
@@ -228,18 +240,12 @@ class LegacyEventsListQuery:
"""
tzinfo = self.team.timezone_info
- # Resolve the [after_dt, before_dt) bounds. `before` defaults to just past now (so events
- # ingested a moment ago aren't excluded); `after` stays open unless requested.
+ # Resolve the [after_dt, before_dt) bounds. `after` stays open unless the caller passes one.
# PATCH_EVENT_LIST_MAX_OFFSET clamps this deprecated endpoint's ClickHouse cost as a graduated
- # rollout (0 = off, 1 = migration, 2 = enabled; see settings/data_stores.py): at 2 a missing
- # `after` defaults to before-24h, and a range over a year is rejected (always at 2, ~1% at 1).
- before_dt = relative_date_parse(before, tzinfo) if before else datetime.now(tzinfo) + timedelta(seconds=5)
- if after:
- after_dt: Optional[datetime] = relative_date_parse(after, tzinfo)
- elif settings.PATCH_EVENT_LIST_MAX_OFFSET > 1:
- after_dt = before_dt - timedelta(hours=24)
- else:
- after_dt = None
+ # rollout (0 = off, 1 = migration, 2 = enabled; see settings/data_stores.py): a range over a
+ # year is rejected (always at 2, ~1% at 1).
+ before_dt = resolve_before(before, tzinfo)
+ after_dt: Optional[datetime] = relative_date_parse(after, tzinfo) if after else None
if settings.PATCH_EVENT_LIST_MAX_OFFSET > 0 and after_dt is not None:
if (before_dt - after_dt) > timedelta(days=366) and (
diff --git a/posthog/settings/web.py b/posthog/settings/web.py
index e9df48a0786c..165c2567b749 100644
--- a/posthog/settings/web.py
+++ b/posthog/settings/web.py
@@ -316,6 +316,7 @@ LOGIN_REDIRECT_URL = "/"
APPEND_SLASH = False
CORS_URLS_REGEX = r"^(/site_app/|/array/|/static/|/oauth/token/?|/toolbar_oauth/check|/api/(?!early_access_features|surveys|web_experiments).*$)"
CORS_ALLOW_HEADERS = default_headers + CORS_ALLOWED_TRACING_HEADERS + ("x-posthog-client-provenance",)
+CORS_EXPOSE_HEADERS = ["X-PostHog-Events-Default-After"]
X_FRAME_OPTIONS = "SAMEORIGIN"
SOCIAL_AUTH_JSONFIELD_ENABLED = True