Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add pendo event and track type tables (#124786)

Tom Owers8be1038e7dcc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 8cc8a6bede2b..189ca9fbe50e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9270,10 +9270,10 @@ Today (5): `accounts`, `features`, `guides`, `pages`, `visitors`
Diffed against: <https://engageapi.pendo.io/api/collections/16265887/Tzm6jvKG>
- [ ] `aggregation source: events` — the raw usage event stream - Pendo's headline dataset and the basis of every engagement metric (high)
-- [ ] `aggregation source: featureEvents` — feature click events joined to the features table already synced (high)
-- [ ] `aggregation source: pageEvents` — page view events joined to the pages table already synced (high)
-- [ ] `GET /api/v1/tracktype` — lookup resolving custom track event type IDs; also needed to interpret trackEvents (high)
-- [ ] `aggregation source: trackEvents` — custom-instrumented events, the analog of product analytics custom events (high)
+- [x] `aggregation source: featureEvents` — feature click events joined to the features table already synced (high)
+- [x] `aggregation source: pageEvents` — page view events joined to the pages table already synced (high)
+- [x] `GET /api/v1/tracktype` — lookup resolving custom track event type IDs; also needed to interpret trackEvents (high)
+- [x] `aggregation source: trackEvents` — custom-instrumented events, the analog of product analytics custom events (high)
- [ ] `aggregation source: guideEvents (plus guidesSeen)` — guide impressions and interactions for the guides table already synced (medium)
- [ ] `aggregation source: pollEvents / pollsSeen` — poll and NPS responses, the source of Pendo's NPS reporting (medium)
- [ ] `aggregation source: surveyResponses` — survey response rows for voice-of-customer analysis (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py
index 12fbfa184f7d..f1316e3452c2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py
@@ -77,4 +77,81 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"metadata": "Custom and agent-collected metadata fields attached to the account.",
},
},
+ "track_types": {
+ "description": "A custom track event type, named by your track event calls and identified by a Pendo-generated ID.",
+ "docs_url": "https://engageapi.pendo.io/#a55c1ce4-1e9c-4e64-a78e-fc824ade8edd",
+ "columns": {
+ "id": "Unique identifier for the track type.",
+ "name": "The track type's display name.",
+ "trackTypeName": "The event name sent in track event calls.",
+ "appId": "ID of the application the track type belongs to.",
+ "eventPropertyNameList": "Names of the event properties seen on this track type.",
+ "isCoreEvent": "Whether the track type is marked as a core event.",
+ "createdAt": "Time at which the track type was created, as a Unix timestamp in milliseconds.",
+ "lastUpdatedAt": "Time at which the track type was last updated, as a Unix timestamp in milliseconds.",
+ },
+ },
+ "page_events": {
+ "description": "Daily count of page load events for tagged pages, grouped per visitor, account, page and browser context.",
+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",
+ "columns": {
+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",
+ "pageId": "The ID of the page these events are associated with.",
+ "visitorId": "The visitor ID associated with the events.",
+ "accountId": "The account ID associated with the events.",
+ "appId": "ID of the application the events were collected in.",
+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",
+ "numEvents": "Number of page load events in this bucket.",
+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",
+ "server": "The hostname of the URL at the time of the events.",
+ "remoteIp": "The IP address captured at the time of the events.",
+ "userAgent": "The browser and operating system of the visitor.",
+ "tabId": "Identifier of the browser window or tab the events came from.",
+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",
+ "properties": "Historical metadata and event properties captured with the events.",
+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",
+ },
+ },
+ "feature_events": {
+ "description": "Daily count of click events for tagged features, grouped per visitor, account, feature and browser context.",
+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",
+ "columns": {
+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",
+ "featureId": "The ID of the feature these events are associated with.",
+ "visitorId": "The visitor ID associated with the events.",
+ "accountId": "The account ID associated with the events.",
+ "appId": "ID of the application the events were collected in.",
+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",
+ "numEvents": "Number of click events in this bucket.",
+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",
+ "server": "The hostname of the URL at the time of the events.",
+ "remoteIp": "The IP address captured at the time of the events.",
+ "userAgent": "The browser and operating system of the visitor.",
+ "tabId": "Identifier of the browser window or tab the events came from.",
+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",
+ "properties": "Historical metadata and event properties captured with the events.",
+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",
+ },
+ },
+ "track_events": {
+ "description": "Daily count of custom track events, grouped per visitor, account, track type and browser context.",
+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",
+ "columns": {
+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",
+ "trackTypeId": "The ID of the track type these events are associated with.",
+ "visitorId": "The visitor ID associated with the events.",
+ "accountId": "The account ID associated with the events.",
+ "appId": "ID of the application the events were collected in.",
+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",
+ "numEvents": "Number of track events in this bucket.",
+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",
+ "server": "The hostname of the URL at the time of the events.",
+ "remoteIp": "The IP address captured at the time of the events.",
+ "userAgent": "The browser and operating system of the visitor.",
+ "tabId": "Identifier of the browser window or tab the events came from.",
+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",
+ "properties": "Historical metadata and event properties captured with the events.",
+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
index 980bd8e71411..f088c67f68c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
@@ -1,11 +1,15 @@
-import dataclasses
+import json
+import hashlib
from collections.abc import Iterator
+from datetime import UTC, date, datetime, time, timedelta
from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
make_tracked_session,
retry_with_deadline,
@@ -14,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.settings import (
DEFAULT_REGION,
+ EVENTS_DEFAULT_LOOKBACK_DAYS,
PENDO_ENDPOINTS,
PENDO_REGION_BASE_URLS,
)
@@ -26,20 +31,39 @@ AGGREGATION_PAGE_SIZE = 5000
# unbounded list in memory and so resume state can advance per chunk.
LIST_CHUNK_SIZE = 1000
REQUEST_TIMEOUT_SECONDS = 60
+# Pendo itself cuts an aggregation off after 5 minutes, and a busy day of events can take most of that.
+EVENTS_REQUEST_TIMEOUT_SECONDS = 300
MAX_RETRIES = 5
+# Pendo buckets events by day in the subscription's timezone, which the API does not expose. Noon UTC
+# falls in the same local day for every offset from -11h to +14h, so stepping a noon-UTC anchor one
+# day at a time requests each local day exactly once.
+EVENT_WINDOW_ANCHOR = timedelta(hours=12)
+EVENT_ALL_APPS = 'expandAppIds("*")'
+# Pendo's daily and weekly reprocessing can add backdated events up to 7 days after the fact, so
+# an incremental run re-reads that many days before the watermark's day.
+EVENTS_REPROCESSING_OVERLAP_DAYS = 7
+# A busy day can hold a very large number of buckets; hand them to the pipeline in bounded batches.
+EVENT_BATCH_SIZE = 5000
+# Measures Pendo sums into each event bucket. The remaining fields identify the bucket, so they
+# alone form its id and a re-read of a still-open day updates the row instead of duplicating it.
+EVENT_MEASURE_FIELDS = frozenset(
+ {"numEvents", "numMinutes", "rageClickCount", "errorClickCount", "deadClickCount", "uTurnCount", "lastTime"}
+)
class PendoRetryableError(Exception):
pass
+@frozen
class PendoResumeConfig:
# Number of rows already yielded for a list endpoint schema; on resume we skip past them.
offset: int = 0
# Sort-field value of the last row yielded for an aggregation endpoint schema. Pendo's
# aggregation pipeline has no skip/offset stage, so resuming re-filters for values past this.
last_id: Optional[str] = None
+ # ISO date of the next UTC day to request for an event endpoint schema.
+ next_day: Optional[str] = None
def get_base_url(region: Optional[str]) -> str:
@@ -65,9 +89,10 @@ def _request(
method: str,
url: str,
logger: FilteringBoundLogger,
+ timeout: float = REQUEST_TIMEOUT_SECONDS,
**kwargs: Any,
) -> requests.Response:
- response = session.request(method, url, timeout=REQUEST_TIMEOUT_SECONDS, **kwargs)
+ response = session.request(method, url, timeout=timeout, **kwargs)
if response.status_code == 429 or response.status_code >= 500:
raise PendoRetryableError(f"Pendo API error (retryable): status={response.status_code}, url={url}")
@@ -173,12 +198,85 @@ def _iter_aggregation(
break
+def _to_epoch_ms(value: Any) -> Optional[int]:
+ if isinstance(value, bool) or value is None:
+ return None
+ if isinstance(value, int | float):
+ return int(value)
+ if isinstance(value, str):
+ try:
+ return int(value)
+ except ValueError:
+ return None
+ return None
+
+
+def _events_start_day(db_incremental_field_last_value: Any, today: date) -> date:
+ watermark = _to_epoch_ms(db_incremental_field_last_value)
+ if watermark is None:
+ return today - timedelta(days=EVENTS_DEFAULT_LOOKBACK_DAYS)
+ return datetime.fromtimestamp(watermark / 1000, tz=UTC).date() - timedelta(days=EVENTS_REPROCESSING_OVERLAP_DAYS)
+
+
+def _with_bucket_id(row: dict[str, Any]) -> dict[str, Any]:
+ identity = {key: value for key, value in row.items() if key not in EVENT_MEASURE_FIELDS}
+ digest = hashlib.sha256(json.dumps(identity, sort_keys=True, default=str).encode()).hexdigest()
+ return {"id": digest, **row}
+
+
+def _iter_events(
+ session: requests.Session,
+ base_url: str,
+ event_source: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PendoResumeConfig],
+ start_day: date,
+ end_day: date,
+) -> Iterator[list[dict[str, Any]]]:
+ url = f"{base_url}{AGGREGATION_PATH}"
+ day = start_day
+
+ while day <= end_day:
+ anchor = datetime.combine(day, time(), tzinfo=UTC) + EVENT_WINDOW_ANCHOR
+ body = {
+ "response": {"mimeType": "application/json"},
+ "request": {
+ "requestId": f"posthog-{event_source}",
+ "pipeline": [
+ {
+ "source": {
+ event_source: {"appId": EVENT_ALL_APPS},
+ "timeSeries": {"period": "dayRange", "first": int(anchor.timestamp() * 1000), "count": 1},
+ }
+ }
+ ],
+ },
+ }
+ response = _request(session, "POST", url, logger, timeout=EVENTS_REQUEST_TIMEOUT_SECONDS, json=body)
+ data = response.json()
+ del response
+ rows: list[dict[str, Any]] = (data.get("results") or []) if isinstance(data, dict) else data
+
+ day += timedelta(days=1)
+ if not rows:
+ resumable_source_manager.save_state(PendoResumeConfig(next_day=day.isoformat()))
+ resumable_source_manager.safe_point()
+ continue
+
+ for start in range(0, len(rows), EVENT_BATCH_SIZE):
+ # Only the day's last batch moves the cursor past it, so a resume re-reads a partly written day.
+ if start + EVENT_BATCH_SIZE >= len(rows):
+ resumable_source_manager.save_state(PendoResumeConfig(next_day=day.isoformat()))
+ yield [_with_bucket_id(row) for row in rows[start : start + EVENT_BATCH_SIZE]]
+
+
def get_rows(
integration_key: str,
region: Optional[str],
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[PendoResumeConfig],
+ db_incremental_field_last_value: Any = None,
) -> Iterator[list[dict[str, Any]]]:
config = PENDO_ENDPOINTS[endpoint]
base_url = get_base_url(region)
@@ -187,7 +285,24 @@ def get_rows(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
- if config.is_aggregation:
+ if config.event_source is not None:
+ # The last day whose anchor is not in the future.
+ end_day = (datetime.now(tz=UTC) - EVENT_WINDOW_ANCHOR).date()
+ if resume and resume.next_day:
+ start_day = date.fromisoformat(resume.next_day)
+ logger.debug(f"Pendo: resuming endpoint={endpoint} from day={start_day}")
+ else:
+ start_day = _events_start_day(db_incremental_field_last_value, end_day)
+ yield from _iter_events(
+ session,
+ base_url,
+ config.event_source,
+ logger,
+ resumable_source_manager,
+ start_day,
+ end_day,
+ )
+ elif config.is_aggregation:
if config.aggregation_source is None:
raise ValueError(f"Endpoint '{endpoint}' is marked as aggregation but has no aggregation_source")
start_after = resume.last_id if resume else None
@@ -224,6 +339,7 @@ def pendo_source(
endpoint: str,
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[PendoResumeConfig],
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
config = PENDO_ENDPOINTS[endpoint]
@@ -235,6 +351,7 @@ def pendo_source(
endpoint=endpoint,
logger=logger,
resumable_source_manager=resumable_source_manager,
+ db_incremental_field_last_value=db_incremental_field_last_value,
),
primary_keys=config.primary_keys,
# No source-level partitioning: Pendo timestamps are epoch-milliseconds, which the
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py
index 04a173320420..22f418f77f6c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py
@@ -1,7 +1,9 @@
-from dataclasses import dataclass, field
+from dataclasses import field
from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# Pendo hosts each subscription in a region-specific data center and the integration key is
# region-scoped, so the base URL has to match where the subscription lives.
@@ -16,32 +18,65 @@ PENDO_REGION_BASE_URLS: dict[str, str] = {
DEFAULT_REGION = "us"
+# How many days of events a full refresh or a first incremental sync reads.
+EVENTS_DEFAULT_LOOKBACK_DAYS = 90
+
-@dataclass
+@frozen
class PendoEndpointConfig:
name: str
primary_keys: list[str]
- # GET list endpoints (feature/page/guide) set `path`; aggregation endpoints
- # (visitors/accounts) set `aggregation_source` and POST to /api/v1/aggregation.
+ # GET list endpoints (feature/page/guide/tracktype) set `path`; aggregation endpoints
+ # (visitors/accounts) set `aggregation_source` and POST to /api/v1/aggregation. Event
+ # endpoints set `event_source` and POST a day-windowed `timeSeries` to the same endpoint.
path: Optional[str] = None
aggregation_source: Optional[str] = None
+ event_source: Optional[str] = None
incremental_fields: list[IncrementalField] = field(default_factory=list)
@property
def is_aggregation(self) -> bool:
return self.aggregation_source is not None
+ @property
+ def is_events(self) -> bool:
+ return self.event_source is not None
+
+
+# Event rows are daily buckets keyed by `day`, the bucket's local midnight in epoch milliseconds.
+DAY_INCREMENTAL_FIELDS: list[IncrementalField] = [
+ {
+ "label": "day",
+ "type": IncrementalFieldType.Integer,
+ "field": "day",
+ "field_type": IncrementalFieldType.Integer,
+ }
+]
-# All endpoints are full refresh: Pendo's list endpoints expose no server-side timestamp
-# filter, and the aggregation endpoint's time filters only apply to event sources, not the
-# visitor/account metadata we pull here. A client-side cursor would still read every row each
-# run, so it would not be a genuine incremental sync.
+# Object and visitor/account endpoints are full refresh: Pendo's list endpoints expose no
+# server-side timestamp filter, and the aggregation endpoint's time filters only apply to event
+# sources. A client-side cursor would still read every row each run, so it would not be a
+# genuine incremental sync. Event sources do take a `timeSeries` range, so they sync incrementally.
PENDO_ENDPOINTS: dict[str, PendoEndpointConfig] = {
"features": PendoEndpointConfig(name="features", path="/api/v1/feature", primary_keys=["id"]),
"pages": PendoEndpointConfig(name="pages", path="/api/v1/page", primary_keys=["id"]),
"guides": PendoEndpointConfig(name="guides", path="/api/v1/guide", primary_keys=["id"]),
+ "track_types": PendoEndpointConfig(name="track_types", path="/api/v1/tracktype", primary_keys=["id"]),
"visitors": PendoEndpointConfig(name="visitors", aggregation_source="visitors", primary_keys=["visitorId"]),
"accounts": PendoEndpointConfig(name="accounts", aggregation_source="accounts", primary_keys=["accountId"]),
+ # Event rows have no id of their own, so `id` is a hash of the bucket's grouping fields.
+ "page_events": PendoEndpointConfig(
+ name="page_events", event_source="pageEvents", primary_keys=["id"], incremental_fields=DAY_INCREMENTAL_FIELDS
+ ),
+ "feature_events": PendoEndpointConfig(
+ name="feature_events",
+ event_source="featureEvents",
+ primary_keys=["id"],
+ incremental_fields=DAY_INCREMENTAL_FIELDS,
+ ),
+ "track_events": PendoEndpointConfig(
+ name="track_events", event_source="trackEvents", primary_keys=["id"], incremental_fields=DAY_INCREMENTAL_FIELDS
+ ),
}
ENDPOINTS = tuple(PENDO_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py
index 42b351f657d1..825f370e5748 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py
@@ -28,7 +28,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.pend
)
from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.settings import (
ENDPOINTS,
+ EVENTS_DEFAULT_LOOKBACK_DAYS,
INCREMENTAL_FIELDS,
+ PENDO_ENDPOINTS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -134,14 +136,20 @@ Create an integration key in Pendo under **Settings > Integrations > Integration
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh: Pendo exposes no server-side timestamp filter for
- # this metadata, so neither incremental nor append would avoid re-reading every row.
+ # Metadata endpoints are full refresh: Pendo exposes no server-side timestamp filter for
+ # them. Event endpoints sync incrementally by day, but each run re-reads the days Pendo may
+ # still reprocess, so append would duplicate their rows — merge only.
schemas = [
SourceSchema(
name=endpoint,
- supports_incremental=False,
+ supports_incremental=PENDO_ENDPOINTS[endpoint].is_events,
supports_append=False,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
+ description=(
+ f"Daily event buckets. Only syncs the last {EVENTS_DEFAULT_LOOKBACK_DAYS} days on initial sync"
+ if PENDO_ENDPOINTS[endpoint].is_events
+ else None
+ ),
)
for endpoint in ENDPOINTS
]
@@ -182,4 +190,7 @@ Create an integration key in Pendo under **Settings > Integrations > Integration
endpoint=inputs.schema_name,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value
+ if inputs.should_use_incremental_field
+ else None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py
index 936c512ebbe2..d9b06c90d65e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py
@@ -1,4 +1,9 @@
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from typing import Any
+
import pytest
+import time_machine
from unittest.mock import patch
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
@@ -110,13 +115,90 @@ class TestGetRowsAggregation:
assert {"filter": 'visitorId>"ab\\"cd"'} in pipeline
+DAY_MS = 24 * 60 * 60 * 1000
+
+
+def _ms(value: str) -> int:
+ return int(datetime.fromisoformat(value).replace(tzinfo=UTC).timestamp() * 1000)
+
+
+def _event_window_starts(result: Any) -> list[int]:
+ return [request.json()["request"]["pipeline"][0]["source"]["timeSeries"]["first"] for request in result.requests]
+
+
+class TestGetRowsEvents:
+ @pytest.fixture(autouse=True)
+ def _frozen_clock(self) -> Iterator[None]:
+ with time_machine.travel(datetime(2026, 3, 10, 6, 0, tzinfo=UTC), tick=False):
+ yield
+
+ def test_full_refresh_walks_the_lookback_up_to_the_last_started_anchor(self) -> None:
+ result = _driver().run("page_events", always(ScriptedResponse(json={"results": []})))
+
+ assert result.raised is None
+ windows = _event_window_starts(result)
+ assert windows[0] == _ms("2025-12-09T12:00:00")
+ # Today's noon-UTC anchor is still in the future at 06:00 UTC.
+ assert windows[-1] == _ms("2026-03-09T12:00:00")
+ assert all(later - earlier == DAY_MS for earlier, later in zip(windows, windows[1:]))
+
+ source = result.requests[0].json()["request"]["pipeline"][0]["source"]
+ assert source["pageEvents"] == {"appId": 'expandAppIds("*")'}
+ assert source["timeSeries"]["period"] == "dayRange"
+ assert source["timeSeries"]["count"] == 1
+
+ @pytest.mark.parametrize("utc_offset_hours", [-11, -5, 0, 10, 12, 14])
+ def test_incremental_rereads_the_watermark_day_and_the_reprocessing_week(self, utc_offset_hours: int) -> None:
+ watermark = _ms("2026-03-05T00:00:00") - utc_offset_hours * 60 * 60 * 1000
+
+ result = _driver().run(
+ "page_events",
+ always(ScriptedResponse(json={"results": []})),
+ incremental_field="day",
+ db_incremental_field_last_value=watermark,
+ )
+
+ assert result.raised is None
+ windows = _event_window_starts(result)
+ assert any(watermark <= anchor < watermark + DAY_MS for anchor in windows)
+ assert windows[0] <= watermark - 7 * DAY_MS + DAY_MS
+
+ @patch(f"{PENDO_PATH}.EVENT_BATCH_SIZE", 1)
+ def test_resumes_from_saved_day_and_keys_buckets_without_measures(self) -> None:
+ bucket = {"day": _ms("2026-03-09T05:00:00"), "trackTypeId": "t1", "visitorId": "v1", "tabId": "a"}
+ first_read = {**bucket, "numEvents": 1, "numMinutes": 1, "lastTime": 1}
+ re_read = {**bucket, "numEvents": 7, "numMinutes": 3, "lastTime": 2}
+ other_tab = {**bucket, "tabId": "b", "numEvents": 1, "numMinutes": 1, "lastTime": 1}
+
+ result = _driver().run(
+ "track_events",
+ [
+ ScriptedResponse(json={"results": [first_read, other_tab]}),
+ ScriptedResponse(json={"results": [re_read]}),
+ ],
+ resume_state=PendoResumeConfig(next_day="2026-03-08"),
+ )
+
+ assert result.raised is None
+ assert _event_window_starts(result) == [_ms("2026-03-08T12:00:00"), _ms("2026-03-09T12:00:00")]
+ # The first of the day's two batches leaves the cursor on that day.
+ assert result.committed_states == [
+ PendoResumeConfig(next_day="2026-03-08"),
+ PendoResumeConfig(next_day="2026-03-09"),
+ PendoResumeConfig(next_day="2026-03-10"),
+ ]
+ first_id, other_tab_id, re_read_id = (row["id"] for row in result.rows)
+ assert first_id == re_read_id
+ assert first_id != other_tab_id
+
+
class TestPendoSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = PENDO_ENDPOINTS[endpoint]
- payload: object = {"results": []} if config.is_aggregation else []
+ payload: object = [] if config.path else {"results": []}
- result = _driver().run(endpoint, [ScriptedResponse(json=payload)])
+ result = _driver().run(endpoint, always(ScriptedResponse(json=payload)))
assert result.raised is None
assert result.response is not None