:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add pendo event and track type tables (#124786)
Tom Owers8be1038e7dcc
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 8cc8a6bede2b..189ca9fbe50e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9270,10 +9270,10 @@ Today (5): `accounts`, `features`, `guides`, `pages`, `visitors`Diffed against: <https://engageapi.pendo.io/api/collections/16265887/Tzm6jvKG>- [ ] `aggregation source: events` — the raw usage event stream - Pendo's headline dataset and the basis of every engagement metric (high)-- [ ] `aggregation source: featureEvents` — feature click events joined to the features table already synced (high)-- [ ] `aggregation source: pageEvents` — page view events joined to the pages table already synced (high)-- [ ] `GET /api/v1/tracktype` — lookup resolving custom track event type IDs; also needed to interpret trackEvents (high)-- [ ] `aggregation source: trackEvents` — custom-instrumented events, the analog of product analytics custom events (high)+- [x] `aggregation source: featureEvents` — feature click events joined to the features table already synced (high)+- [x] `aggregation source: pageEvents` — page view events joined to the pages table already synced (high)+- [x] `GET /api/v1/tracktype` — lookup resolving custom track event type IDs; also needed to interpret trackEvents (high)+- [x] `aggregation source: trackEvents` — custom-instrumented events, the analog of product analytics custom events (high)- [ ] `aggregation source: guideEvents (plus guidesSeen)` — guide impressions and interactions for the guides table already synced (medium)- [ ] `aggregation source: pollEvents / pollsSeen` — poll and NPS responses, the source of Pendo's NPS reporting (medium)- [ ] `aggregation source: surveyResponses` — survey response rows for voice-of-customer analysis (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.pyindex 12fbfa184f7d..f1316e3452c2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/canonical_descriptions.py@@ -77,4 +77,81 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"metadata": "Custom and agent-collected metadata fields attached to the account.",},},+ "track_types": {+ "description": "A custom track event type, named by your track event calls and identified by a Pendo-generated ID.",+ "docs_url": "https://engageapi.pendo.io/#a55c1ce4-1e9c-4e64-a78e-fc824ade8edd",+ "columns": {+ "id": "Unique identifier for the track type.",+ "name": "The track type's display name.",+ "trackTypeName": "The event name sent in track event calls.",+ "appId": "ID of the application the track type belongs to.",+ "eventPropertyNameList": "Names of the event properties seen on this track type.",+ "isCoreEvent": "Whether the track type is marked as a core event.",+ "createdAt": "Time at which the track type was created, as a Unix timestamp in milliseconds.",+ "lastUpdatedAt": "Time at which the track type was last updated, as a Unix timestamp in milliseconds.",+ },+ },+ "page_events": {+ "description": "Daily count of page load events for tagged pages, grouped per visitor, account, page and browser context.",+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",+ "columns": {+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",+ "pageId": "The ID of the page these events are associated with.",+ "visitorId": "The visitor ID associated with the events.",+ "accountId": "The account ID associated with the events.",+ "appId": "ID of the application the events were collected in.",+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",+ "numEvents": "Number of page load events in this bucket.",+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",+ "server": "The hostname of the URL at the time of the events.",+ "remoteIp": "The IP address captured at the time of the events.",+ "userAgent": "The browser and operating system of the visitor.",+ "tabId": "Identifier of the browser window or tab the events came from.",+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",+ "properties": "Historical metadata and event properties captured with the events.",+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",+ },+ },+ "feature_events": {+ "description": "Daily count of click events for tagged features, grouped per visitor, account, feature and browser context.",+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",+ "columns": {+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",+ "featureId": "The ID of the feature these events are associated with.",+ "visitorId": "The visitor ID associated with the events.",+ "accountId": "The account ID associated with the events.",+ "appId": "ID of the application the events were collected in.",+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",+ "numEvents": "Number of click events in this bucket.",+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",+ "server": "The hostname of the URL at the time of the events.",+ "remoteIp": "The IP address captured at the time of the events.",+ "userAgent": "The browser and operating system of the visitor.",+ "tabId": "Identifier of the browser window or tab the events came from.",+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",+ "properties": "Historical metadata and event properties captured with the events.",+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",+ },+ },+ "track_events": {+ "description": "Daily count of custom track events, grouped per visitor, account, track type and browser context.",+ "docs_url": "https://engageapi.pendo.io/#5a695fb5-3da2-4a0e-9bbd-c7b276dbc4b8",+ "columns": {+ "id": "Surrogate key for the bucket, a hash of every field except the counts.",+ "trackTypeId": "The ID of the track type these events are associated with.",+ "visitorId": "The visitor ID associated with the events.",+ "accountId": "The account ID associated with the events.",+ "appId": "ID of the application the events were collected in.",+ "day": "Midnight of the bucket's day in the subscription's timezone, as a Unix timestamp in milliseconds.",+ "numEvents": "Number of track events in this bucket.",+ "numMinutes": "Number of unique minutes with activity across the events in this bucket.",+ "server": "The hostname of the URL at the time of the events.",+ "remoteIp": "The IP address captured at the time of the events.",+ "userAgent": "The browser and operating system of the visitor.",+ "tabId": "Identifier of the browser window or tab the events came from.",+ "analyticsSessionId": "Identifier of the web analytics session active when the events were logged.",+ "properties": "Historical metadata and event properties captured with the events.",+ "lastTime": "Time of the most recent event in this bucket, as a Unix timestamp in milliseconds.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.pyindex 980bd8e71411..f088c67f68c6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py@@ -1,11 +1,15 @@-import dataclasses+import json+import hashlibfrom collections.abc import Iterator+from datetime import UTC, date, datetime, time, timedeltafrom typing import Any, Optionalimport requestsfrom structlog.types import FilteringBoundLoggerfrom tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (make_tracked_session,retry_with_deadline,@@ -14,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.pendo.settings import (DEFAULT_REGION,+ EVENTS_DEFAULT_LOOKBACK_DAYS,PENDO_ENDPOINTS,PENDO_REGION_BASE_URLS,)@@ -26,20 +31,39 @@ AGGREGATION_PAGE_SIZE = 5000# unbounded list in memory and so resume state can advance per chunk.LIST_CHUNK_SIZE = 1000REQUEST_TIMEOUT_SECONDS = 60+# Pendo itself cuts an aggregation off after 5 minutes, and a busy day of events can take most of that.+EVENTS_REQUEST_TIMEOUT_SECONDS = 300MAX_RETRIES = 5+# Pendo buckets events by day in the subscription's timezone, which the API does not expose. Noon UTC+# falls in the same local day for every offset from -11h to +14h, so stepping a noon-UTC anchor one+# day at a time requests each local day exactly once.+EVENT_WINDOW_ANCHOR = timedelta(hours=12)+EVENT_ALL_APPS = 'expandAppIds("*")'+# Pendo's daily and weekly reprocessing can add backdated events up to 7 days after the fact, so+# an incremental run re-reads that many days before the watermark's day.+EVENTS_REPROCESSING_OVERLAP_DAYS = 7+# A busy day can hold a very large number of buckets; hand them to the pipeline in bounded batches.+EVENT_BATCH_SIZE = 5000+# Measures Pendo sums into each event bucket. The remaining fields identify the bucket, so they+# alone form its id and a re-read of a still-open day updates the row instead of duplicating it.+EVENT_MEASURE_FIELDS = frozenset(+ {"numEvents", "numMinutes", "rageClickCount", "errorClickCount", "deadClickCount", "uTurnCount", "lastTime"}+)class PendoRetryableError(Exception):pass+@frozenclass PendoResumeConfig:# Number of rows already yielded for a list endpoint schema; on resume we skip past them.offset: int = 0# Sort-field value of the last row yielded for an aggregation endpoint schema. Pendo's# aggregation pipeline has no skip/offset stage, so resuming re-filters for values past this.last_id: Optional[str] = None+ # ISO date of the next UTC day to request for an event endpoint schema.+ next_day: Optional[str] = Nonedef get_base_url(region: Optional[str]) -> str:@@ -65,9 +89,10 @@ def _request(method: str,url: str,logger: FilteringBoundLogger,+ timeout: float = REQUEST_TIMEOUT_SECONDS,**kwargs: Any,) -> requests.Response:- response = session.request(method, url, timeout=REQUEST_TIMEOUT_SECONDS, **kwargs)+ response = session.request(method, url, timeout=timeout, **kwargs)if response.status_code == 429 or response.status_code >= 500:raise PendoRetryableError(f"Pendo API error (retryable): status={response.status_code}, url={url}")@@ -173,12 +198,85 @@ def _iter_aggregation(break+def _to_epoch_ms(value: Any) -> Optional[int]:+ if isinstance(value, bool) or value is None:+ return None+ if isinstance(value, int | float):+ return int(value)+ if isinstance(value, str):+ try:+ return int(value)+ except ValueError:+ return None+ return None+++def _events_start_day(db_incremental_field_last_value: Any, today: date) -> date:+ watermark = _to_epoch_ms(db_incremental_field_last_value)+ if watermark is None:+ return today - timedelta(days=EVENTS_DEFAULT_LOOKBACK_DAYS)+ return datetime.fromtimestamp(watermark / 1000, tz=UTC).date() - timedelta(days=EVENTS_REPROCESSING_OVERLAP_DAYS)+++def _with_bucket_id(row: dict[str, Any]) -> dict[str, Any]:+ identity = {key: value for key, value in row.items() if key not in EVENT_MEASURE_FIELDS}+ digest = hashlib.sha256(json.dumps(identity, sort_keys=True, default=str).encode()).hexdigest()+ return {"id": digest, **row}+++def _iter_events(+ session: requests.Session,+ base_url: str,+ event_source: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PendoResumeConfig],+ start_day: date,+ end_day: date,+) -> Iterator[list[dict[str, Any]]]:+ url = f"{base_url}{AGGREGATION_PATH}"+ day = start_day++ while day <= end_day:+ anchor = datetime.combine(day, time(), tzinfo=UTC) + EVENT_WINDOW_ANCHOR+ body = {+ "response": {"mimeType": "application/json"},+ "request": {+ "requestId": f"posthog-{event_source}",+ "pipeline": [+ {+ "source": {+ event_source: {"appId": EVENT_ALL_APPS},+ "timeSeries": {"period": "dayRange", "first": int(anchor.timestamp() * 1000), "count": 1},+ }+ }+ ],+ },+ }+ response = _request(session, "POST", url, logger, timeout=EVENTS_REQUEST_TIMEOUT_SECONDS, json=body)+ data = response.json()+ del response+ rows: list[dict[str, Any]] = (data.get("results") or []) if isinstance(data, dict) else data++ day += timedelta(days=1)+ if not rows:+ resumable_source_manager.save_state(PendoResumeConfig(next_day=day.isoformat()))+ resumable_source_manager.safe_point()+ continue++ for start in range(0, len(rows), EVENT_BATCH_SIZE):+ # Only the day's last batch moves the cursor past it, so a resume re-reads a partly written day.+ if start + EVENT_BATCH_SIZE >= len(rows):+ resumable_source_manager.save_state(PendoResumeConfig(next_day=day.isoformat()))+ yield [_with_bucket_id(row) for row in rows[start : start + EVENT_BATCH_SIZE]]++def get_rows(integration_key: str,region: Optional[str],endpoint: str,logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[PendoResumeConfig],+ db_incremental_field_last_value: Any = None,) -> Iterator[list[dict[str, Any]]]:config = PENDO_ENDPOINTS[endpoint]base_url = get_base_url(region)@@ -187,7 +285,24 @@ def get_rows(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None- if config.is_aggregation:+ if config.event_source is not None:+ # The last day whose anchor is not in the future.+ end_day = (datetime.now(tz=UTC) - EVENT_WINDOW_ANCHOR).date()+ if resume and resume.next_day:+ start_day = date.fromisoformat(resume.next_day)+ logger.debug(f"Pendo: resuming endpoint={endpoint} from day={start_day}")+ else:+ start_day = _events_start_day(db_incremental_field_last_value, end_day)+ yield from _iter_events(+ session,+ base_url,+ config.event_source,+ logger,+ resumable_source_manager,+ start_day,+ end_day,+ )+ elif config.is_aggregation:if config.aggregation_source is None:raise ValueError(f"Endpoint '{endpoint}' is marked as aggregation but has no aggregation_source")start_after = resume.last_id if resume else None@@ -224,6 +339,7 @@ def pendo_source(endpoint: str,logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[PendoResumeConfig],+ db_incremental_field_last_value: Any = None,) -> SourceResponse:config = PENDO_ENDPOINTS[endpoint]@@ -235,6 +351,7 @@ def pendo_source(endpoint=endpoint,logger=logger,resumable_source_manager=resumable_source_manager,+ db_incremental_field_last_value=db_incremental_field_last_value,),primary_keys=config.primary_keys,# No source-level partitioning: Pendo timestamps are epoch-milliseconds, which thediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.pyindex 04a173320420..22f418f77f6c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/settings.py@@ -1,7 +1,9 @@-from dataclasses import dataclass, field+from dataclasses import fieldfrom typing import Optional-from products.warehouse_sources.backend.types import IncrementalField+from posthog.dataclasses import frozen++from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType# Pendo hosts each subscription in a region-specific data center and the integration key is# region-scoped, so the base URL has to match where the subscription lives.@@ -16,32 +18,65 @@ PENDO_REGION_BASE_URLS: dict[str, str] = {DEFAULT_REGION = "us"+# How many days of events a full refresh or a first incremental sync reads.+EVENTS_DEFAULT_LOOKBACK_DAYS = 90+-@dataclass+@frozenclass PendoEndpointConfig:name: strprimary_keys: list[str]- # GET list endpoints (feature/page/guide) set `path`; aggregation endpoints- # (visitors/accounts) set `aggregation_source` and POST to /api/v1/aggregation.+ # GET list endpoints (feature/page/guide/tracktype) set `path`; aggregation endpoints+ # (visitors/accounts) set `aggregation_source` and POST to /api/v1/aggregation. Event+ # endpoints set `event_source` and POST a day-windowed `timeSeries` to the same endpoint.path: Optional[str] = Noneaggregation_source: Optional[str] = None+ event_source: Optional[str] = Noneincremental_fields: list[IncrementalField] = field(default_factory=list)@propertydef is_aggregation(self) -> bool:return self.aggregation_source is not None+ @property+ def is_events(self) -> bool:+ return self.event_source is not None+++# Event rows are daily buckets keyed by `day`, the bucket's local midnight in epoch milliseconds.+DAY_INCREMENTAL_FIELDS: list[IncrementalField] = [+ {+ "label": "day",+ "type": IncrementalFieldType.Integer,+ "field": "day",+ "field_type": IncrementalFieldType.Integer,+ }+]-# All endpoints are full refresh: Pendo's list endpoints expose no server-side timestamp-# filter, and the aggregation endpoint's time filters only apply to event sources, not the-# visitor/account metadata we pull here. A client-side cursor would still read every row each-# run, so it would not be a genuine incremental sync.+# Object and visitor/account endpoints are full refresh: Pendo's list endpoints expose no+# server-side timestamp filter, and the aggregation endpoint's time filters only apply to event+# sources. A client-side cursor would still read every row each run, so it would not be a+# genuine incremental sync. Event sources do take a `timeSeries` range, so they sync incrementally.PENDO_ENDPOINTS: dict[str, PendoEndpointConfig] = {"features": PendoEndpointConfig(name="features", path="/api/v1/feature", primary_keys=["id"]),"pages": PendoEndpointConfig(name="pages", path="/api/v1/page", primary_keys=["id"]),"guides": PendoEndpointConfig(name="guides", path="/api/v1/guide", primary_keys=["id"]),+ "track_types": PendoEndpointConfig(name="track_types", path="/api/v1/tracktype", primary_keys=["id"]),"visitors": PendoEndpointConfig(name="visitors", aggregation_source="visitors", primary_keys=["visitorId"]),"accounts": PendoEndpointConfig(name="accounts", aggregation_source="accounts", primary_keys=["accountId"]),+ # Event rows have no id of their own, so `id` is a hash of the bucket's grouping fields.+ "page_events": PendoEndpointConfig(+ name="page_events", event_source="pageEvents", primary_keys=["id"], incremental_fields=DAY_INCREMENTAL_FIELDS+ ),+ "feature_events": PendoEndpointConfig(+ name="feature_events",+ event_source="featureEvents",+ primary_keys=["id"],+ incremental_fields=DAY_INCREMENTAL_FIELDS,+ ),+ "track_events": PendoEndpointConfig(+ name="track_events", event_source="trackEvents", primary_keys=["id"], incremental_fields=DAY_INCREMENTAL_FIELDS+ ),}ENDPOINTS = tuple(PENDO_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.pyindex 42b351f657d1..825f370e5748 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/source.py@@ -28,7 +28,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.pend)from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.settings import (ENDPOINTS,+ EVENTS_DEFAULT_LOOKBACK_DAYS,INCREMENTAL_FIELDS,+ PENDO_ENDPOINTS,)from products.warehouse_sources.backend.types import ExternalDataSourceType@@ -134,14 +136,20 @@ Create an integration key in Pendo under **Settings > Integrations > Integrationforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Every endpoint is full refresh: Pendo exposes no server-side timestamp filter for- # this metadata, so neither incremental nor append would avoid re-reading every row.+ # Metadata endpoints are full refresh: Pendo exposes no server-side timestamp filter for+ # them. Event endpoints sync incrementally by day, but each run re-reads the days Pendo may+ # still reprocess, so append would duplicate their rows — merge only.schemas = [SourceSchema(name=endpoint,- supports_incremental=False,+ supports_incremental=PENDO_ENDPOINTS[endpoint].is_events,supports_append=False,incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),+ description=(+ f"Daily event buckets. Only syncs the last {EVENTS_DEFAULT_LOOKBACK_DAYS} days on initial sync"+ if PENDO_ENDPOINTS[endpoint].is_events+ else None+ ),)for endpoint in ENDPOINTS]@@ -182,4 +190,7 @@ Create an integration key in Pendo under **Settings > Integrations > Integrationendpoint=inputs.schema_name,logger=inputs.logger,resumable_source_manager=resumable_source_manager,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.pyindex 936c512ebbe2..d9b06c90d65e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/tests/test_pendo.py@@ -1,4 +1,9 @@+from collections.abc import Iterator+from datetime import UTC, datetime+from typing import Any+import pytest+import time_machinefrom unittest.mock import patchfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (@@ -110,13 +115,90 @@ class TestGetRowsAggregation:assert {"filter": 'visitorId>"ab\\"cd"'} in pipeline+DAY_MS = 24 * 60 * 60 * 1000+++def _ms(value: str) -> int:+ return int(datetime.fromisoformat(value).replace(tzinfo=UTC).timestamp() * 1000)+++def _event_window_starts(result: Any) -> list[int]:+ return [request.json()["request"]["pipeline"][0]["source"]["timeSeries"]["first"] for request in result.requests]+++class TestGetRowsEvents:+ @pytest.fixture(autouse=True)+ def _frozen_clock(self) -> Iterator[None]:+ with time_machine.travel(datetime(2026, 3, 10, 6, 0, tzinfo=UTC), tick=False):+ yield++ def test_full_refresh_walks_the_lookback_up_to_the_last_started_anchor(self) -> None:+ result = _driver().run("page_events", always(ScriptedResponse(json={"results": []})))++ assert result.raised is None+ windows = _event_window_starts(result)+ assert windows[0] == _ms("2025-12-09T12:00:00")+ # Today's noon-UTC anchor is still in the future at 06:00 UTC.+ assert windows[-1] == _ms("2026-03-09T12:00:00")+ assert all(later - earlier == DAY_MS for earlier, later in zip(windows, windows[1:]))++ source = result.requests[0].json()["request"]["pipeline"][0]["source"]+ assert source["pageEvents"] == {"appId": 'expandAppIds("*")'}+ assert source["timeSeries"]["period"] == "dayRange"+ assert source["timeSeries"]["count"] == 1++ @pytest.mark.parametrize("utc_offset_hours", [-11, -5, 0, 10, 12, 14])+ def test_incremental_rereads_the_watermark_day_and_the_reprocessing_week(self, utc_offset_hours: int) -> None:+ watermark = _ms("2026-03-05T00:00:00") - utc_offset_hours * 60 * 60 * 1000++ result = _driver().run(+ "page_events",+ always(ScriptedResponse(json={"results": []})),+ incremental_field="day",+ db_incremental_field_last_value=watermark,+ )++ assert result.raised is None+ windows = _event_window_starts(result)+ assert any(watermark <= anchor < watermark + DAY_MS for anchor in windows)+ assert windows[0] <= watermark - 7 * DAY_MS + DAY_MS++ @patch(f"{PENDO_PATH}.EVENT_BATCH_SIZE", 1)+ def test_resumes_from_saved_day_and_keys_buckets_without_measures(self) -> None:+ bucket = {"day": _ms("2026-03-09T05:00:00"), "trackTypeId": "t1", "visitorId": "v1", "tabId": "a"}+ first_read = {**bucket, "numEvents": 1, "numMinutes": 1, "lastTime": 1}+ re_read = {**bucket, "numEvents": 7, "numMinutes": 3, "lastTime": 2}+ other_tab = {**bucket, "tabId": "b", "numEvents": 1, "numMinutes": 1, "lastTime": 1}++ result = _driver().run(+ "track_events",+ [+ ScriptedResponse(json={"results": [first_read, other_tab]}),+ ScriptedResponse(json={"results": [re_read]}),+ ],+ resume_state=PendoResumeConfig(next_day="2026-03-08"),+ )++ assert result.raised is None+ assert _event_window_starts(result) == [_ms("2026-03-08T12:00:00"), _ms("2026-03-09T12:00:00")]+ # The first of the day's two batches leaves the cursor on that day.+ assert result.committed_states == [+ PendoResumeConfig(next_day="2026-03-08"),+ PendoResumeConfig(next_day="2026-03-09"),+ PendoResumeConfig(next_day="2026-03-10"),+ ]+ first_id, other_tab_id, re_read_id = (row["id"] for row in result.rows)+ assert first_id == re_read_id+ assert first_id != other_tab_id++class TestPendoSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = PENDO_ENDPOINTS[endpoint]- payload: object = {"results": []} if config.is_aggregation else []+ payload: object = [] if config.path else {"results": []}- result = _driver().run(endpoint, [ScriptedResponse(json=payload)])+ result = _driver().run(endpoint, always(ScriptedResponse(json=payload)))assert result.raised is Noneassert result.response is not None