Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(sequenzy): add 4 missing tables to the sequenzy source (#130158)

Tom Owers288de7de95c0
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 76758658ac74..1347680ecda1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10850,14 +10850,14 @@ Note: SentinelOne publishes no public API reference. The console API doc (https:
## Sequenzy — **thin**
-Today (7): `campaigns`, `email_metrics`, `lists`, `segments`, `sequences`, `subscribers`, `tags`
+Today (11): `campaign_events`, `campaigns`, `email_metrics`, `email_sends`, `lists`, `segments`, `sequence_enrollments`, `sequence_events`, `sequences`, `subscribers`, `tags`
Diffed against: <https://docs.sequenzy.com/openapi.yaml>
-- [ ] `Campaign events (GET /api/v1/metrics/campaigns/{campaignId}/events)` — analyze recipient-level deliveries, opens, clicks, bounces, and unsubscribes instead of email totals alone (high)
-- [ ] `Sequence events (GET /api/v1/metrics/sequences/{sequenceId}/events)` — connect recipient engagement to individual automation steps and product behavior (high)
-- [ ] `Sequence enrollments (GET /api/v1/sequences/{sequenceId}/enrollments)` — measure automation participation, current steps, waiting contacts, and completion states (high)
-- [ ] `Sent emails (GET /api/v1/email-sends)` — join individual delivery outcomes and recipient records to campaigns, sequences, and transactional sends (high)
+- [x] `Campaign events (GET /api/v1/metrics/campaigns/{campaignId}/events)` — analyze recipient-level deliveries, opens, clicks, bounces, and unsubscribes instead of email totals alone (high) → `campaign_events` (incremental merge on `eventTime`, fan-out over `campaigns`)
+- [x] `Sequence events (GET /api/v1/metrics/sequences/{sequenceId}/events)` — connect recipient engagement to individual automation steps and product behavior (high) → `sequence_events` (incremental merge on `eventTime`, fan-out over `sequences`)
+- [x] `Sequence enrollments (GET /api/v1/sequences/{sequenceId}/enrollments)` — measure automation participation, current steps, waiting contacts, and completion states (high) → `sequence_enrollments` (full refresh, fan-out over `sequences`)
+- [x] `Sent emails (GET /api/v1/email-sends)` — join individual delivery outcomes and recipient records to campaigns, sequences, and transactional sends (high) → `email_sends` (merge on `createdAt`, re-reads the 14-day retained window)
- [ ] `Recipient engagement (GET /api/v1/metrics/recipients)` — compare opens, clicked URLs, and unsubscribe behavior across recipients and campaigns (medium)
- [ ] `Accounts and account members (GET /api/v1/accounts; GET /api/v1/accounts/{externalId}/members)` — group subscriber engagement by business account and member role (medium)
- [ ] `Account events (GET /api/v1/accounts/{externalId}/events)` — connect account activity to subscriber engagement and automation results (medium)
diff --git a/products/warehouse_sources/sources/sequenzy/canonical_descriptions.py b/products/warehouse_sources/sources/sequenzy/canonical_descriptions.py
index 93679d773673..e64e26ceaf03 100644
--- a/products/warehouse_sources/sources/sequenzy/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/sequenzy/canonical_descriptions.py
@@ -7,6 +7,27 @@ Keyed by the endpoint names in `settings.py` `ENDPOINTS`, which match the
from products.warehouse_sources.sources.sdk import CanonicalDescriptions
+_EMAIL_EVENT_COLUMNS: dict[str, str] = {
+ "id": "Unique identifier for the event.",
+ "emailSendId": "ID of the sent email the event belongs to.",
+ "subscriberId": "ID of the recipient subscriber.",
+ "email": "Recipient email address.",
+ "emailName": "Name of the email the event belongs to.",
+ "emailType": "Whether the email was a campaign, sequence, or transactional email.",
+ "abTestId": "A/B test the send belonged to, if any.",
+ "abTestVariantId": "A/B test variant the recipient received, if any.",
+ "eventType": "Event type: send, delivery, bounce, complaint, open, click, unsubscribe, delivery_delay, or transport_failure.",
+ "bounceType": "Bounce type for bounce events.",
+ "bounceSubType": "Bounce subtype for bounce events.",
+ "complaintType": "Complaint type for complaint events.",
+ "clickedUrl": "URL the recipient clicked, for click events.",
+ "countryCode": "Country of the recipient when the event was recorded.",
+ "metadata": "Event details from the delivery provider. Receiver replies are masked for bounced and blocked outcomes.",
+ "outcome": "Why the email did not arrive: bounced, blocked, delayed, or failed. Null for emails that arrived or have not finished.",
+ "reason": "One plain sentence about the outcome. Null when outcome is null.",
+ "eventTime": "When the event happened.",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"subscribers": {
"description": "An email contact in the workspace, with status, tags, and custom attributes.",
@@ -119,4 +140,69 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"revenueCents": "Attributed revenue in cents from purchase events.",
},
},
+ "email_sends": {
+ "description": "One email delivered to one recipient, with its delivery status and engagement timestamps. The API keeps 14 days of history.",
+ "docs_url": "https://docs.sequenzy.com/api-reference/email-sends/list",
+ "columns": {
+ "id": "Unique identifier for the sent email.",
+ "type": "Send source: campaign, transactional, or sequence.",
+ "emailType": "Delivery policy: marketing or transactional.",
+ "isTestEmail": "Whether the send was a test send.",
+ "campaignId": "ID of the campaign that sent the email, if any.",
+ "transactionalEmailId": "ID of the transactional email template, if any.",
+ "automationNodeId": "Node ID of the sequence email step that sent the email, if any.",
+ "subscriberId": "ID of the recipient subscriber.",
+ "recipientEmail": "Recipient email address.",
+ "subject": "Subject line of the email.",
+ "senderEmail": "Sender email address.",
+ "status": "Delivery status: pending, sent, delivered, opened, clicked, bounced, complained, failed, or suppressed.",
+ "outcome": "Why the email did not arrive: bounced, blocked, delayed, or failed. Null for emails that arrived or have not finished.",
+ "reason": "One plain sentence about the outcome. Null when outcome is null.",
+ "suppressionReason": "Why the send was suppressed: bounced, complaint, or unsubscribed.",
+ "sentAt": "When the email was sent.",
+ "deliveredAt": "When the email was delivered.",
+ "openedAt": "When the email was first opened.",
+ "clickedAt": "When a link in the email was first clicked.",
+ "bouncedAt": "When the email bounced.",
+ "createdAt": "When the send was created.",
+ },
+ },
+ "campaign_events": {
+ "description": "A raw email event (send, delivery, open, click, bounce, and so on) for one recipient of a campaign. The API returns up to 90 days of events.",
+ "docs_url": "https://docs.sequenzy.com/api-reference/analytics/campaign-events",
+ "columns": {
+ **_EMAIL_EVENT_COLUMNS,
+ "campaignId": "ID of the campaign the event belongs to.",
+ },
+ },
+ "sequence_events": {
+ "description": "A raw email event (send, delivery, open, click, bounce, and so on) for one recipient of a sequence email step. The API returns up to 90 days of events.",
+ "docs_url": "https://docs.sequenzy.com/api-reference/analytics/sequence-events",
+ "columns": {
+ **_EMAIL_EVENT_COLUMNS,
+ "campaignId": "Automation node ID of the sequence email step the event belongs to.",
+ "sequenceId": "ID of the sequence the event belongs to.",
+ },
+ },
+ "sequence_enrollments": {
+ "description": "One contact's enrollment in a sequence, with its status and the step it is on.",
+ "docs_url": "https://docs.sequenzy.com/api-reference/sequences/list-enrollments",
+ "columns": {
+ "enrollmentId": "Unique identifier for this one run of a contact through the sequence.",
+ "sequenceId": "ID of the sequence.",
+ "subscriberId": "ID of the enrolled subscriber.",
+ "email": "Subscriber email address, or the address captured at enrollment when the subscriber no longer exists.",
+ "subscriberStatus": "The subscriber's current status.",
+ "status": "Enrollment status: active, waiting, paused, completed, failed, or cancelled.",
+ "currentNodeId": "Sequence node the enrollment is currently on.",
+ "currentNodeType": "Type of the current node. Omitted when the node no longer exists in the sequence.",
+ "currentNodeLabel": "Label or email subject of the current node.",
+ "enrollmentStartedAt": "When the contact entered the sequence.",
+ "waitUntil": "When a waiting enrollment is scheduled to resume. Null when nothing is scheduled.",
+ "lastUpdatedAt": "Last change to the enrollment. For a waiting enrollment, when it arrived at its current node.",
+ "failedReason": "Why the enrollment stopped, for status failed.",
+ "pausedAt": "When the current pause began. Null when the enrollment is not paused.",
+ "enteredVia": "What put the contact into the sequence: a list, tag, segment, event, manual enrollment, and so on.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/sequenzy/sequenzy.py b/products/warehouse_sources/sources/sequenzy/sequenzy.py
index 9a38843dce0f..4a4f73d8c3c7 100644
--- a/products/warehouse_sources/sources/sequenzy/sequenzy.py
+++ b/products/warehouse_sources/sources/sequenzy/sequenzy.py
@@ -1,8 +1,9 @@
-import dataclasses
+from datetime import UTC, datetime
from typing import Any, Optional
from products.warehouse_sources.sources.sdk import (
BasePaginator,
+ ClientConfig,
EndpointResource,
JSONResponseCursorPaginator,
OffsetPaginator,
@@ -11,12 +12,14 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
+ build_dependent_resource,
frozen,
make_tracked_session,
rest_api_resource,
)
from products.warehouse_sources.sources.sequenzy.settings import (
ENDPOINTS_CONFIG,
+ EVENTS_MAX_WINDOW,
SEQUENZY_BASE_URL,
SequenzyEndpointConfig,
)
@@ -26,27 +29,56 @@ from products.warehouse_sources.sources.sequenzy.settings import (
class SequenzyResumeConfig:
"""Snapshot of the framework paginator's resume state.
- Exactly one field is set, matching the endpoint's pagination style: `cursor` for
- subscribers, `offset` for campaigns/sequences, `page` for email metrics. The field
- names mirror the keys of the framework paginators' `get_resume_state` dicts so the
- two shapes convert without a mapping table.
+ For a top-level endpoint exactly one of `cursor`, `offset`, `page` is set, matching its
+ pagination style. The names mirror the keys of the framework paginators'
+ `get_resume_state` dicts so the two shapes convert without a mapping table. A fan-out
+ endpoint sets `fanout_state` (the shared dependent-resource paginator's state) and
+ `window_end`, the end of the event time range, so a resumed run reads the same range.
"""
cursor: str | None = None
offset: int | None = None
page: int | None = None
+ fanout_state: dict[str, Any] | None = None
+ window_end: str | None = None
def _paginator_state(resume: SequenzyResumeConfig) -> Optional[dict[str, Any]]:
- state = {key: value for key, value in dataclasses.asdict(resume).items() if value is not None}
+ state = {key: getattr(resume, key) for key in ("cursor", "offset", "page") if getattr(resume, key) is not None}
return state or None
+def _to_utc(value: datetime | str) -> datetime:
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if isinstance(value, str) else value
+ return parsed.replace(tzinfo=UTC) if parsed.tzinfo is None else parsed.astimezone(UTC)
+
+
+def _format_timestamp(value: datetime) -> str:
+ return value.astimezone(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+
+
+def _time_filter_params(
+ config: SequenzyEndpointConfig,
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+ now: datetime,
+) -> dict[str, Any]:
+ if not config.event_time_range:
+ return {}
+
+ # Every request of a run, resumed attempts included, shares one `end`. A relative
+ # `period` would move with each request, so events for a parent read early could fall
+ # behind the watermark that later parents advance.
+ start = now - EVENTS_MAX_WINDOW
+ if should_use_incremental_field and db_incremental_field_last_value:
+ start = max(_to_utc(db_incremental_field_last_value), start)
+ return {"start": _format_timestamp(start), "end": _format_timestamp(now)}
+
+
def _make_paginator(config: SequenzyEndpointConfig) -> BasePaginator:
if config.pagination == "cursor":
return JSONResponseCursorPaginator(cursor_path="pagination.nextCursor", cursor_param="cursor")
if config.pagination == "offset":
- assert config.page_size is not None
return OffsetPaginator(
limit=config.page_size,
offset_param="offset",
@@ -58,15 +90,19 @@ def _make_paginator(config: SequenzyEndpointConfig) -> BasePaginator:
return SinglePagePaginator()
-def get_resource(name: str) -> EndpointResource:
- config = ENDPOINTS_CONFIG[name]
-
+def _request_params(config: SequenzyEndpointConfig, time_filter_params: dict[str, Any] | None = None) -> dict[str, Any]:
params: dict[str, Any] = dict(config.params)
# The cursor and page-number paginators only inject their own pagination param, so
# the page size rides along as a plain query param. The offset paginator sends
# `limit` itself.
- if config.pagination in ("cursor", "page") and config.page_size is not None:
+ if config.pagination in ("cursor", "page"):
params["limit"] = config.page_size
+ params.update(time_filter_params or {})
+ return params
+
+
+def get_resource(name: str, time_filter_params: dict[str, Any] | None = None) -> EndpointResource:
+ config = ENDPOINTS_CONFIG[name]
return {
"name": config.name,
@@ -75,7 +111,7 @@ def get_resource(name: str) -> EndpointResource:
"endpoint": {
"path": config.path,
"data_selector": config.data_selector,
- "params": params,
+ "params": _request_params(config, time_filter_params),
"paginator": _make_paginator(config),
},
"table_format": "delta",
@@ -89,6 +125,8 @@ def sequenzy_source(
job_id: str,
resumable_source_manager: ResumableSourceManager[SequenzyResumeConfig],
company_id: str | None = None,
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Any = None,
) -> SourceResponse:
endpoint_config = ENDPOINTS_CONFIG[endpoint]
@@ -98,56 +136,98 @@ def sequenzy_source(
# named per request; workspace keys (seq_live_) are already bound to one.
headers["x-company-id"] = company_id
- config: RESTAPIConfig = {
- "client": {
- "base_url": SEQUENZY_BASE_URL,
- "auth": {
- "type": "bearer",
- "token": api_key,
- },
- "headers": headers,
- },
- "resource_defaults": {
- "write_disposition": "replace",
+ client_config: ClientConfig = {
+ "base_url": SEQUENZY_BASE_URL,
+ "auth": {
+ "type": "bearer",
+ "token": api_key,
},
- "resources": [get_resource(endpoint)],
+ "headers": headers,
}
- initial_paginator_state: Optional[dict[str, Any]] = None
+ resume_config: SequenzyResumeConfig | None = None
if resumable_source_manager.can_resume():
resume_config = resumable_source_manager.load_state()
- if resume_config is not None:
- initial_paginator_state = _paginator_state(resume_config)
-
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- if not state:
- return
- resumable_source_manager.save_state(
- SequenzyResumeConfig(
- cursor=state.get("cursor"),
- offset=state.get("offset"),
- page=state.get("page"),
- )
- )
- resource = rest_api_resource(
- config,
- team_id,
- job_id,
- None,
- resume_hook=save_checkpoint,
- initial_paginator_state=initial_paginator_state,
+ now = datetime.now(UTC)
+ if resume_config is not None and resume_config.window_end:
+ now = _to_utc(resume_config.window_end)
+ time_filter_params = _time_filter_params(
+ endpoint_config, should_use_incremental_field, db_incremental_field_last_value, now
)
+ resource: Any
+ if endpoint_config.fanout is not None:
+ parent_config = ENDPOINTS_CONFIG[endpoint_config.fanout.parent_name]
+ window_end = _format_timestamp(now)
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state:
+ resumable_source_manager.save_state(SequenzyResumeConfig(fanout_state=state, window_end=window_end))
+
+ resource = build_dependent_resource(
+ endpoint_configs=ENDPOINTS_CONFIG,
+ child_endpoint=endpoint,
+ fanout=endpoint_config.fanout,
+ client_config=client_config,
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ should_use_incremental_field=should_use_incremental_field,
+ incremental_config_factory=lambda _field: None,
+ page_size_param=None,
+ parent_endpoint_extra={
+ "paginator": _make_paginator(parent_config),
+ "data_selector": parent_config.data_selector,
+ },
+ child_endpoint_extra={
+ "paginator": _make_paginator(endpoint_config),
+ "data_selector": endpoint_config.data_selector,
+ },
+ child_params_extra=_request_params(endpoint_config, time_filter_params),
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=resume_config.fanout_state if resume_config else None,
+ resume_parent_pagination=True,
+ )
+ else:
+ config: RESTAPIConfig = {
+ "client": client_config,
+ "resource_defaults": {
+ "write_disposition": "replace",
+ },
+ "resources": [get_resource(endpoint, time_filter_params)],
+ }
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if not state or not endpoint_config.resume_pages:
+ return
+ resumable_source_manager.save_state(
+ SequenzyResumeConfig(
+ cursor=state.get("cursor"),
+ offset=state.get("offset"),
+ page=state.get("page"),
+ )
+ )
+
+ resource = rest_api_resource(
+ config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=(
+ _paginator_state(resume_config) if resume_config and endpoint_config.resume_pages else None
+ ),
+ )
+
return SourceResponse(
name=endpoint,
items=lambda: resource,
primary_keys=list(endpoint_config.primary_keys),
partition_mode="datetime" if endpoint_config.partition_key else None,
partition_keys=[endpoint_config.partition_key] if endpoint_config.partition_key else None,
- # The paginated collections come back newest-first with no ascending option, and
- # the single-page collections document no order, so never claim "asc".
- sort_mode="desc",
+ sort_mode=endpoint_config.sort_mode,
)
diff --git a/products/warehouse_sources/sources/sequenzy/settings.py b/products/warehouse_sources/sources/sequenzy/settings.py
index fac203ef31a2..5ec96d8f2441 100644
--- a/products/warehouse_sources/sources/sequenzy/settings.py
+++ b/products/warehouse_sources/sources/sequenzy/settings.py
@@ -1,10 +1,24 @@
from dataclasses import field
+from datetime import timedelta
from typing import Literal
-from products.warehouse_sources.sources.sdk import IncrementalField, frozen
+from products.warehouse_sources.sources.sdk import (
+ DependentEndpointConfig,
+ IncrementalField,
+ IncrementalFieldType,
+ frozen,
+)
SEQUENZY_BASE_URL = "https://api.sequenzy.com/api/v1"
+EMAIL_SENDS_RETENTION_DAYS = 14
+
+# The event endpoints reject a `start`/`end` range longer than 90 days. Stay just inside it.
+EVENTS_MAX_WINDOW = timedelta(days=89, hours=23)
+
+# Both event endpoints return delivery events only unless asked for every type.
+ALL_EMAIL_EVENT_TYPES = "send,delivery,bounce,complaint,open,click,unsubscribe,delivery_delay,transport_failure"
+
@frozen
class SequenzyEndpointConfig:
@@ -15,15 +29,31 @@ class SequenzyEndpointConfig:
pagination: Literal["cursor", "offset", "page", "single_page"]
primary_keys: list[str] = field(default_factory=lambda: ["id"])
partition_key: str | None = None
- page_size: int | None = None
+ page_size: int = 100
params: dict[str, str | int] = field(default_factory=dict)
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: str | None = None
+ event_time_range: bool = False
+ resume_pages: bool = True
+ fanout: DependentEndpointConfig | None = None
+ sort_mode: Literal["asc", "desc"] | None = "desc"
+
+
+def _datetime_field(name: str) -> IncrementalField:
+ return IncrementalField(
+ label=name,
+ type=IncrementalFieldType.DateTime,
+ field=name,
+ field_type=IncrementalFieldType.DateTime,
+ )
# Sequenzy paginates each collection differently: subscribers use an opaque cursor
-# (`pagination.nextCursor`), campaigns and sequences use limit/offset, per-email metrics use
-# page numbers, and the small definition collections (tags, lists, segments) return the whole
-# collection in one response. Subscribers, campaigns, and sequences are returned newest-first
-# with no way to request ascending order, hence sort_mode="desc" in sequenzy.py.
+# (`pagination.nextCursor`), campaigns, sequences, and enrollments use limit/offset, per-email
+# metrics, email sends, and events use page numbers, and the small definition collections
+# (tags, lists, segments) return the whole collection in one response. Subscribers, campaigns,
+# and sequences are returned newest-first with no way to request ascending order, hence the
+# default sort_mode="desc".
ENDPOINTS_CONFIG: dict[str, SequenzyEndpointConfig] = {
"subscribers": SequenzyEndpointConfig(
name="subscribers",
@@ -85,11 +115,95 @@ ENDPOINTS_CONFIG: dict[str, SequenzyEndpointConfig] = {
# two ID namespaces.
primary_keys=["emailType", "emailId"],
),
+ "email_sends": SequenzyEndpointConfig(
+ name="email_sends",
+ path="/email-sends",
+ data_selector="emailSends",
+ pagination="page",
+ page_size=100,
+ # A send's status changes after `createdAt`, so every run reads the whole retained
+ # window and the merge updates older rows.
+ params={"days": EMAIL_SENDS_RETENTION_DAYS, "sortField": "createdAt", "sortOrder": "asc"},
+ partition_key="createdAt",
+ incremental_fields=[_datetime_field("createdAt")],
+ default_incremental_field="createdAt",
+ # The `days` window counts back from the server's clock, so a page number saved by one
+ # attempt points at other rows on the next.
+ resume_pages=False,
+ sort_mode=None,
+ ),
+ "campaign_events": SequenzyEndpointConfig(
+ name="campaign_events",
+ path="/metrics/campaigns/{campaignId}/events",
+ data_selector="events",
+ pagination="page",
+ page_size=500,
+ params={"eventTypes": ALL_EMAIL_EVENT_TYPES},
+ primary_keys=["campaignId", "id"],
+ partition_key="eventTime",
+ incremental_fields=[_datetime_field("eventTime")],
+ default_incremental_field="eventTime",
+ event_time_range=True,
+ fanout=DependentEndpointConfig(
+ parent_name="campaigns",
+ resolve_param="campaignId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "campaignId"},
+ ),
+ sort_mode=None,
+ ),
+ "sequence_events": SequenzyEndpointConfig(
+ name="sequence_events",
+ path="/metrics/sequences/{sequenceId}/events",
+ data_selector="events",
+ pagination="page",
+ page_size=500,
+ params={"eventTypes": ALL_EMAIL_EVENT_TYPES},
+ # An event's `campaignId` holds the sequence email step here, so the parent sequence
+ # rides along as `sequenceId`.
+ primary_keys=["sequenceId", "id"],
+ partition_key="eventTime",
+ incremental_fields=[_datetime_field("eventTime")],
+ default_incremental_field="eventTime",
+ event_time_range=True,
+ fanout=DependentEndpointConfig(
+ parent_name="sequences",
+ resolve_param="sequenceId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "sequenceId"},
+ ),
+ sort_mode=None,
+ ),
+ "sequence_enrollments": SequenzyEndpointConfig(
+ name="sequence_enrollments",
+ path="/sequences/{sequenceId}/enrollments",
+ data_selector="enrollments",
+ pagination="offset",
+ page_size=500,
+ # The default status filter returns only contacts still in the sequence.
+ params={"status": "active,waiting,paused,completed,failed,cancelled", "sort": "enrolled_at_asc"},
+ primary_keys=["sequenceId", "enrollmentId"],
+ partition_key="enrollmentStartedAt",
+ fanout=DependentEndpointConfig(
+ parent_name="sequences",
+ resolve_param="sequenceId",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "sequenceId"},
+ ),
+ sort_mode=None,
+ ),
}
ENDPOINTS = tuple(ENDPOINTS_CONFIG.keys())
-# The API exposes no server-side created/updated timestamp filter on any list endpoint
-# (subscribers only filter on `unsubscribedAt`, which covers opted-out contacts alone),
-# so every endpoint is full-refresh only.
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+# The API expires email sends and events, so these tables merge to keep older rows. The other
+# list endpoints have no created/updated filter, so they are full-refresh only.
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ name: config.incremental_fields for name, config in ENDPOINTS_CONFIG.items() if config.incremental_fields
+}
+
+# A re-read window overlaps rows that were already synced, so append would duplicate them.
+MERGE_ONLY_ENDPOINTS = tuple(INCREMENTAL_FIELDS.keys())
diff --git a/products/warehouse_sources/sources/sequenzy/source.py b/products/warehouse_sources/sources/sequenzy/source.py
index 26947630de0a..6420a8c9ecfb 100644
--- a/products/warehouse_sources/sources/sequenzy/source.py
+++ b/products/warehouse_sources/sources/sequenzy/source.py
@@ -24,7 +24,7 @@ from products.warehouse_sources.sources.sequenzy.sequenzy import (
sequenzy_source,
validate_credentials as validate_sequenzy_credentials,
)
-from products.warehouse_sources.sources.sequenzy.settings import ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.sources.sequenzy.settings import ENDPOINTS, INCREMENTAL_FIELDS, MERGE_ONLY_ENDPOINTS
@SourceRegistry.register
@@ -58,7 +58,7 @@ class SequenzySource(ResumableSource[SequenzySourceConfig, SequenzyResumeConfig]
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=MERGE_ONLY_ENDPOINTS)
def validate_credentials(
self,
@@ -85,6 +85,8 @@ class SequenzySource(ResumableSource[SequenzySourceConfig, SequenzyResumeConfig]
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
company_id=config.company_id,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
@property
diff --git a/products/warehouse_sources/sources/sequenzy/tests/test_sequenzy.py b/products/warehouse_sources/sources/sequenzy/tests/test_sequenzy.py
index da9eb03dc6ae..763c948b9884 100644
--- a/products/warehouse_sources/sources/sequenzy/tests/test_sequenzy.py
+++ b/products/warehouse_sources/sources/sequenzy/tests/test_sequenzy.py
@@ -1,12 +1,16 @@
+import datetime as dt
+from collections.abc import Iterator
from typing import Any
import pytest
+import time_machine
from products.warehouse_sources.sources.sdk.testing import (
DriveResult,
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.sources.sequenzy._config import SequenzySourceConfig
@@ -18,14 +22,36 @@ def _make_http_response(body: dict[str, Any], status_code: int = 200) -> Scripte
return ScriptedResponse(status=status_code, json=body)
+NOW = dt.datetime(2026, 6, 15, 12, 0, tzinfo=dt.UTC)
+
+
[email protected](autouse=True)
+def frozen_now() -> Iterator[None]:
+ with time_machine.travel(NOW, tick=False):
+ yield
+
+
def _drive(
endpoint: str,
- responses: list[ScriptedResponse],
+ responses: Any,
resume: SequenzyResumeConfig | None = None,
company_id: str | None = None,
+ **inputs: Any,
) -> DriveResult:
driver = SourceDriver(SequenzySource(), SequenzySourceConfig(api_key="test-key", company_id=company_id))
- return driver.run(endpoint, responses, resume_state=resume)
+ return driver.run(endpoint, responses, resume_state=resume, **inputs)
+
+
+def _incremental(last_value: dt.datetime) -> dict[str, Any]:
+ return {"should_use_incremental_field": True, "db_incremental_field_last_value": last_value}
+
+
+def _offset_page(selector: str, rows: list[dict[str, Any]], total: int) -> ScriptedResponse:
+ return _make_http_response({"success": True, selector: rows, "pagination": {"total": total}})
+
+
+def _numbered_page(selector: str, rows: list[dict[str, Any]], total_pages: int) -> ScriptedResponse:
+ return _make_http_response({"success": True, selector: rows, "pagination": {"totalPages": total_pages}})
class TestSequenzyPagination:
@@ -55,6 +81,130 @@ class TestSequenzyPagination:
assert len(result.rows) == 150
+class TestSequenzyFanOut:
+ @pytest.mark.parametrize(
+ ("endpoint", "parent_path", "parent_selector", "child_prefix", "parent_key"),
+ [
+ ("campaign_events", "/campaigns", "campaigns", "/metrics/campaigns", "campaignId"),
+ ("sequence_events", "/sequences", "sequences", "/metrics/sequences", "sequenceId"),
+ ],
+ )
+ def test_events_walk_every_parent_and_page_with_all_event_types(
+ self, endpoint: str, parent_path: str, parent_selector: str, child_prefix: str, parent_key: str
+ ) -> None:
+ script = route(
+ {
+ parent_path: [_offset_page(parent_selector, [{"id": "p1"}, {"id": "p2"}], total=2)],
+ f"{child_prefix}/p1/events": [
+ _numbered_page("events", [{"id": "e1", "campaignId": "node_1"}], total_pages=2),
+ _numbered_page("events", [{"id": "e2", "campaignId": "node_1"}], total_pages=2),
+ ],
+ f"{child_prefix}/p2/events": [_numbered_page("events", [], total_pages=0)],
+ }
+ )
+ result = _drive(endpoint, script)
+
+ assert result.raised is None
+ child_requests = [r for r in result.requests if r.path.endswith("/events")]
+ assert [(r.path.split("/")[-2], r.param("page")) for r in child_requests] == [
+ ("p1", "1"),
+ ("p1", "2"),
+ ("p2", "1"),
+ ]
+ assert {r.param("eventTypes") for r in child_requests} == {
+ "send,delivery,bounce,complaint,open,click,unsubscribe,delivery_delay,transport_failure"
+ }
+ assert {(r.param("start"), r.param("end"), r.param("period")) for r in child_requests} == {
+ ("2026-03-17T13:00:00.000Z", "2026-06-15T12:00:00.000Z", None)
+ }
+ assert {r.param("limit") for r in child_requests} == {"500"}
+ assert [(row["id"], row[parent_key]) for row in result.rows] == [("e1", "p1"), ("e2", "p1")]
+
+ def test_enrollments_page_by_offset_with_every_status(self) -> None:
+ script = route(
+ {
+ "/sequences": [_offset_page("sequences", [{"id": "seq_1"}], total=1)],
+ "/sequences/seq_1/enrollments": [
+ _offset_page("enrollments", [{"enrollmentId": f"tok_{i}"} for i in range(500)], total=501),
+ _offset_page("enrollments", [{"enrollmentId": "tok_500"}], total=501),
+ ],
+ }
+ )
+ result = _drive("sequence_enrollments", script)
+
+ assert result.raised is None
+ child_requests = [r for r in result.requests if r.path.endswith("/enrollments")]
+ assert [(r.param("offset"), r.param("limit")) for r in child_requests] == [("0", "500"), ("500", "500")]
+ assert {r.param("status") for r in child_requests} == {"active,waiting,paused,completed,failed,cancelled"}
+ assert {r.param("sort") for r in child_requests} == {"enrolled_at_asc"}
+ assert len(result.rows) == 501
+ assert {row["sequenceId"] for row in result.rows} == {"seq_1"}
+
+ def test_resumed_events_run_keeps_the_original_time_range(self) -> None:
+ last_value = NOW - dt.timedelta(days=1)
+
+ def script() -> Any:
+ return route(
+ {
+ "/campaigns": [_offset_page("campaigns", [{"id": "c1"}], total=1)] * 2,
+ "/metrics/campaigns/c1/events": [
+ _numbered_page("events", [{"id": "e1"}], total_pages=2),
+ _numbered_page("events", [{"id": "e2"}], total_pages=2),
+ ],
+ }
+ )
+
+ first = _drive("campaign_events", script(), **_incremental(last_value))
+ checkpoint = first.committed_states[0]
+
+ with time_machine.travel(NOW + dt.timedelta(hours=3), tick=False):
+ resumed = _drive("campaign_events", script(), resume=checkpoint, **_incremental(last_value))
+
+ resumed_child = [r for r in resumed.requests if r.path.endswith("/events")]
+ assert resumed_child[0].param("page") == "2"
+ assert resumed_child[0].param("end") == "2026-06-15T12:00:00.000Z"
+
+
+class TestSequenzyTimeFilter:
+ @pytest.mark.parametrize(
+ "inputs",
+ [
+ {},
+ _incremental(NOW - dt.timedelta(minutes=5)),
+ ],
+ )
+ def test_email_sends_read_the_whole_retained_window(self, inputs: dict[str, Any]) -> None:
+ result = _drive("email_sends", [_numbered_page("emailSends", [{"id": "s1"}], total_pages=1)], **inputs)
+
+ request = result.requests[0]
+ assert request.param("days") == "14"
+ assert (request.param("sortField"), request.param("sortOrder")) == ("createdAt", "asc")
+
+ @pytest.mark.parametrize(
+ ("last_value", "expected_start"),
+ [
+ (NOW - dt.timedelta(days=1), "2026-06-14T12:00:00.000Z"),
+ ("2026-06-14T12:00:00Z", "2026-06-14T12:00:00.000Z"),
+ (NOW - dt.timedelta(days=200), "2026-03-17T13:00:00.000Z"),
+ ],
+ )
+ def test_incremental_events_send_a_bounded_range(self, last_value: Any, expected_start: str) -> None:
+ script = route(
+ {
+ "/campaigns": [_offset_page("campaigns", [{"id": "c1"}], total=1)],
+ "/metrics/campaigns/c1/events": [_numbered_page("events", [], total_pages=0)],
+ }
+ )
+ result = _drive("campaign_events", script, **_incremental(last_value))
+
+ child = next(r for r in result.requests if r.path.endswith("/events"))
+ assert (child.param("start"), child.param("end"), child.param("period")) == (
+ expected_start,
+ "2026-06-15T12:00:00.000Z",
+ None,
+ )
+
+
class TestSequenzyResume:
@pytest.mark.parametrize(
("endpoint", "resume", "param", "expected"),
@@ -62,12 +212,18 @@ class TestSequenzyResume:
("subscribers", SequenzyResumeConfig(cursor="cur_saved"), "cursor", "cur_saved"),
("campaigns", SequenzyResumeConfig(offset=100), "offset", "100"),
("email_metrics", SequenzyResumeConfig(page=2), "page", "2"),
+ ("email_sends", SequenzyResumeConfig(page=3), "page", "1"),
],
)
def test_resume_seeds_first_request(
self, endpoint: str, resume: SequenzyResumeConfig, param: str, expected: Any
) -> None:
- selector = {"subscribers": "subscribers", "campaigns": "campaigns", "email_metrics": "emails"}[endpoint]
+ selector = {
+ "subscribers": "subscribers",
+ "campaigns": "campaigns",
+ "email_metrics": "emails",
+ "email_sends": "emailSends",
+ }[endpoint]
responses = [
_make_http_response({"success": True, selector: [], "pagination": {"nextCursor": None}}),
]