Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(mailchimp): add incremental email activity and open event tables (#123945)

Chris Shine2ce80f69ac0a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS.md
index dbfcd6b6aeda..761b12fbc772 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS.md
@@ -640,9 +640,10 @@ Thirty-four tables.
### Mailchimp — spec-verified
Diffed against Mailchimp's published Swagger schema.
-Thirty-two tables. The per-recipient engagement layer this section flagged as missing has shipped.
+Thirty-four tables. The per-recipient engagement layer this section flagged as missing has shipped.
- [x] `reports/*/email-activity` — per-member opens and clicks. The single biggest gap.
+- [x] Incremental `reports/*/email-activity` and `reports/*/open-details`, one row per event (`report_email_activity_events`, `report_open_events`).
- [x] `reports/*/open-details`, `reports/*/click-details`, `reports/*/sent-to`, `reports/*/unsubscribed`, `reports/*/abuse-reports`.
- [x] `lists/*/segments` — segment definitions and membership.
- [x] `automations` and `automations/*/emails` — automated journeys, entirely absent.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/canonical_descriptions.py
index 7328d328cbdc..f0f36ea89dfc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/canonical_descriptions.py
@@ -382,6 +382,35 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"opens": "An array of timestamps for each time the member opened the campaign.",
},
},
+ "report_email_activity_events": {
+ "description": "Opens, clicks, and bounces on sent campaigns, one row per event. Mailchimp returns at most 1,000 opens and 1,000 clicks for each recipient of a campaign, so this table can miss events for very active recipients. Each incremental sync re-reads the last day of events, so an event that Mailchimp reports more than a day late can be missing. To catch late events, set a lookback window in the table's sync settings. Each sync re-reads that window on top of the one day. To recover events already missed, run a full resync.",
+ "docs_url": "https://mailchimp.com/developer/marketing/api/email-activity-reports/",
+ "columns": {
+ "campaign_id": "The unique id for the campaign.",
+ "list_id": "The unique id for the audience the campaign was sent to.",
+ "email_id": "The MD5 hash of the recipient's lowercase email address.",
+ "email_address": "Email address for the recipient.",
+ "action": "The interaction with the email: open, click, or bounce.",
+ "type": "For a bounce, the type of bounce received: hard or soft.",
+ "timestamp": "The date and time of the event in ISO 8601 format.",
+ "url": "For a click, the URL the recipient clicked.",
+ "ip": "The IP address recorded for the event.",
+ "event_id": "A unique id for the event, built by PostHog from the campaign, the recipient, and the event details. Mailchimp doesn't give these events an id.",
+ },
+ },
+ "report_open_events": {
+ "description": "Opens of sent campaigns, one row per open. To count a recipient's opens, count their rows. To leave out proxy opens, keep only rows where is_proxy_open is false. Each incremental sync re-reads the last day of opens, so an open that Mailchimp reports more than a day late can be missing. To catch late opens, set a lookback window in the table's sync settings. Each sync re-reads that window on top of the one day. To recover opens already missed, run a full resync.",
+ "docs_url": "https://mailchimp.com/developer/marketing/api/open-reports/",
+ "columns": {
+ "campaign_id": "The unique id for the campaign.",
+ "list_id": "The unique id for the audience the campaign was sent to.",
+ "email_id": "The MD5 hash of the recipient's lowercase email address.",
+ "email_address": "Email address for the recipient.",
+ "timestamp": "The date and time of the open in ISO 8601 format.",
+ "is_proxy_open": "Whether the open came from an email client that uses proxies.",
+ "event_id": "A unique id for the open, built by PostHog from the campaign, the recipient, and the open details. Mailchimp doesn't give opens an id.",
+ },
+ },
"report_click_details": {
"description": "Per-link click performance for a sent campaign, one row per tracked URL.",
"docs_url": "https://mailchimp.com/developer/marketing/api/click-reports/",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/mailchimp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/mailchimp.py
index f0699655ee49..c51dac74f325 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/mailchimp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/mailchimp.py
@@ -1,12 +1,16 @@
+import json
+import hashlib
import dataclasses
+from collections import Counter
from collections.abc import Iterator
-from datetime import date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from requests import Request, Response, Session
from requests.exceptions import HTTPError, RequestException
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
@@ -20,12 +24,19 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.
MAILCHIMP_ENDPOINTS,
MAX_PAGE_SIZE,
MailchimpEndpointConfig,
+ MailchimpEventsConfig,
MailchimpPagination,
MailchimpParentConfig,
)
+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType
REQUEST_TIMEOUT_SECONDS = 120
MAX_RETRY_ATTEMPTS = 5
+EVENT_CUTOFF_MARGIN = timedelta(minutes=10)
+# Mailchimp does not say how soon an event shows up in a report, so each run re-reads this much.
+EVENT_LOOKBACK = timedelta(days=1)
+EVENT_MEMBER_FIELDS = ("list_id", "email_id", "email_address")
+EVENT_TABLE_APPEND_ERROR = "Mailchimp event tables do not support append sync"
class MailchimpRetryableError(Exception):
@@ -59,7 +70,7 @@ def _get_with_retry(
return response
[email protected](frozen=False)
class MailchimpResumeConfig:
"""Resume state for Mailchimp endpoints.
@@ -79,6 +90,7 @@ class MailchimpResumeConfig:
offset: int
list_id: Optional[str] = None
parent_ids: Optional[list[str]] = None
+ events_until: Optional[str] = None
def extract_data_center(api_key: str) -> str:
@@ -408,7 +420,8 @@ def _iter_pages(
yield offset, records
- if pagination != "offset" or not records:
+ # Only `total_items` ends the walk, because a `since_*` filter can empty a page in the middle.
+ if pagination != "offset":
return
total_items = data.get("total_items", 0)
@@ -452,6 +465,55 @@ def _resolve_parent_id_maps(
return id_maps
+def _event_cutoff(
+ config: MailchimpEndpointConfig,
+ should_use_incremental_field: bool,
+ resume_config: Optional[MailchimpResumeConfig],
+) -> Optional[datetime]:
+ """The newest event time an incremental run keeps.
+
+ A fan-out reads one campaign after another, so an event can land on a page the run already read.
+ A cursor past that event would skip it for good, so the run keeps only events from before it
+ started, less a margin for clock skew with Mailchimp, and the next run reads the rest. A resumed
+ attempt keeps the first attempt's cutoff, because the pages read before the hand-off used it.
+ """
+ if config.events is None or not should_use_incremental_field:
+ return None
+ if resume_config is not None and resume_config.events_until is not None:
+ return datetime.fromisoformat(resume_config.events_until)
+ return datetime.now(UTC) - EVENT_CUTOFF_MARGIN
+
+
+def _explode_records(
+ records: list[dict[str, Any]],
+ events: MailchimpEventsConfig,
+ id_map: dict[str, str],
+ events_until: Optional[datetime],
+) -> Iterator[dict[str, Any]]:
+ """One row per event in each record's events array, carrying the recipient's stable fields."""
+ for record in records:
+ member = {key: record[key] for key in EVENT_MEMBER_FIELDS if key in record}
+ occurrences: Counter[str] = Counter()
+ for event in record.get(events.array_field) or []:
+ moment = parse_datetime_value(event.get("timestamp"))
+ # Mailchimp gives events no id and opens no `url`, so the id hashes fixed fields plus an
+ # occurrence count, which stays stable because identical events share a timestamp.
+ identity = json.dumps(
+ {
+ **id_map,
+ "email_id": record.get("email_id"),
+ **{key: event.get(key) for key in events.key_fields},
+ "timestamp": moment.isoformat() if moment else event.get("timestamp"),
+ },
+ sort_keys=True,
+ )
+ occurrences[identity] += 1
+ if events_until is not None and moment is not None and moment > events_until:
+ continue
+ event_id = hashlib.sha256(f"{identity}#{occurrences[identity]}".encode()).hexdigest()
+ yield {**member, **event, "event_id": event_id}
+
+
def _fetch_endpoint_rows(
session: Session,
base_url: str,
@@ -462,6 +524,7 @@ def _fetch_endpoint_rows(
start_offset: int,
resumable_source_manager: ResumableSourceManager[MailchimpResumeConfig],
parent_ids: Optional[list[str]],
+ events_until: Optional[datetime],
) -> Iterator[dict[str, Any]]:
if config.single_object:
response = _get_with_retry(
@@ -482,16 +545,24 @@ def _fetch_endpoint_rows(
extra_params,
start_offset,
):
- if not records:
- continue
-
if config.pagination == "offset":
# Checkpoint the page we are about to yield, matching the contacts path: a resume
# re-fetches it and the primary key dedupes the overlap.
- resumable_source_manager.save_state(MailchimpResumeConfig(offset=offset, parent_ids=parent_ids))
+ resumable_source_manager.save_state(
+ MailchimpResumeConfig(
+ offset=offset,
+ parent_ids=parent_ids,
+ events_until=events_until.isoformat() if events_until else None,
+ )
+ )
- for record in records:
- yield {**record, **id_map}
+ rows = _explode_records(records, config.events, id_map, events_until) if config.events else records
+ yielded = False
+ for row in rows:
+ yielded = True
+ yield {**row, **id_map}
+ if not yielded:
+ resumable_source_manager.safe_point()
def _get_endpoint_iterator(
@@ -517,10 +588,12 @@ def _get_endpoint_iterator(
if not config.parents:
start_offset = 0
+ top_level_resume: MailchimpResumeConfig | None = None
if config.pagination == "offset" and resumable_source_manager.can_resume():
loaded = resumable_source_manager.load_state()
if loaded is not None and loaded.parent_ids is None:
start_offset = loaded.offset
+ top_level_resume = loaded
yield from _fetch_endpoint_rows(
session,
@@ -532,6 +605,7 @@ def _get_endpoint_iterator(
start_offset,
resumable_source_manager,
parent_ids=None,
+ events_until=_event_cutoff(config, should_use_incremental_field, top_level_resume),
)
return
@@ -547,6 +621,7 @@ def _get_endpoint_iterator(
known_chains = {tuple(id_map[key] for key in parent_keys) for id_map in id_maps}
if tuple(loaded.parent_ids) in known_chains:
resume_config = loaded
+ events_until = _event_cutoff(config, should_use_incremental_field, resume_config)
for id_map in id_maps:
parent_ids = [id_map[key] for key in parent_keys]
@@ -569,6 +644,7 @@ def _get_endpoint_iterator(
start_offset,
resumable_source_manager,
parent_ids=parent_ids,
+ events_until=events_until,
)
@@ -581,10 +657,18 @@ def mailchimp_source(
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
incremental_field: str | None = None,
+ sync_type: Optional[ExternalDataSchemaSyncType] = None,
) -> SourceResponse:
"""Create a Mailchimp data source for the specified endpoint."""
endpoint_config = MAILCHIMP_ENDPOINTS[endpoint]
+ if endpoint_config.events is not None:
+ # The schema API does not enforce `supports_append`, and append would keep re-read events.
+ if sync_type == ExternalDataSchemaSyncType.APPEND:
+ raise ValueError(EVENT_TABLE_APPEND_ERROR)
+ if isinstance(db_incremental_field_last_value, datetime):
+ db_incremental_field_last_value -= EVENT_LOOKBACK
+
# Contacts endpoint is special - fetches from all lists
if endpoint == "contacts":
return SourceResponse(
@@ -605,7 +689,7 @@ def mailchimp_source(
# Fan-out endpoints and the collections Mailchimp serves without offset pagination can't be
# expressed as a single `rest_api_resource`, so they go through the generic iterator.
- if endpoint_config.parents or endpoint_config.pagination != "offset":
+ if endpoint_config.parents or endpoint_config.events or endpoint_config.pagination != "offset":
return SourceResponse(
name=endpoint,
items=lambda: _get_endpoint_iterator(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/settings.py
index 1704f3ff46e2..d79dc8f206cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/settings.py
@@ -1,6 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
# How a collection endpoint pages:
@@ -28,7 +29,16 @@ class MailchimpParentConfig:
inject_as: str = "id"
-@dataclass
+@dataclass(frozen=True)
+class MailchimpEventsConfig:
+ """A per-recipient array to expand into one row per event, so each event's own timestamp can be the cursor."""
+
+ array_field: str
+ key_fields: tuple[str, ...]
+ """The event fields that identify one event. A fixed list keeps `event_id` stable when Mailchimp adds a field."""
+
+
+@dataclass(frozen=False)
class MailchimpEndpointConfig:
name: str
path: str
@@ -46,6 +56,7 @@ class MailchimpEndpointConfig:
"""Maps an incremental field name to the Mailchimp query param that filters on it server-side."""
chunk_size: Optional[int] = None
chunk_size_bytes: Optional[int] = None
+ events: Optional[MailchimpEventsConfig] = None
_LISTS_PARENT = MailchimpParentConfig(path="/lists", data_selector="lists", inject_as="list_id")
@@ -296,6 +307,32 @@ MAILCHIMP_ENDPOINTS: dict[str, MailchimpEndpointConfig] = {
incremental_fields=[],
primary_keys=["campaign_id", "email_id"],
),
+ # The tables above have no cursor column, and merging a `since` response on `email_id` would drop
+ # older events, so these tables hold one row per event.
+ "report_email_activity_events": MailchimpEndpointConfig(
+ name="report_email_activity_events",
+ path="/reports/{campaign_id}/email-activity",
+ data_selector="emails",
+ parents=(_REPORTS_PARENT,),
+ events=MailchimpEventsConfig(array_field="activity", key_fields=("action", "type", "timestamp", "url")),
+ primary_keys=["campaign_id", "event_id"],
+ partition_key="timestamp",
+ default_incremental_field="timestamp",
+ incremental_params={"timestamp": "since"},
+ incremental_fields=[incremental_field("timestamp")],
+ ),
+ "report_open_events": MailchimpEndpointConfig(
+ name="report_open_events",
+ path="/reports/{campaign_id}/open-details",
+ data_selector="members",
+ parents=(_REPORTS_PARENT,),
+ events=MailchimpEventsConfig(array_field="opens", key_fields=("timestamp",)),
+ primary_keys=["campaign_id", "event_id"],
+ partition_key="timestamp",
+ default_incremental_field="timestamp",
+ incremental_params={"timestamp": "since"},
+ incremental_fields=[incremental_field("timestamp")],
+ ),
"report_click_details": MailchimpEndpointConfig(
name="report_click_details",
path="/reports/{campaign_id}/click-details",
@@ -380,6 +417,8 @@ MAILCHIMP_ENDPOINTS: dict[str, MailchimpEndpointConfig] = {
ENDPOINTS = tuple(MAILCHIMP_ENDPOINTS.keys())
+EVENT_ENDPOINTS = frozenset(name for name, config in MAILCHIMP_ENDPOINTS.items() if config.events is not None)
+
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
name: config.incremental_fields for name, config in MAILCHIMP_ENDPOINTS.items()
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/source.py
index 713f1b36bb46..b0702c9472de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/source.py
@@ -14,18 +14,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.can
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
+ SourceSchema,
+ build_endpoint_schemas,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailchimp import (
MailchimpSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.mailchimp import (
+ EVENT_TABLE_APPEND_ERROR,
MailchimpResumeConfig,
mailchimp_source,
validate_credentials as validate_mailchimp_credentials,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.settings import (
ENDPOINTS,
+ EVENT_ENDPOINTS,
INCREMENTAL_FIELDS,
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
@@ -120,6 +125,10 @@ The API key format is: `key-dc` (e.g., `abc123def456-us6`), where `dc` is the da
"401 Client Error": "Invalid Mailchimp API key. Please check your API key and try again.",
"403 Client Error": "Access forbidden. Your API key may lack required permissions.",
"Invalid Mailchimp API key format": "Invalid API key format. Expected format: key-dc (e.g., abc123-us6)",
+ EVENT_TABLE_APPEND_ERROR: (
+ "Append only replication isn't supported on this table. Switch it to incremental or full table "
+ "replication in its sync settings, then re-enable the sync."
+ ),
}
def get_retryable_errors(self) -> set[str]:
@@ -143,19 +152,7 @@ The API key format is: `key-dc` (e.g., `abc123def456-us6`), where `dc` is the da
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- schemas = [
- SourceSchema(
- name=endpoint,
- supports_incremental=bool(INCREMENTAL_FIELDS.get(endpoint)),
- supports_append=bool(INCREMENTAL_FIELDS.get(endpoint)),
- incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
- )
- for endpoint in list(ENDPOINTS)
- ]
- if names is not None:
- names_set = set(names)
- schemas = [s for s in schemas if s.name in names_set]
- return schemas
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=EVENT_ENDPOINTS)
def validate_credentials(
self,
@@ -186,4 +183,5 @@ The API key format is: `key-dc` (e.g., `abc123def456-us6`), where `dc` is the da
if inputs.should_use_incremental_field
else None,
incremental_field=inputs.incremental_field,
+ sync_type=inputs.sync_type,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/tests/test_mailchimp.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/tests/test_mailchimp.py
index 959476f0c98c..e8e1df34d24c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/tests/test_mailchimp.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/tests/test_mailchimp.py
@@ -1,7 +1,7 @@
import re
import json
import dataclasses
-from datetime import date, datetime
+from datetime import UTC, date, datetime, timedelta
from typing import Any
import pytest
@@ -16,14 +16,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
always,
route,
scripted_network,
+ source_inputs,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailchimp import (
MailchimpSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.mailchimp import (
+ EVENT_TABLE_APPEND_ERROR,
MAX_RETRY_ATTEMPTS,
MailchimpResumeConfig,
MailchimpRetryableError,
+ _explode_records,
_format_incremental_value,
_get_with_retry,
_incremental_query_params,
@@ -32,6 +35,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.
)
from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.settings import MAILCHIMP_ENDPOINTS
from products.warehouse_sources.backend.temporal.data_imports.sources.mailchimp.source import MailchimpSource
+from products.warehouse_sources.backend.types import ExternalDataSchemaSyncType
class TestExtractDataCenter:
@@ -143,6 +147,10 @@ def _driver() -> SourceDriver:
return SourceDriver(MailchimpSource(), MailchimpSourceConfig(api_key="key-us6"))
+_TOMORROW = (datetime.now(UTC) + timedelta(days=1)).isoformat()
+_CLICK = {"action": "click", "timestamp": "2024-01-03T00:00:00+00:00", "url": "https://example.com", "ip": "192.0.2.1"}
+
+
class TestFetchContactsForList:
@pytest.mark.parametrize(
("label", "start_offset", "members", "total_items", "expected_ids", "expected_checkpoint"),
@@ -451,6 +459,164 @@ class TestGenericEndpointIterator:
}
assert len(result.rows) == 1
+ @pytest.mark.parametrize(
+ ("endpoint", "path", "data_selector", "field", "kept_events", "late_event"),
+ [
+ (
+ "report_email_activity_events",
+ "email-activity",
+ "emails",
+ "activity",
+ [
+ {"action": "open", "timestamp": "2024-01-03T00:00:00+00:00", "ip": "192.0.2.1"},
+ {"action": "click", "timestamp": "2024-01-03T00:01:00+00:00", "url": "https://example.com"},
+ {"action": "click", "timestamp": "2024-01-03T00:01:00+00:00", "url": "https://example.com"},
+ ],
+ {"action": "open", "timestamp": _TOMORROW},
+ ),
+ (
+ "report_open_events",
+ "open-details",
+ "members",
+ "opens",
+ [
+ {"timestamp": "2024-01-03T00:00:00+00:00", "is_proxy_open": True},
+ {"timestamp": "2024-01-03T00:01:00+00:00", "is_proxy_open": False},
+ {"timestamp": "2024-01-03T00:01:00+00:00", "is_proxy_open": False},
+ ],
+ {"timestamp": _TOMORROW, "is_proxy_open": False},
+ ),
+ ],
+ )
+ def test_event_tables_send_since_to_every_campaign_and_yield_one_row_per_event(
+ self,
+ endpoint: str,
+ path: str,
+ data_selector: str,
+ field: str,
+ kept_events: list[dict[str, Any]],
+ late_event: dict[str, Any],
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ "/reports": [ScriptedResponse(json={"reports": [{"id": "c1"}, {"id": "c2"}], "total_items": 2})],
+ f"/reports/c1/{path}": [
+ ScriptedResponse(
+ json={
+ data_selector: [
+ {
+ "email_id": "e1",
+ "opens_count": 3,
+ "_links": [{"rel": "self"}],
+ field: [*kept_events, late_event],
+ },
+ {"email_id": "e2", field: []},
+ ],
+ "total_items": 2,
+ }
+ )
+ ],
+ f"/reports/c2/{path}": [ScriptedResponse(json={data_selector: [], "total_items": 0})],
+ }
+ ),
+ incremental_field="timestamp",
+ db_incremental_field_last_value=datetime(2024, 1, 2, 3, 4, 5),
+ )
+
+ assert result.raised is None
+ assert result.params("since") == [None, "2024-01-01T03:04:05+00:00", "2024-01-01T03:04:05+00:00"]
+ assert [{key: value for key, value in row.items() if key != "event_id"} for row in result.rows] == [
+ {"email_id": "e1", **event, "campaign_id": "c1"} for event in kept_events
+ ]
+ assert len({row["event_id"] for row in result.rows}) == len(kept_events)
+ assert [state.parent_ids for state in result.saved_states] == [["c1"], ["c2"]]
+ assert len({state.events_until for state in result.saved_states} - {None}) == 1
+
+ def test_resumed_event_sync_keeps_the_first_attempt_cutoff(self) -> None:
+ opens = [
+ {"timestamp": "2024-01-04T00:00:00+00:00", "is_proxy_open": False},
+ {"timestamp": "2024-01-06T00:00:00+00:00", "is_proxy_open": False},
+ ]
+ result = _driver().run(
+ "report_open_events",
+ route(
+ {
+ "/reports": [ScriptedResponse(json={"reports": [{"id": "c1"}], "total_items": 1})],
+ "/reports/c1/open-details": [
+ ScriptedResponse(json={"members": [{"email_id": "e1", "opens": opens}], "total_items": 1})
+ ],
+ }
+ ),
+ incremental_field="timestamp",
+ db_incremental_field_last_value=datetime(2024, 1, 1),
+ resume_state=MailchimpResumeConfig(offset=0, parent_ids=["c1"], events_until="2024-01-05T00:00:00+00:00"),
+ )
+
+ assert result.raised is None
+ assert [row["timestamp"] for row in result.rows] == ["2024-01-04T00:00:00+00:00"]
+
+ def test_filtered_walk_reads_past_an_empty_middle_page(self) -> None:
+ later_page = {"members": [{"email_id": "e1", "opens": [{"timestamp": "2024-01-03T00:00:00+00:00"}]}]}
+ result = _driver().run(
+ "report_open_events",
+ route(
+ {
+ "/reports": [ScriptedResponse(json={"reports": [{"id": "c1"}], "total_items": 1})],
+ "/reports/c1/open-details": [
+ ScriptedResponse(json={"members": [], "total_items": 1500}),
+ ScriptedResponse(json={**later_page, "total_items": 1500}),
+ ],
+ }
+ ),
+ incremental_field="timestamp",
+ db_incremental_field_last_value=datetime(2024, 1, 2),
+ )
+
+ assert result.raised is None
+ assert result.params("offset") == ["0", "0", "1000"]
+ assert [row["email_id"] for row in result.rows] == ["e1"]
+
+ def test_event_tables_refuse_append(self) -> None:
+ source = MailchimpSource()
+ inputs = dataclasses.replace(
+ source_inputs("report_open_events", incremental_field="timestamp"),
+ sync_type=ExternalDataSchemaSyncType.APPEND,
+ )
+
+ with pytest.raises(ValueError, match=EVENT_TABLE_APPEND_ERROR):
+ source.source_for_pipeline(
+ MailchimpSourceConfig(api_key="key-us6"), source.get_resumable_source_manager(inputs), inputs
+ )
+
+ @pytest.mark.parametrize(
+ "later_read",
+ [
+ pytest.param({"email_id": "e1", "opens_count": 5, "activity": [_CLICK]}, id="member-counters-change"),
+ pytest.param(
+ {"email_id": "e1", "opens_count": 1, "activity": [{**_CLICK, "user_agent": "Mail"}]},
+ id="new-event-field",
+ ),
+ pytest.param({"email_id": "e1", "opens_count": 1, "activity": [{**_CLICK, "ip": None}]}, id="ip-blanked"),
+ pytest.param(
+ {"email_id": "e1", "opens_count": 1, "activity": [{**_CLICK, "timestamp": "2024-01-03T00:00:00Z"}]},
+ id="same-instant-other-iso-form",
+ ),
+ ],
+ )
+ def test_event_id_ignores_changes_that_are_not_the_event(self, later_read: dict[str, Any]) -> None:
+ events = MAILCHIMP_ENDPOINTS["report_email_activity_events"].events
+ assert events is not None
+ first_read = {"email_id": "e1", "opens_count": 1, "activity": [_CLICK]}
+
+ first_id = next(_explode_records([first_read], events, {"campaign_id": "c1"}, None))["event_id"]
+ later_id = next(_explode_records([later_read], events, {"campaign_id": "c1"}, None))["event_id"]
+
+ assert first_id == later_id
+ # Stored rows merge on this id, so a change to how it is built duplicates every re-read event.
+ assert first_id == "3ae9e90e7be319ae0a3d75038b8b55abe5502a3462db61f522ac4b3e6499f1cf"
+
def test_fan_out_injects_parent_id_onto_rows_the_api_omits(self) -> None:
# /reports/{id}/locations rows carry no campaign_id, so without injection the
# primary key would collapse every campaign's regions onto each other.