Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the Courier source (#120497)

Tom Owers697267bac626
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 98f0ab986996..eccbbcff6374 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -3356,12 +3356,12 @@ Diffed against: <https://www.courier.com/docs/llms.txt>
- [x] `messages/{id}/history` — per-message state transition history (queued, sent, delivered, opened, clicked) - the deliverability funnel (high)
- [x] `audiences/{audience_id}/members` — membership table for audiences we already sync; audiences without members are just filter definitions (high)
- [x] `tenants/{tenant_id}/users` — membership table joining users to the tenants we already sync (medium)
-- [ ] `automations (GET /automations)` — lookup for saved automation templates that trigger sends (medium)
+- [x] `automations (GET /automations)` — lookup for saved automation templates that trigger sends (medium) — added as `Automations`
- [x] `journeys (GET /journeys, plus journey versions)` — journey definitions and versions needed to attribute messages to a flow (medium)
- [x] `digests (list digest instances)` — digest batching data explaining why messages were grouped or delayed (medium). Digest schedules have no listing endpoint; their ids are flattened out of `GET /preferences/sections`, where Courier nests them under each topic's digest config.
-- [ ] `notification-templates/{id}/versions` — template version history for before/after performance comparison (low)
-- [ ] `workspace-preferences (topics and sections)` — lookup for subscription topics referenced by user preference data (low)
-- [ ] `user-preferences (GET /users/{id}/preferences)` — per-user subscription state for opt-out analysis (low)
+- [x] `notification-templates/{id}/versions` — template version history for before/after performance comparison (low) — added as `NotificationTemplateVersions`
+- [x] `workspace-preferences (topics and sections)` — lookup for subscription topics referenced by user preference data (low) — added as `PreferenceSections` and `PreferenceTopics`
+- [x] `user-preferences (GET /users/{id}/preferences)` — per-user subscription state for opt-out analysis (low) — added as `PreferenceChanges` from `GET /preferences/logs`, because Courier has no user listing to fan the per-user endpoint out over
Note: The payload's docs URLs (www.courier.com/docs/reference/...) now 404; the live reference lives under /docs/api-reference/ and is indexed in llms.txt. Providers and routing-strategies were excluded as configuration.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/canonical_descriptions.py
index 71fe6934e364..2ace4a64b3a5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/canonical_descriptions.py
@@ -116,6 +116,80 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"event_ids": "Event ids mapped to this template.",
},
},
+ "NotificationTemplateVersions": {
+ "description": "A version of a notification template, most recent first, for comparing performance across publishes.",
+ "docs_url": "https://www.courier.com/docs/api-reference/templates/list-notification-template-versions",
+ "columns": {
+ "template_id": "Unique identifier of the notification template this version belongs to.",
+ "version": "Version identifier: `draft`, `published:vNNN` for the current published version, or `vNNN` for a historical version.",
+ "created": "Timestamp at which this version was created.",
+ "creator": "User id of the version creator.",
+ "has_changes": "Whether the draft has unpublished changes. Only present on the draft version.",
+ },
+ },
+ "Automations": {
+ "description": "A saved automation template in the workspace, the legacy workflow product that Journeys supersede.",
+ "docs_url": "https://www.courier.com/docs/api-reference/automations/list-automations",
+ "columns": {
+ "id": "Unique identifier of the automation template.",
+ "name": "Name of the automation template.",
+ "version": "Which version of the template this row describes (`published` or `draft`).",
+ "createdAt": "Timestamp at which the template was created.",
+ "updatedAt": "Timestamp at which the template was last updated.",
+ },
+ },
+ "PreferenceSections": {
+ "description": "A preference section in the workspace, grouping the subscription topics shown on the preference page.",
+ "docs_url": "https://www.courier.com/docs/api-reference/preference-topics/list-preference-sections",
+ "columns": {
+ "id": "Unique identifier of the preference section.",
+ "name": "Human-readable name of the section.",
+ "description": "Optional description shown under the section on the hosted preferences page.",
+ "routing_options": "Default channels for the section. May be empty.",
+ "has_custom_routing": "Whether the section defines custom routing for its topics.",
+ "topics": "The subscription topics contained in this section.",
+ "created": "Timestamp at which the section was created.",
+ "creator": "Id of the creator.",
+ "updated": "Timestamp of the last update.",
+ "updater": "Id of the last updater.",
+ },
+ },
+ "PreferenceTopics": {
+ "description": "A subscription topic in the workspace, the topic that user preferences and preference changes refer to.",
+ "docs_url": "https://www.courier.com/docs/api-reference/preference-topics/list-topics-in-section",
+ "columns": {
+ "section_id": "Unique identifier of the preference section the topic belongs to.",
+ "id": "Unique identifier of the subscription topic.",
+ "name": "Human-readable name of the topic.",
+ "description": "Optional description shown under the topic on the hosted preferences page.",
+ "default_status": "Subscription status applied when a recipient has not set their own (`OPTED_IN`, `OPTED_OUT` or `REQUIRED`).",
+ "routing_options": "Default channels delivered for this topic. May be empty.",
+ "allowed_preferences": "Preference controls a recipient may customize. May be empty.",
+ "include_unsubscribe_header": "Whether a list-unsubscribe header is included on emails for this topic.",
+ "topic_data": "Arbitrary metadata associated with the topic.",
+ "digest": "The topic's digest configuration, when one is set.",
+ "created": "Timestamp at which the topic was created.",
+ "creator": "Id of the creator.",
+ "updated": "Timestamp of the last update.",
+ "updater": "Id of the last updater.",
+ },
+ },
+ "PreferenceChanges": {
+ "description": "One change a user made to one subscription topic, with the value before it. The latest change per user and topic gives that user's current subscription state.",
+ "docs_url": "https://www.courier.com/docs/api-reference/preference-topics/list-preference-changes",
+ "columns": {
+ "id": "Unique identifier for this change.",
+ "timestamp": "Timestamp at which the change was made.",
+ "user_id": "The user whose preference changed.",
+ "tenant_id": "The tenant context the change was made in. Absent when the user set the preference outside any tenant.",
+ "topic_id": "The subscription topic the change applies to.",
+ "topic_name": "Display name of the topic when the change was made.",
+ "status": "The subscription status the change set (`OPTED_IN`, `OPTED_OUT` or `REQUIRED`).",
+ "custom_routing": "The channels chosen for this topic. Empty unless `has_custom_routing` is true.",
+ "has_custom_routing": "Whether specific delivery channels were chosen rather than the topic's default routing.",
+ "previous": "The status and routing before this change, where it was recorded.",
+ },
+ },
"TenantUsers": {
"description": "A user's membership of a tenant, which is who a tenant-scoped send reaches.",
"docs_url": "https://www.courier.com/docs/api-reference/tenants/get-users-in-tenant",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/courier.py b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/courier.py
index 5f00db4bf211..de1482cfed17 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/courier.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/courier.py
@@ -34,7 +34,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sou
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.courier.settings import (
COURIER_BASE_URL,
- COURIER_PAGE_SIZE,
ENDPOINTS_CONFIG,
FANOUT_ENDPOINT_CONFIGS,
CourierEndpointConfig,
@@ -126,7 +125,7 @@ def _encode_parent_field(source_field: str, target_field: str) -> Callable[[dict
def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResource:
config = ENDPOINTS_CONFIG[name]
- params: dict[str, Any] = {"limit": COURIER_PAGE_SIZE}
+ params: dict[str, Any] = {"limit": config.page_size} if config.paginated else {}
if should_use_incremental_field and config.incremental_param and config.incremental_fields:
params[config.incremental_param] = {
"type": "incremental",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/settings.py
index cdab1d763430..14dda29c291b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/settings.py
@@ -218,6 +218,91 @@ ENDPOINTS_CONFIG: dict[str, CourierEndpointConfig] = {
data_selector="results",
primary_keys=("id",),
),
+ # Each template's draft and published versions, for comparing performance across publishes.
+ # Courier relabels the current version ("published:v003" becomes "v003" on the next publish),
+ # so full refresh only.
+ "NotificationTemplateVersions": CourierEndpointConfig(
+ name="NotificationTemplateVersions",
+ path="/notifications/{template_id}/versions",
+ data_selector="versions",
+ # The version entry carries no reference back to its template, so the parent's id is
+ # projected in to make rows joinable and unique across templates.
+ primary_keys=("template_id", "version"),
+ partition_key="created",
+ timestamp_fields=("created",),
+ # Courier caps this endpoint at 10 versions per page and rejects a larger `limit`.
+ page_size=10,
+ fanout=DependentEndpointConfig(
+ parent_name="NotificationTemplates",
+ resolve_param="template_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "template_id"},
+ # A template deleted between the listing and this fetch 404s; skip that parent rather
+ # than failing the whole fan-out.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ # Saved automation templates (the workflow product Journeys supersede, still running). No
+ # server-side timestamp filter, so full refresh only.
+ "Automations": CourierEndpointConfig(
+ name="Automations",
+ path="/automations",
+ data_selector="templates",
+ primary_keys=("id",),
+ partition_key="createdAt",
+ timestamp_fields=("createdAt", "updatedAt"),
+ cursor_path="cursor",
+ ),
+ # The preference sections that group subscription topics. One unpaginated response that
+ # documents no params at all.
+ "PreferenceSections": CourierEndpointConfig(
+ name="PreferenceSections",
+ path="/preferences/sections",
+ data_selector="results",
+ primary_keys=("id",),
+ partition_key="created",
+ timestamp_fields=("created", "updated"),
+ paginated=False,
+ ),
+ # The subscription topics that user preferences and preference changes refer to by
+ # `topic_id`. Fetched per section so each topic row carries its section.
+ "PreferenceTopics": CourierEndpointConfig(
+ name="PreferenceTopics",
+ path="/preferences/sections/{section_id}/topics",
+ data_selector="results",
+ primary_keys=("section_id", "id"),
+ partition_key="created",
+ timestamp_fields=("created", "updated"),
+ paginated=False,
+ # Neither the section listing nor the topic listing documents a page-size param.
+ fanout_page_size_param=None,
+ fanout=DependentEndpointConfig(
+ parent_name="PreferenceSections",
+ resolve_param="section_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "section_id"},
+ # A section archived between the listing and this fetch 404s; skip that parent rather
+ # than failing the whole fan-out.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ # Every preference change users made, with the value before it. Courier has no listing of
+ # users to fan `GET /users/{id}/preferences` out over, so this environment-wide log is where
+ # per-user subscription state comes from. `since` is a documented server-side filter.
+ "PreferenceChanges": CourierEndpointConfig(
+ name="PreferenceChanges",
+ path="/preferences/logs",
+ data_selector="items",
+ primary_keys=("id",),
+ incremental_fields=[incremental_field("timestamp")],
+ incremental_param="since",
+ partition_key="timestamp",
+ # Documented as newest first.
+ sort_mode="desc",
+ timestamp_fields=("timestamp",),
+ ),
# Multi-tenant scoping objects. No timestamp fields at all, so no partitioning.
"Tenants": CourierEndpointConfig(
name="Tenants",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier.py b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier.py
index 3e276073e745..6df0dadcfa09 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier.py
@@ -89,6 +89,8 @@ class TestGetResource:
("NotificationTemplates",),
("Tenants",),
("Journeys",),
+ ("Automations",),
+ ("PreferenceChanges",),
]
)
def test_full_refresh_sends_no_timestamp_filter(self, endpoint: str) -> None:
@@ -106,6 +108,7 @@ class TestGetResource:
("NotificationTemplates",),
("Tenants",),
("Journeys",),
+ ("Automations",),
]
)
def test_endpoints_without_a_server_filter_ignore_incremental_flag(self, endpoint: str) -> None:
@@ -127,6 +130,9 @@ class TestCourierSourceTransport:
("NotificationTemplates",),
("Tenants",),
("Journeys",),
+ ("Automations",),
+ ("PreferenceSections",),
+ ("PreferenceChanges",),
]
)
def test_rows_extracted_from_endpoint_envelope(self, endpoint: str) -> None:
@@ -150,7 +156,7 @@ class TestCourierSourceTransport:
assert result.saved_states == [CourierResumeConfig(cursor="cursor-1"), CourierResumeConfig(cursor="cursor-2")]
- @parameterized.expand([("Tenants",), ("Journeys",)])
+ @parameterized.expand([("Tenants",), ("Journeys",), ("Automations",)])
def test_pages_on_top_level_cursor(self, endpoint: str) -> None:
# These endpoints return their cursor at the response's top level rather than nested
# under `paging`, so a regression here silently disables pagination.
@@ -163,6 +169,14 @@ class TestCourierSourceTransport:
assert result.params("cursor") == [None, "cursor-1"]
assert [row["id"] for row in result.rows] == ["r1", "r2"]
+ def test_unpaginated_endpoint_sends_no_params(self) -> None:
+ # The preference sections listing documents no params at all, so the page size every
+ # other endpoint gets must not be sent to it.
+ result = _drive("PreferenceSections", [_page("PreferenceSections", [{"id": "ps_1"}], cursor=None)])
+
+ assert result.paths == ["/preferences/sections"]
+ assert result.queries == [{}]
+
def test_resume_seeds_paginator_with_saved_cursor(self) -> None:
responses = [_page("Messages", [{"id": "m9"}], cursor=None)]
result = _drive("Messages", responses, resume_state=CourierResumeConfig(cursor="cursor-resumed"))
@@ -174,29 +188,28 @@ class TestCourierSourceTransport:
assert result.saved_states == []
- def test_incremental_run_sends_watermark_as_iso8601(self) -> None:
- responses = [_page("Messages", [{"id": "m1"}], cursor=None)]
- result = _drive(
- "Messages",
- responses,
- should_use_incremental_field=True,
- db_incremental_field_last_value=datetime(2026, 5, 1, 12, 30),
- )
-
- assert result.params("enqueued_after") == ["2026-05-01T12:30:00"]
-
- def test_incremental_first_sync_defaults_to_epoch_start(self) -> None:
- # No prior watermark (first incremental sync): must fall back to a real ISO-8601 value,
- # not stringify `None` into the literal "enqueued_after=None".
- responses = [_page("Messages", [{"id": "m1"}], cursor=None)]
+ @parameterized.expand(
+ [
+ ("messages", "Messages", "enqueued_after", datetime(2026, 5, 1, 12, 30), "2026-05-01T12:30:00"),
+ ("preference_changes", "PreferenceChanges", "since", datetime(2026, 5, 1, 12, 30), "2026-05-01T12:30:00"),
+ # No prior watermark (first incremental sync): must fall back to a real ISO-8601
+ # value, not stringify `None` into the literal "since=None".
+ ("messages_first_sync", "Messages", "enqueued_after", None, "1970-01-01T00:00:00Z"),
+ ("preference_changes_first_sync", "PreferenceChanges", "since", None, "1970-01-01T00:00:00Z"),
+ ]
+ )
+ def test_incremental_run_sends_watermark_as_iso8601(
+ self, _name: str, endpoint: str, param: str, last_value: datetime | None, expected: str
+ ) -> None:
+ responses = [_page(endpoint, [{"id": "r1"}], cursor=None)]
result = _drive(
- "Messages",
+ endpoint,
responses,
should_use_incremental_field=True,
- db_incremental_field_last_value=None,
+ db_incremental_field_last_value=last_value,
)
- assert result.params("enqueued_after") == ["1970-01-01T00:00:00Z"]
+ assert result.params(param) == [expected]
def test_epoch_millis_fields_are_converted_to_datetime(self) -> None:
# `enqueued`/`sent`/etc. arrive as epoch-millisecond ints; left unconverted, the
@@ -224,6 +237,9 @@ class TestCourierSourceTransport:
("NotificationTemplates", None, "asc"),
("Tenants", None, "asc"),
("Journeys", ["createdAt"], "asc"),
+ ("Automations", ["createdAt"], "asc"),
+ ("PreferenceSections", ["created"], "asc"),
+ ("PreferenceChanges", ["timestamp"], "desc"),
]
)
def test_source_response_partitioning_and_sort_mode(
@@ -378,6 +394,45 @@ class TestCourierFanout:
assert result.paths == ["/journeys", "/journeys/jry-1/versions", "/journeys/jry-1/versions"]
assert [row["version"] for row in result.rows] == ["v1", "v1"]
+ def test_notification_template_versions_carry_their_template_id_and_page_at_ten(self) -> None:
+ # Courier rejects a `limit` above 10 on the versions endpoint, so the child must not
+ # inherit the page size of its template listing.
+ result = _drive(
+ "NotificationTemplateVersions",
+ [
+ _page("NotificationTemplates", [{"id": "nt_1"}], cursor=None),
+ _page("NotificationTemplateVersions", [{"version": "v002", "created": 1_700_000_000_000}], "c-1"),
+ _page("NotificationTemplateVersions", [{"version": "v001", "created": 1_690_000_000_000}], None),
+ ],
+ )
+
+ assert result.paths == ["/notifications", "/notifications/nt_1/versions", "/notifications/nt_1/versions"]
+ assert result.params("limit") == ["100", "10", "10"]
+ assert result.params("cursor") == [None, None, "c-1"]
+ assert [(row["template_id"], row["version"]) for row in result.rows] == [("nt_1", "v002"), ("nt_1", "v001")]
+ assert result.rows[0]["created"] == datetime.fromtimestamp(1_700_000_000_000 / 1000, tz=UTC)
+
+ def test_preference_topics_are_fetched_per_section_without_params(self) -> None:
+ # Neither listing documents a page-size param, and the topic carries no reference back to
+ # its section, so the section id has to be projected in from the parent.
+ result = _drive(
+ "PreferenceTopics",
+ [
+ _page("PreferenceSections", [{"id": "ps_1"}, {"id": "ps_2"}], cursor=None),
+ _page("PreferenceTopics", [{"id": "pt_1", "created": "2026-01-15T10:30:00.000Z"}], cursor=None),
+ _page("PreferenceTopics", [{"id": "pt_2"}], cursor=None),
+ ],
+ )
+
+ assert result.paths == [
+ "/preferences/sections",
+ "/preferences/sections/ps_1/topics",
+ "/preferences/sections/ps_2/topics",
+ ]
+ assert result.queries == [{}, {}, {}]
+ assert [(row["section_id"], row["id"]) for row in result.rows] == [("ps_1", "pt_1"), ("ps_2", "pt_2")]
+ assert result.rows[0]["created"] == datetime(2026, 1, 15, 10, 30, 0, tzinfo=UTC)
+
def test_digest_instances_flatten_schedule_ids_out_of_the_preferences_response(self) -> None:
# Courier exposes no listing of digest schedules, so the ids come from the schedules
# nested inside each preference topic's digest config.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier_source.py
index 55e39d5fed73..41829ae67720 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/courier/tests/test_courier_source.py
@@ -30,9 +30,13 @@ class TestGetSchemas:
def test_only_server_side_filtered_endpoints_are_incremental(self) -> None:
schemas = {s.name: s for s in CourierSource().get_schemas(_config(), team_id=1)}
assert set(schemas) == set(ENDPOINTS)
- # Messages filters on `enqueued_after`; MessageHistory has no filter of its own and
- # inherits that window through its parent listing.
- assert {name for name, s in schemas.items() if s.supports_incremental} == {"Messages", "MessageHistory"}
+ # Messages filters on `enqueued_after` and PreferenceChanges on `since`; MessageHistory has
+ # no filter of its own and inherits that window through its parent listing.
+ assert {name for name, s in schemas.items() if s.supports_incremental} == {
+ "Messages",
+ "MessageHistory",
+ "PreferenceChanges",
+ }
assert [f["field"] for f in schemas["Messages"].incremental_fields] == ["enqueued"]
def test_lists_tables_without_credentials(self) -> None: