Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Systeme.io source (#124921)

Tom Owersd74c107c609d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 6a7f514f83f8..20a29a5a1363 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11940,8 +11940,8 @@ Diffed against: <https://developer.systeme.io/llms.txt>
- [ ] `Email statistics (GET /api/mailing/statistics/newsletters; GET /api/mailing/statistics/campaign-steps; GET /api/mailing/statistics/automation-emails)` — compare messages sent, opens, clicks, bounces, and spam reports across email types (high)
- [x] `Digital products (GET /api/payment/digital-products)` — analyze the digital product catalog alongside courses and subscription offerings (medium)
- [x] `Physical products (GET /api/payment/physical-products)` — analyze physical product offerings and compare them with digital products (medium)
-- [ ] `Campaigns (GET /api/mailing/campaigns)` — analyze automated email campaigns alongside newsletters (medium)
-- [ ] `Campaign steps (GET /api/mailing/campaigns/{campaignId}/steps)` — connect email performance to each message and its position in a campaign (medium)
+- [x] `Campaigns (GET /api/mailing/campaigns)` — analyze automated email campaigns alongside newsletters (medium)
+- [x] `Campaign steps (GET /api/mailing/campaigns/{campaignId}/steps)` — connect email performance to each message and its position in a campaign (medium)
- [ ] `Funnels (GET /api/funnels)` — identify marketing funnels for comparisons with contacts and their source URLs (medium)
- [ ] `Funnel steps (GET /api/funnels/{funnelId}/steps)` — describe funnel pages and their sequence for acquisition analysis (medium)
- [ ] `Bookings (GET /api/booking-calendar/bookings)` — analyze appointment demand and booking activity alongside contact growth (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py
index 52df9c5235cc..1cf7b8ef2aa4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py
@@ -110,4 +110,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"stock": "Number of units in stock when the stock is limited.",
},
},
+ "campaigns": {
+ "description": "Automated email campaigns that send a sequence of steps to contacts.",
+ "docs_url": "https://developer.systeme.io/reference/api_mailingcampaigns_get_collection",
+ "columns": {
+ "id": "Unique campaign identifier.",
+ "name": "Campaign name.",
+ "senderEmail": "Email address the campaign sends from.",
+ },
+ },
+ "campaign_steps": {
+ "description": "Email steps of each campaign, with content metadata and position in the sequence.",
+ "docs_url": "https://developer.systeme.io/reference/api_mailingcampaigns_campaignidsteps_get_collection",
+ "columns": {
+ "id": "Unique campaign step identifier.",
+ "campaignId": "Identifier of the campaign that contains the step.",
+ "content": "Step email subject, preview text, editor type, and sender details.",
+ "state": "Whether the step is active, and its position in the campaign.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py
index 7f6ba672d57e..5acf0e88c32e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py
@@ -13,8 +13,13 @@ ENDPOINTS = {
"memberships": "community/memberships",
"digital_products": "payment/digital-products",
"physical_products": "payment/physical-products",
+ "campaigns": "mailing/campaigns",
+ "campaign_steps": "mailing/campaigns/{campaignId}/steps",
}
+# Child endpoint -> parent endpoint whose `id` fills the child path's `{campaignId}`.
+FANOUT_PARENTS = {"campaign_steps": "campaigns"}
+
INCREMENTAL_FIELDS = {"contacts": [incremental_field("registeredAt")]}
PARTITION_KEYS = {
"contacts": "registeredAt",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py
index c809b06bfaed..2ce486a9c271 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py
@@ -6,16 +6,19 @@ from requests import HTTPError, Response
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
+ ClientConfig,
Endpoint,
EndpointResource,
RESTAPIConfig,
rest_api_resource,
+ rest_api_resources,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponseCursorPaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
@@ -24,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.systeme.se
AUTH_ERROR,
BASE_URL,
ENDPOINTS,
+ FANOUT_PARENTS,
PAGE_SIZE,
PARTITION_KEYS,
PERMISSION_ERROR,
@@ -34,6 +38,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.systeme.se
class SystemeResumeConfig:
cursor: int | None = None
completed: bool = False
+ fanout_state: dict[str, Any] | None = None
class SystemePaginator(JSONResponseCursorPaginator):
@@ -54,7 +59,8 @@ class SystemePaginator(JSONResponseCursorPaginator):
def validate_credentials(api_key: str, schema_name: str | None = None) -> tuple[bool, str | None]:
- path = schema_for_resource(ENDPOINTS, schema_name or "contacts")
+ resource_name = schema_name or "contacts"
+ path = schema_for_resource(ENDPOINTS, FANOUT_PARENTS.get(resource_name, resource_name))
client = RESTClient(
base_url=BASE_URL,
auth=APIKeyAuth(api_key=api_key, name="X-API-Key"),
@@ -73,6 +79,64 @@ def validate_credentials(api_key: str, schema_name: str | None = None) -> tuple[
return True, None
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": BASE_URL,
+ "auth": {"type": "api_key", "name": "X-API-Key", "api_key": api_key, "location": "header"},
+ "paginator": SystemePaginator(),
+ "request_timeout": 30,
+ }
+
+
+def _fanout_resource(
+ api_key: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[SystemeResumeConfig],
+) -> Resource:
+ parent = FANOUT_PARENTS[endpoint]
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state:
+ resumable_source_manager.save_state(SystemeResumeConfig(fanout_state=state))
+
+ config: RESTAPIConfig = {
+ "client": _client_config(api_key),
+ "resources": [
+ {
+ "name": parent,
+ "endpoint": {
+ "path": schema_for_resource(ENDPOINTS, parent),
+ "params": {"limit": PAGE_SIZE, "order": "desc"},
+ "data_selector": "items",
+ "data_selector_required": True,
+ },
+ },
+ {
+ "name": endpoint,
+ "endpoint": {
+ # The child collection documents no limit, cursor or order parameter.
+ "path": schema_for_resource(ENDPOINTS, endpoint),
+ "params": {"campaignId": {"type": "resolve", "resource": parent, "field": "id"}},
+ "data_selector": "items",
+ "data_selector_required": True,
+ },
+ },
+ ],
+ }
+ resources = rest_api_resources(
+ config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=resume.fanout_state if resume else None,
+ )
+ return next(resource for resource in resources if resource.name == endpoint)
+
+
def systeme_source(
api_key: str,
endpoint: str,
@@ -83,6 +147,10 @@ def systeme_source(
db_incremental_field_last_value: datetime | str | None = None,
) -> SourceResponse:
path = schema_for_resource(ENDPOINTS, endpoint)
+ if endpoint in FANOUT_PARENTS:
+ fanout_resource = _fanout_resource(api_key, endpoint, team_id, job_id, resumable_source_manager)
+ return SourceResponse(name=endpoint, items=lambda: fanout_resource, primary_keys=["id"])
+
params: dict[str, Any] = {"limit": PAGE_SIZE, "order": "desc"}
if endpoint == "contacts" and should_use_incremental_field and db_incremental_field_last_value is not None:
params["registeredAfter"] = (
@@ -106,12 +174,7 @@ def systeme_source(
}
resource_config: EndpointResource = {"name": endpoint, "endpoint": endpoint_config}
config: RESTAPIConfig = {
- "client": {
- "base_url": BASE_URL,
- "auth": {"type": "api_key", "name": "X-API-Key", "api_key": api_key, "location": "header"},
- "paginator": SystemePaginator(),
- "request_timeout": 30,
- },
+ "client": _client_config(api_key),
"resources": [resource_config],
}
resource = rest_api_resource(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py
index 68b8f81b56f3..01ae2411f090 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py
@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
@@ -60,6 +61,74 @@ def test_incremental_filter_and_full_refresh(
assert result.params("registeredAfter") == [expected, expected]
+CAMPAIGN_20_STEPS = "/api/mailing/campaigns/20/steps"
+CAMPAIGN_10_STEPS = "/api/mailing/campaigns/10/steps"
+STEP_202 = {"id": 202, "campaignId": 20}
+STEP_201 = {"id": 201, "campaignId": 20}
+STEP_101 = {"id": 101, "campaignId": 10}
+
+
+ "fanout_state,steps_20_pages,expected_rows,expected_step_cursors",
+ [
+ (None, [[STEP_202], [STEP_201]], [STEP_202, STEP_201, STEP_101], [None, "202", None]),
+ (
+ {"completed": ["mailing/campaigns/20/steps"], "current": None, "child_state": None},
+ [],
+ [STEP_101],
+ [None],
+ ),
+ (
+ {"completed": [], "current": "mailing/campaigns/20/steps", "child_state": {"cursor": 202}},
+ [[STEP_201]],
+ [STEP_201, STEP_101],
+ ["202", None],
+ ),
+ ],
+)
+def test_campaign_steps_fan_out_and_resume(
+ fanout_state: dict[str, Any] | None,
+ steps_20_pages: list[list[dict[str, Any]]],
+ expected_rows: list[dict[str, Any]],
+ expected_step_cursors: list[str | None],
+) -> None:
+ steps_20 = [
+ ScriptedResponse(json={"items": page, "hasMore": index < len(steps_20_pages) - 1})
+ for index, page in enumerate(steps_20_pages)
+ ]
+ result = driver().run(
+ "campaign_steps",
+ route(
+ {
+ "/api/mailing/campaigns": [
+ ScriptedResponse(json={"items": [{"id": 20, "name": "Welcome"}], "hasMore": True}),
+ ScriptedResponse(json={"items": [{"id": 10, "name": "Onboarding"}], "hasMore": False}),
+ ],
+ CAMPAIGN_20_STEPS: steps_20,
+ CAMPAIGN_10_STEPS: [ScriptedResponse(json={"items": [STEP_101], "hasMore": False})],
+ }
+ ),
+ resume_state=SystemeResumeConfig(fanout_state=fanout_state) if fanout_state else None,
+ )
+ assert result.raised is None
+ assert result.rows == expected_rows
+ campaign_queries = [dict(request.query) for request in result.requests if request.path == "/api/mailing/campaigns"]
+ assert campaign_queries == [
+ {"limit": ("100",), "order": ("desc",)},
+ {"limit": ("100",), "order": ("desc",), "startingAfter": ("20",)},
+ ]
+ step_requests = [request for request in result.requests if request.path.endswith("/steps")]
+ assert [request.param("startingAfter") for request in step_requests] == expected_step_cursors
+ assert [request.param("limit") for request in step_requests] == [None] * len(step_requests)
+ assert result.final_state == SystemeResumeConfig(
+ fanout_state={
+ "completed": ["mailing/campaigns/10/steps", "mailing/campaigns/20/steps"],
+ "current": None,
+ "child_state": None,
+ }
+ )
+
+
@pytest.mark.parametrize(
"body,message",
[
@@ -92,6 +161,7 @@ def test_invalid_or_repeated_cursor_fails(body: dict[str, Any], message: str) ->
("newsletters", "mailing/newsletters"),
("digital_products", "payment/digital-products"),
("physical_products", "payment/physical-products"),
+ ("campaign_steps", "mailing/campaigns"),
],
)
def test_validate_credentials_status_and_probe(