:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 2 missing tables to the Systeme.io source (#124921)
Tom Owersd74c107c609d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 6a7f514f83f8..20a29a5a1363 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11940,8 +11940,8 @@ Diffed against: <https://developer.systeme.io/llms.txt>- [ ] `Email statistics (GET /api/mailing/statistics/newsletters; GET /api/mailing/statistics/campaign-steps; GET /api/mailing/statistics/automation-emails)` — compare messages sent, opens, clicks, bounces, and spam reports across email types (high)- [x] `Digital products (GET /api/payment/digital-products)` — analyze the digital product catalog alongside courses and subscription offerings (medium)- [x] `Physical products (GET /api/payment/physical-products)` — analyze physical product offerings and compare them with digital products (medium)-- [ ] `Campaigns (GET /api/mailing/campaigns)` — analyze automated email campaigns alongside newsletters (medium)-- [ ] `Campaign steps (GET /api/mailing/campaigns/{campaignId}/steps)` — connect email performance to each message and its position in a campaign (medium)+- [x] `Campaigns (GET /api/mailing/campaigns)` — analyze automated email campaigns alongside newsletters (medium)+- [x] `Campaign steps (GET /api/mailing/campaigns/{campaignId}/steps)` — connect email performance to each message and its position in a campaign (medium)- [ ] `Funnels (GET /api/funnels)` — identify marketing funnels for comparisons with contacts and their source URLs (medium)- [ ] `Funnel steps (GET /api/funnels/{funnelId}/steps)` — describe funnel pages and their sequence for acquisition analysis (medium)- [ ] `Bookings (GET /api/booking-calendar/bookings)` — analyze appointment demand and booking activity alongside contact growth (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.pyindex 52df9c5235cc..1cf7b8ef2aa4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/canonical_descriptions.py@@ -110,4 +110,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"stock": "Number of units in stock when the stock is limited.",},},+ "campaigns": {+ "description": "Automated email campaigns that send a sequence of steps to contacts.",+ "docs_url": "https://developer.systeme.io/reference/api_mailingcampaigns_get_collection",+ "columns": {+ "id": "Unique campaign identifier.",+ "name": "Campaign name.",+ "senderEmail": "Email address the campaign sends from.",+ },+ },+ "campaign_steps": {+ "description": "Email steps of each campaign, with content metadata and position in the sequence.",+ "docs_url": "https://developer.systeme.io/reference/api_mailingcampaigns_campaignidsteps_get_collection",+ "columns": {+ "id": "Unique campaign step identifier.",+ "campaignId": "Identifier of the campaign that contains the step.",+ "content": "Step email subject, preview text, editor type, and sender details.",+ "state": "Whether the step is active, and its position in the campaign.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.pyindex 7f6ba672d57e..5acf0e88c32e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/settings.py@@ -13,8 +13,13 @@ ENDPOINTS = {"memberships": "community/memberships","digital_products": "payment/digital-products","physical_products": "payment/physical-products",+ "campaigns": "mailing/campaigns",+ "campaign_steps": "mailing/campaigns/{campaignId}/steps",}+# Child endpoint -> parent endpoint whose `id` fills the child path's `{campaignId}`.+FANOUT_PARENTS = {"campaign_steps": "campaigns"}+INCREMENTAL_FIELDS = {"contacts": [incremental_field("registeredAt")]}PARTITION_KEYS = {"contacts": "registeredAt",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.pyindex c809b06bfaed..2ce486a9c271 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/systeme.py@@ -6,16 +6,19 @@ from requests import HTTPError, Responsefrom posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (+ ClientConfig,Endpoint,EndpointResource,RESTAPIConfig,rest_api_resource,+ rest_api_resources,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuthfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponseCursorPaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClientfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource@@ -24,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.systeme.seAUTH_ERROR,BASE_URL,ENDPOINTS,+ FANOUT_PARENTS,PAGE_SIZE,PARTITION_KEYS,PERMISSION_ERROR,@@ -34,6 +38,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.systeme.seclass SystemeResumeConfig:cursor: int | None = Nonecompleted: bool = False+ fanout_state: dict[str, Any] | None = Noneclass SystemePaginator(JSONResponseCursorPaginator):@@ -54,7 +59,8 @@ class SystemePaginator(JSONResponseCursorPaginator):def validate_credentials(api_key: str, schema_name: str | None = None) -> tuple[bool, str | None]:- path = schema_for_resource(ENDPOINTS, schema_name or "contacts")+ resource_name = schema_name or "contacts"+ path = schema_for_resource(ENDPOINTS, FANOUT_PARENTS.get(resource_name, resource_name))client = RESTClient(base_url=BASE_URL,auth=APIKeyAuth(api_key=api_key, name="X-API-Key"),@@ -73,6 +79,64 @@ def validate_credentials(api_key: str, schema_name: str | None = None) -> tuple[return True, None+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": BASE_URL,+ "auth": {"type": "api_key", "name": "X-API-Key", "api_key": api_key, "location": "header"},+ "paginator": SystemePaginator(),+ "request_timeout": 30,+ }+++def _fanout_resource(+ api_key: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[SystemeResumeConfig],+) -> Resource:+ parent = FANOUT_PARENTS[endpoint]+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state:+ resumable_source_manager.save_state(SystemeResumeConfig(fanout_state=state))++ config: RESTAPIConfig = {+ "client": _client_config(api_key),+ "resources": [+ {+ "name": parent,+ "endpoint": {+ "path": schema_for_resource(ENDPOINTS, parent),+ "params": {"limit": PAGE_SIZE, "order": "desc"},+ "data_selector": "items",+ "data_selector_required": True,+ },+ },+ {+ "name": endpoint,+ "endpoint": {+ # The child collection documents no limit, cursor or order parameter.+ "path": schema_for_resource(ENDPOINTS, endpoint),+ "params": {"campaignId": {"type": "resolve", "resource": parent, "field": "id"}},+ "data_selector": "items",+ "data_selector_required": True,+ },+ },+ ],+ }+ resources = rest_api_resources(+ config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=resume.fanout_state if resume else None,+ )+ return next(resource for resource in resources if resource.name == endpoint)++def systeme_source(api_key: str,endpoint: str,@@ -83,6 +147,10 @@ def systeme_source(db_incremental_field_last_value: datetime | str | None = None,) -> SourceResponse:path = schema_for_resource(ENDPOINTS, endpoint)+ if endpoint in FANOUT_PARENTS:+ fanout_resource = _fanout_resource(api_key, endpoint, team_id, job_id, resumable_source_manager)+ return SourceResponse(name=endpoint, items=lambda: fanout_resource, primary_keys=["id"])+params: dict[str, Any] = {"limit": PAGE_SIZE, "order": "desc"}if endpoint == "contacts" and should_use_incremental_field and db_incremental_field_last_value is not None:params["registeredAfter"] = (@@ -106,12 +174,7 @@ def systeme_source(}resource_config: EndpointResource = {"name": endpoint, "endpoint": endpoint_config}config: RESTAPIConfig = {- "client": {- "base_url": BASE_URL,- "auth": {"type": "api_key", "name": "X-API-Key", "api_key": api_key, "location": "header"},- "paginator": SystemePaginator(),- "request_timeout": 30,- },+ "client": _client_config(api_key),"resources": [resource_config],}resource = rest_api_resource(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.pyindex 68b8f81b56f3..01ae2411f090 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/systeme/tests/test_systeme.py@@ -15,6 +15,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse@@ -60,6 +61,74 @@ def test_incremental_filter_and_full_refresh(assert result.params("registeredAfter") == [expected, expected]+CAMPAIGN_20_STEPS = "/api/mailing/campaigns/20/steps"+CAMPAIGN_10_STEPS = "/api/mailing/campaigns/10/steps"+STEP_202 = {"id": 202, "campaignId": 20}+STEP_201 = {"id": 201, "campaignId": 20}+STEP_101 = {"id": 101, "campaignId": 10}+++ "fanout_state,steps_20_pages,expected_rows,expected_step_cursors",+ [+ (None, [[STEP_202], [STEP_201]], [STEP_202, STEP_201, STEP_101], [None, "202", None]),+ (+ {"completed": ["mailing/campaigns/20/steps"], "current": None, "child_state": None},+ [],+ [STEP_101],+ [None],+ ),+ (+ {"completed": [], "current": "mailing/campaigns/20/steps", "child_state": {"cursor": 202}},+ [[STEP_201]],+ [STEP_201, STEP_101],+ ["202", None],+ ),+ ],+)+def test_campaign_steps_fan_out_and_resume(+ fanout_state: dict[str, Any] | None,+ steps_20_pages: list[list[dict[str, Any]]],+ expected_rows: list[dict[str, Any]],+ expected_step_cursors: list[str | None],+) -> None:+ steps_20 = [+ ScriptedResponse(json={"items": page, "hasMore": index < len(steps_20_pages) - 1})+ for index, page in enumerate(steps_20_pages)+ ]+ result = driver().run(+ "campaign_steps",+ route(+ {+ "/api/mailing/campaigns": [+ ScriptedResponse(json={"items": [{"id": 20, "name": "Welcome"}], "hasMore": True}),+ ScriptedResponse(json={"items": [{"id": 10, "name": "Onboarding"}], "hasMore": False}),+ ],+ CAMPAIGN_20_STEPS: steps_20,+ CAMPAIGN_10_STEPS: [ScriptedResponse(json={"items": [STEP_101], "hasMore": False})],+ }+ ),+ resume_state=SystemeResumeConfig(fanout_state=fanout_state) if fanout_state else None,+ )+ assert result.raised is None+ assert result.rows == expected_rows+ campaign_queries = [dict(request.query) for request in result.requests if request.path == "/api/mailing/campaigns"]+ assert campaign_queries == [+ {"limit": ("100",), "order": ("desc",)},+ {"limit": ("100",), "order": ("desc",), "startingAfter": ("20",)},+ ]+ step_requests = [request for request in result.requests if request.path.endswith("/steps")]+ assert [request.param("startingAfter") for request in step_requests] == expected_step_cursors+ assert [request.param("limit") for request in step_requests] == [None] * len(step_requests)+ assert result.final_state == SystemeResumeConfig(+ fanout_state={+ "completed": ["mailing/campaigns/10/steps", "mailing/campaigns/20/steps"],+ "current": None,+ "child_state": None,+ }+ )++@pytest.mark.parametrize("body,message",[@@ -92,6 +161,7 @@ def test_invalid_or_repeated_cursor_fails(body: dict[str, Any], message: str) ->("newsletters", "mailing/newsletters"),("digital_products", "payment/digital-products"),("physical_products", "payment/physical-products"),+ ("campaign_steps", "mailing/campaigns"),],)def test_validate_credentials_status_and_probe(