Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add loops workflow_node_metrics table (#119536)

Tom Owers44374a370675
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 2f36dc90433b..e7fd8d401c00 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7491,13 +7491,14 @@ Note: Fetched llms.txt, then parsed the linked Returns, Orders, Products, Custom
## Loops — gaps
-Today (13): `audience_segments`, `campaign_groups`, `campaign_metrics`, `campaigns`, `components`, `contact_properties`, `event_patterns`, `mailing_lists`, `themes`, `transactional_email_metrics`, `transactional_emails`, `transactional_groups`, `workflows`
+Today (14): `audience_segments`, `campaign_groups`, `campaign_metrics`, `campaigns`, `components`, `contact_properties`, `event_patterns`, `mailing_lists`, `themes`, `transactional_email_metrics`, `transactional_emails`, `transactional_groups`, `workflow_node_metrics`, `workflows`
Diffed against: <https://loops.so/docs/openapi.json>
- [x] `GET /v1/event-patterns` — lookup of event definitions that workflows and campaigns trigger on (`event_patterns`)
- [x] `GET /v1/campaigns/{campaignId}/metrics` — all-time engagement counters per sent campaign (`campaign_metrics`)
- [x] `GET /v1/transactional-emails/{transactionalId}/metrics` — all-time delivery counters per transactional email (`transactional_email_metrics`)
+- [x] `GET /v1/workflows/{workflowId}/nodes/{nodeId}/metrics` — all-time engagement counters per workflow email node (`workflow_node_metrics`)
Note: Coverage is close to complete — the spec has ~20 listable GET collections and 10 are already exposed. Loops has no bulk contacts list endpoint (only /v1/contacts/find by email or userId), so a contacts table is not implementable against this API; /v1/email-messages/{id} is likewise single-fetch only, and /v1/contacts/suppression only checks one contact by email or userId rather than listing suppressed contacts. Remaining unexposed paths are config/plumbing (api-key, dedicated-sending-ips, uploads) or per-node workflow mutations.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/canonical_descriptions.py
index 3a80b39c05de..3e3d940cbb6e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/canonical_descriptions.py
@@ -153,4 +153,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"softBounces": "Number of sends that soft bounced.",
},
},
+ "workflow_node_metrics": {
+ "description": "All-time email engagement counters for each email node in a workflow. Opens and clicks count sends, not events.",
+ "docs_url": "https://loops.so/docs/api-reference/get-workflow-node-metrics",
+ "columns": {
+ "workflowId": "The ID of the workflow the email node belongs to.",
+ "nodeId": "The ID of the SendEmailAction node the metrics belong to.",
+ "emailMessageId": "The ID of the email message the node sends.",
+ "subject": "The subject of the email message the node sends.",
+ "sends": "Number of sends.",
+ "opens": "Number of sends that were opened at least once.",
+ "clicks": "Number of sends where at least one link was clicked.",
+ "unsubscribes": "Number of unsubscribes.",
+ "spamReports": "Number of sends reported as spam.",
+ "hardBounces": "Number of sends that hard bounced.",
+ "softBounces": "Number of sends that soft bounced.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/loops.py b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/loops.py
index 7143f2054a85..924643e41916 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/loops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/loops.py
@@ -1,4 +1,5 @@
import dataclasses
+from collections.abc import Iterator
from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -6,6 +7,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (
+ create_response_hooks,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
DependentEndpointConfig,
build_dependent_resource,
@@ -14,10 +19,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
JSONResponseCursorPaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
Endpoint,
EndpointResource,
+ ResponseAction,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -29,6 +36,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.loops.sett
BASE_URL = "https://app.loops.so/api"
+SEND_EMAIL_NODE_TYPE = "SendEmailAction"
+
+# Loops answers 400 for a node with no email message, and 404 for a workflow or node deleted after
+# the parent listing. Neither has metrics.
+_WORKFLOW_NODE_RESPONSE_ACTIONS: list[ResponseAction] = [
+ {"status_code": 400, "action": "ignore"},
+ {"status_code": 404, "action": "ignore"},
+]
+
@dataclasses.dataclass
class LoopsResumeConfig:
@@ -110,12 +126,55 @@ def _fanout_source_response(
return SourceResponse(
name=endpoint_config.name,
items=lambda: resource,
- primary_keys=[endpoint_config.primary_key],
+ primary_keys=list(endpoint_config.primary_keys),
partition_count=1,
partition_size=1,
)
+def _iter_workflow_node_metrics(
+ client: RESTClient,
+ endpoint_config: LoopsEndpointConfig,
+ resumable_source_manager: ResumableSourceManager[LoopsResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ hooks = create_response_hooks(_WORKFLOW_NODE_RESPONSE_ACTIONS, resource_name=endpoint_config.name)
+
+ for workflows in client.paginate(
+ path=LOOPS_ENDPOINTS["workflows"].path,
+ params={"perPage": PAGE_SIZE},
+ paginator=JSONResponseCursorPaginator(cursor_path="pagination.nextCursor", cursor_param="cursor"),
+ data_selector="data",
+ ):
+ for summary in workflows:
+ workflow_id = summary["id"]
+ for details in client.paginate(
+ path=f"/v1/workflows/{workflow_id}", paginator=SinglePagePaginator(), data_selector="$", hooks=hooks
+ ):
+ for workflow in details:
+ for node_id, node in (workflow.get("nodes") or {}).items():
+ if node.get("typeName") != SEND_EMAIL_NODE_TYPE:
+ continue
+ for metrics in client.paginate(
+ path=endpoint_config.path.format(workflowId=workflow_id, nodeId=node_id),
+ paginator=SinglePagePaginator(),
+ data_selector="$",
+ hooks=hooks,
+ ):
+ yield [
+ {
+ **row,
+ "workflowId": workflow_id,
+ "nodeId": node_id,
+ "emailMessageId": node.get("emailMessageId"),
+ "subject": node.get("subject"),
+ }
+ for row in metrics
+ ]
+ # Many nodes return no metrics, so a long run of requests can pass without a yield.
+ resumable_source_manager.safe_point()
+ resumable_source_manager.safe_point()
+
+
def loops_source(
api_key: str,
endpoint: str,
@@ -128,6 +187,21 @@ def loops_source(
if endpoint_config.fanout is not None:
return _fanout_source_response(api_key, endpoint_config, endpoint_config.fanout, team_id, job_id)
+ if endpoint_config.per_workflow_node:
+ client = RESTClient(
+ base_url=BASE_URL,
+ headers={"Accept": "application/json"},
+ auth=BearerTokenAuth(token=api_key),
+ request_timeout=(10, 60),
+ )
+ return SourceResponse(
+ name=endpoint_config.name,
+ items=lambda: _iter_workflow_node_metrics(client, endpoint_config, resumable_source_manager),
+ primary_keys=list(endpoint_config.primary_keys),
+ partition_count=1,
+ partition_size=1,
+ )
+
rest_config: RESTAPIConfig = {
"client": _client_config(api_key),
"resource_defaults": {
@@ -161,7 +235,7 @@ def loops_source(
return SourceResponse(
name=endpoint,
items=lambda: resource,
- primary_keys=[endpoint_config.primary_key],
+ primary_keys=list(endpoint_config.primary_keys),
partition_count=1,
partition_size=1,
partition_mode="datetime" if endpoint_config.partition_key else None,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/settings.py
index d0baab02001c..ba163a3d17e3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/settings.py
@@ -16,7 +16,7 @@ class LoopsEndpointConfig:
name: str
# Path under https://app.loops.so/api (e.g. "/v1/campaigns").
path: str
- primary_key: str = "id"
+ primary_keys: tuple[str, ...] = ("id",)
# Stable created-date field to partition by, or None when the payload has no
# creation timestamp (mailing lists, contact properties, components).
partition_key: Optional[str] = None
@@ -28,6 +28,9 @@ class LoopsEndpointConfig:
extra_params: dict[str, str] = field(default_factory=dict)
# Set for per-object endpoints fetched once per row of a parent list endpoint.
fanout: Optional[DependentEndpointConfig] = None
+ # Set for per-node endpoints. Node ids come only from each workflow's detail, so the fan-out
+ # is two levels deep (workflows list -> workflow -> node) and runs through a custom iterator.
+ per_workflow_node: bool = False
# Unused by Loops' full-refresh endpoints; kept so the config satisfies FanoutEndpointLike.
incremental_fields: list[IncrementalField] = field(default_factory=list)
default_incremental_field: Optional[str] = None
@@ -73,7 +76,7 @@ LOOPS_ENDPOINTS: dict[str, LoopsEndpointConfig] = {
"contact_properties": LoopsEndpointConfig(
name="contact_properties",
path="/v1/contacts/properties",
- primary_key="key",
+ primary_keys=("key",),
paginated=False,
extra_params={"list": "all"},
),
@@ -94,7 +97,7 @@ LOOPS_ENDPOINTS: dict[str, LoopsEndpointConfig] = {
"campaign_metrics": LoopsEndpointConfig(
name="campaign_metrics",
path="/v1/campaigns/{campaignId}/metrics",
- primary_key="campaignId",
+ primary_keys=("campaignId",),
paginated=False,
fanout=DependentEndpointConfig(
parent_name="campaigns",
@@ -114,7 +117,7 @@ LOOPS_ENDPOINTS: dict[str, LoopsEndpointConfig] = {
"transactional_email_metrics": LoopsEndpointConfig(
name="transactional_email_metrics",
path="/v1/transactional-emails/{transactionalId}/metrics",
- primary_key="transactionalId",
+ primary_keys=("transactionalId",),
paginated=False,
fanout=DependentEndpointConfig(
parent_name="transactional_emails",
@@ -126,6 +129,13 @@ LOOPS_ENDPOINTS: dict[str, LoopsEndpointConfig] = {
child_response_actions=[{"status_code": 404, "action": "ignore"}],
),
),
+ "workflow_node_metrics": LoopsEndpointConfig(
+ name="workflow_node_metrics",
+ path="/v1/workflows/{workflowId}/nodes/{nodeId}/metrics",
+ primary_keys=("workflowId", "nodeId"),
+ paginated=False,
+ per_workflow_node=True,
+ ),
}
ENDPOINTS = tuple(LOOPS_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/tests/test_loops.py b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/tests/test_loops.py
index dfcc4bc0cca2..2e6606d8691a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/loops/tests/test_loops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/loops/tests/test_loops.py
@@ -119,10 +119,54 @@ class TestLoopsSource:
assert all(request.origin == "https://app.loops.so" for request in result.requests)
assert result.saved_states == []
+ def test_workflow_node_metrics_fan_out_over_email_nodes(self) -> None:
+ email_node = {"typeName": "SendEmailAction", "nextNodeIds": [], "emailMessageId": "m1", "subject": "Hi"}
+ result = self._driver().run(
+ "workflow_node_metrics",
+ [
+ _page([{"id": "w1"}, {"id": "w2"}], next_cursor="cursor-1"),
+ ScriptedResponse(
+ json={
+ "id": "w1",
+ "nodes": {
+ "t1": {"typeName": "SignupTrigger", "nextNodeIds": ["n1"]},
+ "n1": email_node,
+ "n2": {**email_node, "emailMessageId": None},
+ },
+ }
+ ),
+ ScriptedResponse(json={"sends": 10, "opens": 4}),
+ # A node with no email message answers 400.
+ ScriptedResponse(status=400, json={"message": "Node has no email message."}),
+ # A workflow deleted after the listing answers 404.
+ ScriptedResponse(status=404, json={"message": "Workflow not found."}),
+ _page([{"id": "w3"}], next_cursor=None),
+ ScriptedResponse(json={"id": "w3", "nodes": {"n3": email_node}}),
+ ScriptedResponse(json={"sends": 2, "opens": 1}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {"sends": 10, "opens": 4, "workflowId": "w1", "nodeId": "n1", "emailMessageId": "m1", "subject": "Hi"},
+ {"sends": 2, "opens": 1, "workflowId": "w3", "nodeId": "n3", "emailMessageId": "m1", "subject": "Hi"},
+ ]
+ assert result.paths == [
+ "/api/v1/workflows",
+ "/api/v1/workflows/w1",
+ "/api/v1/workflows/w1/nodes/n1/metrics",
+ "/api/v1/workflows/w1/nodes/n2/metrics",
+ "/api/v1/workflows/w2",
+ "/api/v1/workflows",
+ "/api/v1/workflows/w3",
+ "/api/v1/workflows/w3/nodes/n3/metrics",
+ ]
+ assert result.params("cursor")[5] == "cursor-1"
+
@pytest.mark.parametrize("endpoint", list(LOOPS_ENDPOINTS.keys()))
def test_source_response_primary_and_partition_keys(self, endpoint: str) -> None:
endpoint_config = LOOPS_ENDPOINTS[endpoint]
- if endpoint_config.paginated or endpoint_config.fanout:
+ if endpoint_config.paginated or endpoint_config.fanout or endpoint_config.per_workflow_node:
answer = _page([], next_cursor=None)
else:
answer = ScriptedResponse(json=[])
@@ -131,7 +175,7 @@ class TestLoopsSource:
assert result.raised is None
assert result.response is not None
assert result.response.name == endpoint
- assert result.response.primary_keys == [endpoint_config.primary_key]
+ assert result.response.primary_keys == list(endpoint_config.primary_keys)
if endpoint_config.partition_key:
assert result.response.partition_keys == [endpoint_config.partition_key]
assert result.response.partition_mode == "datetime"