Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add retently score, trend and suppression tables (#127359)

Tom Owers57f0e5d2e15d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index f4504c0e67ea..0f968278559a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10274,9 +10274,9 @@ Today (7): `campaigns`, `companies`, `customers`, `feedback`, `outbox`, `reports
Diffed against: <https://www.retently.com/api/>
-- [ ] `GET /api/v2/nps/score, /csat/score, /ces/score, /star/score` — the vendor's headline metric - the aggregate NPS/CSAT/CES/star score the whole product exists to report (high)
-- [ ] `GET /api/v2/trends and /api/v2/trends/{groupId}` — trend groups and their trends are the topic taxonomy behind feedback; a lookup that turns free-text feedback into a breakdown dimension (high)
-- [ ] `GET /api/v2/suppressions/emails and /suppressions/domains` — suppressed recipients explain gaps in outbox delivery coverage (low)
+- [x] `GET /api/v2/nps/score, /csat/score, /ces/score, /star/score` — the vendor's headline metric - the aggregate NPS/CSAT/CES/star score the whole product exists to report (high)
+- [x] `GET /api/v2/trends and /api/v2/trends/{groupId}` — trend groups and their trends are the topic taxonomy behind feedback; a lookup that turns free-text feedback into a breakdown dimension (high)
+- [x] `GET /api/v2/suppressions/emails and /suppressions/domains` — suppressed recipients explain gaps in outbox delivery coverage (low)
## Reverb — gaps
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py
index 96f7dd725828..2f9c24578407 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py
@@ -126,4 +126,89 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"questionsStats": "Aggregated statistics for each survey question.",
},
},
+ "nps_score": {
+ "description": "The latest NPS score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "ratingDistribution": "The number of responses for each rating value.",
+ "totalResponses": "The total number of responses the score is based on.",
+ },
+ },
+ "csat_score": {
+ "description": "The latest CSAT score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "ratingDistribution": "The number of responses for each rating value.",
+ "totalResponses": "The total number of responses the score is based on.",
+ },
+ },
+ "ces_score": {
+ "description": "The latest CES score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "ratingDistribution": "The number of responses for each rating value.",
+ "totalResponses": "The total number of responses the score is based on.",
+ },
+ },
+ "star_score": {
+ "description": "The latest star rating score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "ratingDistribution": "The number of responses for each rating value.",
+ "totalResponses": "The total number of responses the score is based on.",
+ },
+ },
+ "trend_groups": {
+ "description": "A trend group: a named set of trends that segment survey scores for one metric.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "_id": "The unique ID of the trend group.",
+ "groupName": "The trend group's name.",
+ "metric": "The survey metric the group's trends measure (NPS, CSAT, CES, STAR).",
+ "isDefault": "Whether this is the account's default trend group.",
+ "trendsCount": "The number of trends in the group.",
+ "createdDate": "When the trend group was created.",
+ "updatedDate": "When the trend group was last updated.",
+ },
+ },
+ "trends": {
+ "description": "A trend within a trend group: a segment of responses (by campaign and customer attributes) with its score over the vendor's default date range.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "_id": "The unique ID of the trend.",
+ "groupId": "The ID of the trend group the trend belongs to.",
+ "trendName": "The trend's name.",
+ "metric": "The survey metric the trend measures (NPS, CSAT, CES, STAR).",
+ "score": "The current and previous score for the date range, with the absolute and percentage change.",
+ "respondents": "Respondent counts, percentages and rating distribution for the date range.",
+ "campaigns": "The IDs of the campaigns the trend includes.",
+ "attributes": "The customer attribute filters that define the trend's segment.",
+ "trend": "The trend line: the score, counts and rating distribution for each date in the range.",
+ },
+ },
+ "suppressed_emails": {
+ "description": "An email address on the account's suppression list. Suppressed emails receive no surveys.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "id": "The unique ID of the suppression.",
+ "email": "The suppressed email address.",
+ "reason": "Why the email is suppressed (for example manual or complaint).",
+ "note": "A note explaining the suppression.",
+ "addedBy": "Who added the suppression (api, user or system).",
+ "createdAt": "When the email was suppressed.",
+ },
+ },
+ "suppressed_domains": {
+ "description": "A domain pattern on the account's suppression list. Emails that match it receive no surveys.",
+ "docs_url": "https://www.retently.com/api/",
+ "columns": {
+ "id": "The unique ID of the suppression.",
+ "pattern": "The suppressed domain pattern. Supports wildcards such as *.example.com.",
+ "category": "The pattern's category (other, corporate, disposable or invalid).",
+ "reason": "Why the domain is suppressed.",
+ "note": "A note explaining the suppression.",
+ "addedBy": "Who added the suppression (api, user or system).",
+ "createdAt": "When the domain was suppressed.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py
index 4e19c2725c15..8b1cdf6eb607 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py
@@ -1,6 +1,6 @@
import dataclasses
from datetime import UTC, date, datetime
-from typing import Any, Optional
+from typing import Any, Optional, cast
from requests import Request, Response
@@ -9,20 +9,28 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
BasePaginator,
SinglePagePaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ Endpoint,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.retently.settings import RETENTLY_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.retently.settings import (
+ PAGE_SIZE,
+ RETENTLY_ENDPOINTS,
+ RetentlyEndpointConfig,
+)
RETENTLY_BASE_URL = "https://app.retently.com/api/v2"
-# Documented maximum page size; the largest page minimises round trips against the ~150 req/min
-# rate limit.
-PAGE_SIZE = 1000
FIRST_PAGE = 1
@@ -115,6 +123,45 @@ class RetentlyPaginator(BasePaginator):
return f"RetentlyPaginator(page={self.page})"
+def _client_config(api_key: str) -> ClientConfig:
+ return {
+ "base_url": RETENTLY_BASE_URL,
+ "headers": _non_secret_headers(),
+ "auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},
+ }
+
+
+def _trends_resource(api_key: str, config: RetentlyEndpointConfig, team_id: int, job_id: str) -> Resource:
+ assert config.fanout is not None
+ parent = RETENTLY_ENDPOINTS[config.fanout.parent_name]
+ # Trend groups and the trends of a group are unpaginated, so a full refresh re-fetches the
+ # handful of groups instead of resuming mid fan-out.
+ return cast(
+ Resource,
+ build_dependent_resource(
+ endpoint_configs={config.name: config, parent.name: parent},
+ child_endpoint=config.name,
+ fanout=config.fanout,
+ client_config=_client_config(api_key),
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ page_size_param=None,
+ parent_endpoint_extra={
+ "paginator": SinglePagePaginator(),
+ "data_selector": parent.data_selector,
+ "data_selector_malformed_retryable": True,
+ },
+ child_endpoint_extra={
+ "paginator": SinglePagePaginator(),
+ "data_selector": config.data_selector,
+ "data_selector_malformed_retryable": True,
+ },
+ ),
+ )
+
+
def retently_source(
api_key: str,
endpoint: str,
@@ -127,6 +174,15 @@ def retently_source(
config = RETENTLY_ENDPOINTS[endpoint]
supports_incremental = bool(config.incremental_fields)
+ if config.fanout is not None:
+ trends = _trends_resource(api_key, config, team_id, job_id)
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: trends,
+ primary_keys=config.primary_keys,
+ column_hints=trends.column_hints,
+ )
+
params: dict[str, Any] = {}
paginator: BasePaginator
if config.paginated:
@@ -135,8 +191,8 @@ def retently_source(
params["sort"] = config.sort_param
paginator = RetentlyPaginator()
else:
- # Campaigns, templates and reports are documented without pagination and return the full
- # collection in one response — no page/limit/sort params.
+ # Campaigns, templates, reports, scores and suppressions are documented without pagination
+ # and return the full collection in one response — no page/limit/sort params.
paginator = SinglePagePaginator()
endpoint_config: Endpoint = {
@@ -145,8 +201,9 @@ def retently_source(
"data_selector": config.data_selector,
# A 200 whose payload isn't the expected list shape (an HTML proxy error, a transient
# error envelope) is retried, matching the old transport which raised a retryable error
- # from its extraction step so one malformed page reissues instead of failing the sync.
- "data_selector_malformed_retryable": True,
+ # from its extraction step so one malformed page reissues instead of failing the sync. The
+ # score endpoints legitimately return one object, which this list-shape check would reject.
+ "data_selector_malformed_retryable": not config.single_object,
"paginator": paginator,
}
@@ -161,11 +218,7 @@ def retently_source(
}
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": RETENTLY_BASE_URL,
- "headers": _non_secret_headers(),
- "auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},
- },
+ "client": _client_config(api_key),
"resource_defaults": {},
"resources": [{"name": endpoint, "endpoint": endpoint_config}],
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py
index 75b2cef8438d..ffcd164981da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py
@@ -1,10 +1,17 @@
from dataclasses import dataclass, field
from typing import Optional
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# Documented maximum page size; the largest page minimises round trips against the ~150 req/min
+# rate limit.
+PAGE_SIZE = 1000
-@dataclass
+
+@dataclass(frozen=True)
class RetentlyEndpointConfig:
name: str
path: str
@@ -23,6 +30,11 @@ class RetentlyEndpointConfig:
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Stable creation-time field used for datetime partitioning. Never an updated_at-style field.
partition_key: Optional[str] = None
+ # The /{metric}/score endpoints return one object under `data`, not a list of rows.
+ single_object: bool = False
+ fanout: Optional[DependentEndpointConfig] = None
+ default_incremental_field: Optional[str] = None
+ page_size: int = PAGE_SIZE
def _created_date_incremental_field() -> list[IncrementalField]:
@@ -42,7 +54,9 @@ def _created_date_incremental_field() -> list[IncrementalField]:
# specified and customer records are mutable (properties, tags), so a creation-window incremental
# would silently miss updates — customers stays full refresh. Companies expose no date filter, and
# outbox rows carry no consistently documented unique identifier (`surveyId` is absent from some
-# example rows), so there is no safe merge key and outbox stays full refresh too.
+# example rows), so there is no safe merge key and outbox stays full refresh too. The score, trend
+# and suppression endpoints take no date filter or pagination and return the full collection (or a
+# point-in-time snapshot) in one response, so they are full refresh as well.
RETENTLY_ENDPOINTS: dict[str, RetentlyEndpointConfig] = {
"customers": RetentlyEndpointConfig(
name="customers",
@@ -97,6 +111,55 @@ RETENTLY_ENDPOINTS: dict[str, RetentlyEndpointConfig] = {
primary_keys=["campaignId"],
paginated=False,
),
+ # One table per metric: each metric's score payload has its own keys (e.g. `nps`, `promoters`).
+ **{
+ f"{metric}_score": RetentlyEndpointConfig(
+ name=f"{metric}_score",
+ path=f"/{metric}/score",
+ data_selector="data",
+ paginated=False,
+ single_object=True,
+ )
+ for metric in ("nps", "csat", "ces", "star")
+ },
+ "trend_groups": RetentlyEndpointConfig(
+ name="trend_groups",
+ path="/trends",
+ data_selector="groups",
+ primary_keys=["_id"],
+ paginated=False,
+ ),
+ "trends": RetentlyEndpointConfig(
+ name="trends",
+ path="/trends/{groupId}",
+ data_selector="trends",
+ primary_keys=["groupId", "_id"],
+ paginated=False,
+ fanout=DependentEndpointConfig(
+ parent_name="trend_groups",
+ resolve_param="groupId",
+ resolve_field="_id",
+ include_from_parent=["_id"],
+ parent_field_renames={"_id": "groupId"},
+ # A group deleted between the group listing and this fetch 404s; skip it instead of
+ # failing the whole fan-out.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
+ "suppressed_emails": RetentlyEndpointConfig(
+ name="suppressed_emails",
+ path="/suppressions/emails",
+ data_selector="data",
+ primary_keys=["id"],
+ paginated=False,
+ ),
+ "suppressed_domains": RetentlyEndpointConfig(
+ name="suppressed_domains",
+ path="/suppressions/domains",
+ data_selector="data",
+ primary_keys=["id"],
+ paginated=False,
+ ),
}
ENDPOINTS = tuple(RETENTLY_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py
index 7dd9f1f0bb45..a8e5246764af 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py
@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.retently import (
@@ -56,6 +57,13 @@ class TestEnvelopeShapes:
("bare_list_under_data", "reports", {"data": [{"campaignId": "c1"}]}, [{"campaignId": "c1"}]),
# Campaigns/templates document the array at the top level, outside `data`.
("top_level_array", "campaigns", {"campaigns": [{"id": "c1"}]}, [{"id": "c1"}]),
+ # The score endpoints return one object under `data`, synced as a single row.
+ (
+ "single_object_under_data",
+ "nps_score",
+ {"message": "Success.", "code": 200, "data": {"nps": 39, "totalResponses": 13}},
+ [{"nps": 39, "totalResponses": 13}],
+ ),
]
)
def test_documented_envelope_shapes_yield_rows(self, _name: str, endpoint: str, body: Any, expected: list) -> None:
@@ -113,6 +121,29 @@ class TestPagination:
assert "limit" not in result.requests[0].query
+class TestTrendsFanout:
+ def test_trends_are_fetched_per_group_and_carry_the_group_id(self) -> None:
+ result = _run(
+ "trends",
+ route(
+ {
+ "/trends": [_response({"success": True, "groups": [{"_id": "g1"}, {"_id": "g2"}]})],
+ # A 200 error envelope without `trends` is retried, not synced as an empty group.
+ "/trends/g1": [
+ _response({"success": False, "error": "Internal Server Error"}),
+ _response({"success": True, "trends": [{"_id": "t1", "trendName": "Enterprise"}]}),
+ ],
+ # A group deleted between the listing and its fetch must not fail the sync.
+ "/trends/g2": [_response({"success": False}, status_code=404)],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.paths == ["/api/v2/trends", "/api/v2/trends/g1", "/api/v2/trends/g1", "/api/v2/trends/g2"]
+ assert [(row["groupId"], row["_id"]) for row in result.rows] == [("g1", "t1")]
+
+
class TestIncremental:
def test_start_date_sent_when_incremental(self) -> None:
result = _run(