:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add retently score, trend and suppression tables (#127359)
Tom Owers57f0e5d2e15d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex f4504c0e67ea..0f968278559a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10274,9 +10274,9 @@ Today (7): `campaigns`, `companies`, `customers`, `feedback`, `outbox`, `reportsDiffed against: <https://www.retently.com/api/>-- [ ] `GET /api/v2/nps/score, /csat/score, /ces/score, /star/score` — the vendor's headline metric - the aggregate NPS/CSAT/CES/star score the whole product exists to report (high)-- [ ] `GET /api/v2/trends and /api/v2/trends/{groupId}` — trend groups and their trends are the topic taxonomy behind feedback; a lookup that turns free-text feedback into a breakdown dimension (high)-- [ ] `GET /api/v2/suppressions/emails and /suppressions/domains` — suppressed recipients explain gaps in outbox delivery coverage (low)+- [x] `GET /api/v2/nps/score, /csat/score, /ces/score, /star/score` — the vendor's headline metric - the aggregate NPS/CSAT/CES/star score the whole product exists to report (high)+- [x] `GET /api/v2/trends and /api/v2/trends/{groupId}` — trend groups and their trends are the topic taxonomy behind feedback; a lookup that turns free-text feedback into a breakdown dimension (high)+- [x] `GET /api/v2/suppressions/emails and /suppressions/domains` — suppressed recipients explain gaps in outbox delivery coverage (low)## Reverb — gapsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.pyindex 96f7dd725828..2f9c24578407 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/canonical_descriptions.py@@ -126,4 +126,89 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"questionsStats": "Aggregated statistics for each survey question.",},},+ "nps_score": {+ "description": "The latest NPS score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "ratingDistribution": "The number of responses for each rating value.",+ "totalResponses": "The total number of responses the score is based on.",+ },+ },+ "csat_score": {+ "description": "The latest CSAT score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "ratingDistribution": "The number of responses for each rating value.",+ "totalResponses": "The total number of responses the score is based on.",+ },+ },+ "ces_score": {+ "description": "The latest CES score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "ratingDistribution": "The number of responses for each rating value.",+ "totalResponses": "The total number of responses the score is based on.",+ },+ },+ "star_score": {+ "description": "The latest star rating score for your account, with its rating distribution and response counts. One row, refreshed on each sync.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "ratingDistribution": "The number of responses for each rating value.",+ "totalResponses": "The total number of responses the score is based on.",+ },+ },+ "trend_groups": {+ "description": "A trend group: a named set of trends that segment survey scores for one metric.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "_id": "The unique ID of the trend group.",+ "groupName": "The trend group's name.",+ "metric": "The survey metric the group's trends measure (NPS, CSAT, CES, STAR).",+ "isDefault": "Whether this is the account's default trend group.",+ "trendsCount": "The number of trends in the group.",+ "createdDate": "When the trend group was created.",+ "updatedDate": "When the trend group was last updated.",+ },+ },+ "trends": {+ "description": "A trend within a trend group: a segment of responses (by campaign and customer attributes) with its score over the vendor's default date range.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "_id": "The unique ID of the trend.",+ "groupId": "The ID of the trend group the trend belongs to.",+ "trendName": "The trend's name.",+ "metric": "The survey metric the trend measures (NPS, CSAT, CES, STAR).",+ "score": "The current and previous score for the date range, with the absolute and percentage change.",+ "respondents": "Respondent counts, percentages and rating distribution for the date range.",+ "campaigns": "The IDs of the campaigns the trend includes.",+ "attributes": "The customer attribute filters that define the trend's segment.",+ "trend": "The trend line: the score, counts and rating distribution for each date in the range.",+ },+ },+ "suppressed_emails": {+ "description": "An email address on the account's suppression list. Suppressed emails receive no surveys.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "id": "The unique ID of the suppression.",+ "email": "The suppressed email address.",+ "reason": "Why the email is suppressed (for example manual or complaint).",+ "note": "A note explaining the suppression.",+ "addedBy": "Who added the suppression (api, user or system).",+ "createdAt": "When the email was suppressed.",+ },+ },+ "suppressed_domains": {+ "description": "A domain pattern on the account's suppression list. Emails that match it receive no surveys.",+ "docs_url": "https://www.retently.com/api/",+ "columns": {+ "id": "The unique ID of the suppression.",+ "pattern": "The suppressed domain pattern. Supports wildcards such as *.example.com.",+ "category": "The pattern's category (other, corporate, disposable or invalid).",+ "reason": "Why the domain is suppressed.",+ "note": "A note explaining the suppression.",+ "addedBy": "Who added the suppression (api, user or system).",+ "createdAt": "When the domain was suppressed.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.pyindex 4e19c2725c15..8b1cdf6eb607 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/retently.py@@ -1,6 +1,6 @@import dataclassesfrom datetime import UTC, date, datetime-from typing import Any, Optional+from typing import Any, Optional, castfrom requests import Request, Response@@ -9,20 +9,28 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (BasePaginator,SinglePagePaginator,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ Endpoint,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.retently.settings import RETENTLY_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.retently.settings import (+ PAGE_SIZE,+ RETENTLY_ENDPOINTS,+ RetentlyEndpointConfig,+)RETENTLY_BASE_URL = "https://app.retently.com/api/v2"-# Documented maximum page size; the largest page minimises round trips against the ~150 req/min-# rate limit.-PAGE_SIZE = 1000FIRST_PAGE = 1@@ -115,6 +123,45 @@ class RetentlyPaginator(BasePaginator):return f"RetentlyPaginator(page={self.page})"+def _client_config(api_key: str) -> ClientConfig:+ return {+ "base_url": RETENTLY_BASE_URL,+ "headers": _non_secret_headers(),+ "auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},+ }+++def _trends_resource(api_key: str, config: RetentlyEndpointConfig, team_id: int, job_id: str) -> Resource:+ assert config.fanout is not None+ parent = RETENTLY_ENDPOINTS[config.fanout.parent_name]+ # Trend groups and the trends of a group are unpaginated, so a full refresh re-fetches the+ # handful of groups instead of resuming mid fan-out.+ return cast(+ Resource,+ build_dependent_resource(+ endpoint_configs={config.name: config, parent.name: parent},+ child_endpoint=config.name,+ fanout=config.fanout,+ client_config=_client_config(api_key),+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ page_size_param=None,+ parent_endpoint_extra={+ "paginator": SinglePagePaginator(),+ "data_selector": parent.data_selector,+ "data_selector_malformed_retryable": True,+ },+ child_endpoint_extra={+ "paginator": SinglePagePaginator(),+ "data_selector": config.data_selector,+ "data_selector_malformed_retryable": True,+ },+ ),+ )++def retently_source(api_key: str,endpoint: str,@@ -127,6 +174,15 @@ def retently_source(config = RETENTLY_ENDPOINTS[endpoint]supports_incremental = bool(config.incremental_fields)+ if config.fanout is not None:+ trends = _trends_resource(api_key, config, team_id, job_id)+ return SourceResponse(+ name=endpoint,+ items=lambda: trends,+ primary_keys=config.primary_keys,+ column_hints=trends.column_hints,+ )+params: dict[str, Any] = {}paginator: BasePaginatorif config.paginated:@@ -135,8 +191,8 @@ def retently_source(params["sort"] = config.sort_parampaginator = RetentlyPaginator()else:- # Campaigns, templates and reports are documented without pagination and return the full- # collection in one response — no page/limit/sort params.+ # Campaigns, templates, reports, scores and suppressions are documented without pagination+ # and return the full collection in one response — no page/limit/sort params.paginator = SinglePagePaginator()endpoint_config: Endpoint = {@@ -145,8 +201,9 @@ def retently_source("data_selector": config.data_selector,# A 200 whose payload isn't the expected list shape (an HTML proxy error, a transient# error envelope) is retried, matching the old transport which raised a retryable error- # from its extraction step so one malformed page reissues instead of failing the sync.- "data_selector_malformed_retryable": True,+ # from its extraction step so one malformed page reissues instead of failing the sync. The+ # score endpoints legitimately return one object, which this list-shape check would reject.+ "data_selector_malformed_retryable": not config.single_object,"paginator": paginator,}@@ -161,11 +218,7 @@ def retently_source(}rest_config: RESTAPIConfig = {- "client": {- "base_url": RETENTLY_BASE_URL,- "headers": _non_secret_headers(),- "auth": {"type": "api_key", "api_key": api_key, "name": "X-Api-Key", "location": "header"},- },+ "client": _client_config(api_key),"resource_defaults": {},"resources": [{"name": endpoint, "endpoint": endpoint_config}],}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.pyindex 75b2cef8438d..ffcd164981da 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/settings.py@@ -1,10 +1,17 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+# Documented maximum page size; the largest page minimises round trips against the ~150 req/min+# rate limit.+PAGE_SIZE = 1000-@dataclass++@dataclass(frozen=True)class RetentlyEndpointConfig:name: strpath: str@@ -23,6 +30,11 @@ class RetentlyEndpointConfig:incremental_fields: list[IncrementalField] = field(default_factory=list)# Stable creation-time field used for datetime partitioning. Never an updated_at-style field.partition_key: Optional[str] = None+ # The /{metric}/score endpoints return one object under `data`, not a list of rows.+ single_object: bool = False+ fanout: Optional[DependentEndpointConfig] = None+ default_incremental_field: Optional[str] = None+ page_size: int = PAGE_SIZEdef _created_date_incremental_field() -> list[IncrementalField]:@@ -42,7 +54,9 @@ def _created_date_incremental_field() -> list[IncrementalField]:# specified and customer records are mutable (properties, tags), so a creation-window incremental# would silently miss updates — customers stays full refresh. Companies expose no date filter, and# outbox rows carry no consistently documented unique identifier (`surveyId` is absent from some-# example rows), so there is no safe merge key and outbox stays full refresh too.+# example rows), so there is no safe merge key and outbox stays full refresh too. The score, trend+# and suppression endpoints take no date filter or pagination and return the full collection (or a+# point-in-time snapshot) in one response, so they are full refresh as well.RETENTLY_ENDPOINTS: dict[str, RetentlyEndpointConfig] = {"customers": RetentlyEndpointConfig(name="customers",@@ -97,6 +111,55 @@ RETENTLY_ENDPOINTS: dict[str, RetentlyEndpointConfig] = {primary_keys=["campaignId"],paginated=False,),+ # One table per metric: each metric's score payload has its own keys (e.g. `nps`, `promoters`).+ **{+ f"{metric}_score": RetentlyEndpointConfig(+ name=f"{metric}_score",+ path=f"/{metric}/score",+ data_selector="data",+ paginated=False,+ single_object=True,+ )+ for metric in ("nps", "csat", "ces", "star")+ },+ "trend_groups": RetentlyEndpointConfig(+ name="trend_groups",+ path="/trends",+ data_selector="groups",+ primary_keys=["_id"],+ paginated=False,+ ),+ "trends": RetentlyEndpointConfig(+ name="trends",+ path="/trends/{groupId}",+ data_selector="trends",+ primary_keys=["groupId", "_id"],+ paginated=False,+ fanout=DependentEndpointConfig(+ parent_name="trend_groups",+ resolve_param="groupId",+ resolve_field="_id",+ include_from_parent=["_id"],+ parent_field_renames={"_id": "groupId"},+ # A group deleted between the group listing and this fetch 404s; skip it instead of+ # failing the whole fan-out.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),+ "suppressed_emails": RetentlyEndpointConfig(+ name="suppressed_emails",+ path="/suppressions/emails",+ data_selector="data",+ primary_keys=["id"],+ paginated=False,+ ),+ "suppressed_domains": RetentlyEndpointConfig(+ name="suppressed_domains",+ path="/suppressions/domains",+ data_selector="data",+ primary_keys=["id"],+ paginated=False,+ ),}ENDPOINTS = tuple(RETENTLY_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.pyindex 7dd9f1f0bb45..a8e5246764af 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/retently/tests/test_retently.py@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.retently import (@@ -56,6 +57,13 @@ class TestEnvelopeShapes:("bare_list_under_data", "reports", {"data": [{"campaignId": "c1"}]}, [{"campaignId": "c1"}]),# Campaigns/templates document the array at the top level, outside `data`.("top_level_array", "campaigns", {"campaigns": [{"id": "c1"}]}, [{"id": "c1"}]),+ # The score endpoints return one object under `data`, synced as a single row.+ (+ "single_object_under_data",+ "nps_score",+ {"message": "Success.", "code": 200, "data": {"nps": 39, "totalResponses": 13}},+ [{"nps": 39, "totalResponses": 13}],+ ),])def test_documented_envelope_shapes_yield_rows(self, _name: str, endpoint: str, body: Any, expected: list) -> None:@@ -113,6 +121,29 @@ class TestPagination:assert "limit" not in result.requests[0].query+class TestTrendsFanout:+ def test_trends_are_fetched_per_group_and_carry_the_group_id(self) -> None:+ result = _run(+ "trends",+ route(+ {+ "/trends": [_response({"success": True, "groups": [{"_id": "g1"}, {"_id": "g2"}]})],+ # A 200 error envelope without `trends` is retried, not synced as an empty group.+ "/trends/g1": [+ _response({"success": False, "error": "Internal Server Error"}),+ _response({"success": True, "trends": [{"_id": "t1", "trendName": "Enterprise"}]}),+ ],+ # A group deleted between the listing and its fetch must not fail the sync.+ "/trends/g2": [_response({"success": False}, status_code=404)],+ }+ ),+ )++ assert result.raised is None+ assert result.paths == ["/api/v2/trends", "/api/v2/trends/g1", "/api/v2/trends/g1", "/api/v2/trends/g2"]+ assert [(row["groupId"], row["_id"]) for row in result.rows] == [("g1", "t1")]++class TestIncremental:def test_start_date_sent_when_incremental(self) -> None:result = _run(