Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Trustpilot source (#120443)

Tom Owersc84ce6b12ddd
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 778648745598..98f0ab986996 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -12562,10 +12562,10 @@ Today (3): `business_unit`, `product_reviews`, `service_reviews`
Diffed against: <https://developers.trustpilot.com/product-reviews-api/>
-- [ ] `Catalog products (GET /v1/private/business-units/{businessUnitId}/catalog/products)` — join review product IDs to complete catalog records, variants, prices, and products with no reviews (high)
-- [ ] `Conversations (GET /v1/private/conversations/{conversationId})` — resolve conversation IDs on product reviews into customer and business comments for response analysis (high)
-- [ ] `Locations (GET /v1/private/business-units/{businessUnitId}/locations)` — resolve review location IDs into complete branch records and addresses for geographic comparisons (high)
-- [ ] `Product reviews summaries list (GET /v1/private/product-reviews/business-units/{businessUnitId}/summaries)` — compare official rating averages, distributions, and review counts across products (medium)
+- [x] `Catalog products (GET /v1/private/business-units/{businessUnitId}/catalog/products)` — join review product IDs to complete catalog records, variants, prices, and products with no reviews (high)
+- [x] `Conversations (GET /v1/private/conversations/{conversationId})` — resolve conversation IDs on product reviews into customer and business comments for response analysis (high)
+- [x] `Locations (GET /v1/private/business-units/{businessUnitId}/locations)` — resolve review location IDs into complete branch records and addresses for geographic comparisons (high)
+- [x] `Product reviews summaries list (GET /v1/private/product-reviews/business-units/{businessUnitId}/summaries)` — compare official rating averages, distributions, and review counts across products (medium)
- [ ] `Imported product reviews (GET /v1/product-reviews/business-units/{businessUnitId}/imported-reviews)` — include externally imported feedback in product satisfaction and review volume analysis (medium)
- [ ] `Categories for business unit (GET /v1/business-units/{businessUnitId}/categories)` — compare ratings across business categories and identify each category assignment (medium)
- [ ] `Review likes (GET /v1/reviews/{reviewId}/likes)` — identify which consumers engage with service reviews beyond the embedded like count (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/canonical_descriptions.py
index 7c87847eeb56..7e17c8f188a2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/canonical_descriptions.py
@@ -76,4 +76,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"links": "Related Trustpilot API resource links.",
},
},
+ "product_review_summaries": {
+ "description": "Product review summaries per product SKU: the star average, the review count and the per-star distribution.",
+ "docs_url": "https://developers.trustpilot.com/product-reviews-api/",
+ "columns": {
+ "sku": "Stock-keeping unit (SKU) of the product the summary is for.",
+ "starsAverage": "Average star rating of the product's reviews.",
+ "numberOfReviews": "Review counts for the product: total and per-star breakdown.",
+ },
+ },
+ "catalog_products": {
+ "description": "Products in the business unit's Trustpilot product catalog, with their variants, per-locale content, per-market prices and review aggregates. Includes products with no reviews.",
+ "docs_url": "https://developers.trustpilot.com/private-products-api/",
+ "columns": {
+ "productId": "Unique identifier of the catalog product.",
+ "businessUnitId": "Identifier of the business unit that owns the product.",
+ "brand": "Brand of the product.",
+ "categoryId": "Identifier of the product category.",
+ "customerProductId": "Identifier the business uses for the product in its own catalog.",
+ "content": "Product title and description, keyed by locale.",
+ "averageStarRating": "Average star rating across the product's variants.",
+ "totalReviewsCount": "Total number of reviews across the product's variants.",
+ "variantCount": "Number of variants returned for the product.",
+ "totalVariantCount": "Total number of variants the product has.",
+ "variants": "Product variants: id, SKU, links, attributes by locale, prices by market and processed images.",
+ "metadata": "Product metadata, such as when the product was created.",
+ },
+ },
+ "locations": {
+ "description": "Locations (branches) of the business unit, with their contact details and addresses.",
+ "docs_url": "https://developers.trustpilot.com/locations-api/",
+ "columns": {
+ "id": "Unique identifier of the location.",
+ "name": "Name of the location.",
+ "urlFormattedName": "URL-formatted name of the location.",
+ "customerLocationId": "Identifier the business set for the location during setup.",
+ "businessUnitId": "Identifier of the business unit the location belongs to.",
+ "phone": "Phone number of the location.",
+ "website": "Website of the location.",
+ "description": "Header and text that describe the location.",
+ "locationDetails": "Address of the location: country code, street, locality, postal code, administrative area, formatted address, latitude and longitude.",
+ },
+ },
+ "product_review_conversations": {
+ "description": "Conversations attached to product reviews, with each comment from the consumer or the business. One row per conversation that a synced product review refers to.",
+ "docs_url": "https://developers.trustpilot.com/product-reviews-api/",
+ "columns": {
+ "id": "Unique identifier of the conversation.",
+ "businessUnitId": "Identifier of the business unit the conversation belongs to.",
+ "consumerId": "Identifier of the consumer in the conversation.",
+ "source": "Source of the conversation, for example product reviews.",
+ "sourceId": "Identifier of the item the conversation is about, for example the product review.",
+ "state": "State of the conversation, for example public.",
+ "comments": "Comments in the conversation: id, creation time, author and content.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/settings.py
index be3bf278a604..2c9d4c3229ac 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/settings.py
@@ -1,5 +1,9 @@
from dataclasses import dataclass, field
+from typing import Any
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
BASE_URL = "https://api.trustpilot.com/v1"
@@ -14,6 +18,9 @@ PER_PAGE = 100
# small; only the initial backfill of a very large account can reach this cap.
MAX_PAGES = 1000
+# The catalog products endpoint documents a perPage cap of 1000 (default 100).
+CATALOG_PER_PAGE = 1000
+
@dataclass(frozen=True)
class TrustpilotEndpointConfig:
@@ -32,6 +39,22 @@ class TrustpilotEndpointConfig:
primary_keys: list[str] = field(default_factory=lambda: ["id"])
# Stable creation-time field to partition by.
partition_key: str | None = None
+ # None for endpoints that document no perPage param.
+ per_page: int | None = PER_PAGE
+ # Only the review list endpoints document the 100,000-record query cap.
+ max_pages: int | None = MAX_PAGES
+ params: dict[str, Any] = field(default_factory=dict)
+ # Set for endpoints read one row at a time, keyed by an ID from a parent endpoint's rows.
+ fanout: DependentEndpointConfig | None = None
+
+ # `default_incremental_field` and `page_size` satisfy the shared fan-out helper's protocol.
+ @property
+ def default_incremental_field(self) -> str | None:
+ return None
+
+ @property
+ def page_size(self) -> int:
+ return self.per_page or PER_PAGE
TRUSTPILOT_ENDPOINTS: dict[str, TrustpilotEndpointConfig] = {
@@ -69,6 +92,57 @@ TRUSTPILOT_ENDPOINTS: dict[str, TrustpilotEndpointConfig] = {
# No documented server-side time filter, so this endpoint is full refresh only.
partition_key="createdAt",
),
+ "product_review_summaries": TrustpilotEndpointConfig(
+ name="product_review_summaries",
+ path="/private/product-reviews/business-units/{business_unit_id}/summaries",
+ response_key="summaries",
+ requires_oauth=True,
+ max_pages=None,
+ # One summary per product SKU.
+ primary_keys=["sku"],
+ ),
+ "catalog_products": TrustpilotEndpointConfig(
+ name="catalog_products",
+ path="/private/business-units/{business_unit_id}/catalog/products",
+ response_key="products",
+ requires_oauth=True,
+ per_page=CATALOG_PER_PAGE,
+ max_pages=None,
+ params={
+ "sortBy": "createdAt",
+ "sortOrder": "asc",
+ # Return every stored locale and market, not only the default one, so variant content
+ # and prices are complete.
+ "returnAllLocales": "true",
+ "returnAllMarkets": "true",
+ "includeMatchingProductCount": "false",
+ },
+ primary_keys=["productId"],
+ ),
+ "locations": TrustpilotEndpointConfig(
+ name="locations",
+ path="/private/business-units/{business_unit_id}/locations",
+ response_key="locations",
+ requires_oauth=True,
+ per_page=None,
+ max_pages=None,
+ ),
+ "product_review_conversations": TrustpilotEndpointConfig(
+ name="product_review_conversations",
+ path="/private/conversations/{conversation_id}",
+ response_key=None,
+ requires_oauth=True,
+ paginated=False,
+ fanout=DependentEndpointConfig(
+ parent_name="product_reviews",
+ resolve_param="conversation_id",
+ resolve_field="conversationId",
+ include_from_parent=[],
+ parent_params={"perPage": PER_PAGE},
+ # A conversation removed between the review listing and this fetch must not fail the sync.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(TRUSTPILOT_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/tests/test_trustpilot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/tests/test_trustpilot.py
index 69c62bdc9f62..4829f30ed476 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/tests/test_trustpilot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/tests/test_trustpilot.py
@@ -290,14 +290,108 @@ class TestPagination:
assert first.param("startDateTime") is None
assert first.param("orderBy") is None
+ @parameterized.expand(
+ [
+ (
+ "product_review_summaries",
+ f"/v1/private/product-reviews/business-units/{BUSINESS_UNIT_ID}/summaries",
+ "summaries",
+ "sku",
+ "100",
+ {},
+ ),
+ (
+ "catalog_products",
+ f"/v1/private/business-units/{BUSINESS_UNIT_ID}/catalog/products",
+ "products",
+ "productId",
+ "1000",
+ {
+ "sortBy": "createdAt",
+ "sortOrder": "asc",
+ "returnAllLocales": "true",
+ "returnAllMarkets": "true",
+ "includeMatchingProductCount": "false",
+ },
+ ),
+ # The locations endpoint documents only `page`, so no perPage is sent.
+ ("locations", f"/v1/private/business-units/{BUSINESS_UNIT_ID}/locations", "locations", "id", None, {}),
+ ]
+ )
+ def test_list_endpoint_walks_pages_with_documented_params(
+ self,
+ endpoint: str,
+ expected_path: str,
+ key: str,
+ id_field: str,
+ expected_per_page: str | None,
+ expected_params: dict[str, str],
+ ) -> None:
+ result = _run(
+ _serve(
+ _reviews_response([{id_field: "a"}], key=key),
+ _reviews_response([{id_field: "b"}], key=key),
+ _reviews_response([], key=key),
+ ),
+ endpoint=endpoint,
+ )
+
+ assert [r[id_field] for r in result.rows] == ["a", "b"]
+ requests = [r for r in result.requests if r.path == expected_path]
+ assert [r.param("page") for r in requests] == ["1", "2", "3"]
+ assert requests[0].param("perPage") == expected_per_page
+ for name, value in expected_params.items():
+ assert requests[0].param(name) == value
+
+
+class TestProductReviewConversations:
+ def test_fetches_only_referenced_conversations_and_skips_missing_ones(self) -> None:
+ reviews: list[dict[str, Any]] = [
+ {"id": "r1", "conversationId": "c1"},
+ {"id": "r2", "conversationId": None},
+ {"id": "r3"},
+ {"id": "r4", "conversationId": "c-gone"},
+ ]
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/business-units/find"):
+ return _json({"id": BUSINESS_UNIT_ID})
+ if request.path.endswith("/accesstoken"):
+ return _json({"access_token": "tok", "expires_in": 3600})
+ if request.path.endswith("/reviews"):
+ page = request.param("page")
+ return _json(_reviews_response(reviews if page == "1" else [], key="productReviews"))
+ if request.path.endswith("/conversations/c1"):
+ return _json({"id": "c1", "sourceId": "r1", "comments": [{"commentId": "m1", "content": "Thanks"}]})
+ if request.path.endswith("/conversations/c-gone"):
+ return _json({"details": "not found"}, status=404)
+ raise UnexpectedRequest(f"unexpected request {request.url}")
+
+ result = _run(respond, endpoint="product_review_conversations")
+
+ assert [r["id"] for r in result.rows] == ["c1"]
+ conversation_paths = [r.path for r in result.requests if "/conversations/" in r.path]
+ assert conversation_paths == ["/v1/private/conversations/c1", "/v1/private/conversations/c-gone"]
+ review_request = next(r for r in result.requests if r.path.endswith("/reviews"))
+ assert review_request.path == f"/v1/private/product-reviews/business-units/{BUSINESS_UNIT_ID}/reviews"
+ assert review_request.param("perPage") == "100"
+
class TestPaginatorQueryCap:
- def test_stops_and_warns_at_query_cap(self, caplog: pytest.LogCaptureFixture) -> None:
- paginator = TrustpilotPaginator()
+ @pytest.mark.parametrize(
+ "maximum_page, expect_next",
+ [
+ pytest.param(MAX_PAGES, False, id="review_endpoint_capped"),
+ # Catalog, locations and summaries document no record cap, so they page past it.
+ pytest.param(None, True, id="uncapped_endpoint"),
+ ],
+ )
+ def test_query_cap(self, maximum_page: int | None, expect_next: bool, caplog: pytest.LogCaptureFixture) -> None:
+ paginator = TrustpilotPaginator(maximum_page)
paginator.page = MAX_PAGES
with caplog.at_level(logging.WARNING):
paginator.update_state(mock.MagicMock(), data=[{"id": "r1"}])
- assert paginator.has_next_page is False
- assert any("query cap" in record.message for record in caplog.records)
+ assert paginator.has_next_page is expect_next
+ assert any("query cap" in record.message for record in caplog.records) is not expect_next
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/trustpilot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/trustpilot.py
index 7b68a09c475f..8443f1363bcc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/trustpilot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trustpilot/trustpilot.py
@@ -2,7 +2,7 @@ import re
import logging
import dataclasses
from datetime import UTC, date, datetime
-from typing import Any, Optional
+from typing import Any, Literal, Optional
from requests import Response
@@ -11,11 +11,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
PageNumberPaginator,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ApiKeyAuthConfig,
+ ClientConfig,
OAuth2AuthConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
@@ -23,9 +27,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.trustpilot.settings import (
BASE_URL,
MAX_PAGES,
- PER_PAGE,
TOKEN_URL,
TRUSTPILOT_ENDPOINTS,
+ TrustpilotEndpointConfig,
)
logger = logging.getLogger(__name__)
@@ -219,8 +223,8 @@ class TrustpilotPaginator(PageNumberPaginator):
read as a complete sync, so log what was left behind instead.
"""
- def __init__(self) -> None:
- super().__init__(base_page=1, page_param="page", total_path=None, maximum_page=MAX_PAGES)
+ def __init__(self, maximum_page: int | None = MAX_PAGES) -> None:
+ super().__init__(base_page=1, page_param="page", total_path=None, maximum_page=maximum_page)
def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None:
super().update_state(response, data)
@@ -247,6 +251,59 @@ def _build_auth(api_key: str, api_secret: str, requires_oauth: bool) -> ApiKeyAu
}
+def _client_config(
+ api_key: str, api_secret: str, requires_oauth: bool, paginator: TrustpilotPaginator | Literal["single_page"]
+) -> ClientConfig:
+ return {
+ "base_url": BASE_URL,
+ "headers": {"Accept": "application/json"},
+ "auth": _build_auth(api_key, api_secret, requires_oauth),
+ "paginator": paginator,
+ # capture=False: private review rows carry consumer emails, order references and
+ # review text, which must stay out of HTTP sample capture (still metered and logged).
+ "session": make_tracked_session(redact_values=(api_key, api_secret), capture=False),
+ }
+
+
+def _has_conversation(row: dict[str, Any]) -> dict[str, Any] | list[dict[str, Any]]:
+ # Reviews without a reply carry no conversation; an empty list drops them before the fan-out.
+ return row if row.get("conversationId") else []
+
+
+def _conversations_source(
+ api_key: str,
+ api_secret: str,
+ endpoint: str,
+ config: TrustpilotEndpointConfig,
+ business_unit_id: str,
+ team_id: int,
+ job_id: str,
+) -> SourceResponse:
+ assert config.fanout is not None
+ parent = TRUSTPILOT_ENDPOINTS[config.fanout.parent_name]
+ resource = build_dependent_resource(
+ endpoint_configs=TRUSTPILOT_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ client_config=_client_config(api_key, api_secret, config.requires_oauth, TrustpilotPaginator(parent.max_pages)),
+ path_format_values={"business_unit_id": business_unit_id},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ parent_endpoint_extra={"data_selector": parent.response_key} if parent.response_key else None,
+ parent_data_map=_has_conversation,
+ child_endpoint_extra={"paginator": "single_page"},
+ page_size_param=None,
+ )
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ )
+
+
def trustpilot_source(
api_key: str,
api_secret: str,
@@ -260,6 +317,10 @@ def trustpilot_source(
) -> SourceResponse:
config = TRUSTPILOT_ENDPOINTS[endpoint]
business_unit_id = resolve_business_unit_id(api_key, business_unit)
+
+ if config.fanout is not None:
+ return _conversations_source(api_key, api_secret, endpoint, config, business_unit_id, team_id, job_id)
+
path = config.path.format(business_unit_id=business_unit_id)
resume: TrustpilotResumeConfig | None = None
@@ -274,24 +335,21 @@ def trustpilot_source(
elif db_incremental_field_last_value is not None:
start_date_time = _format_start_datetime(db_incremental_field_last_value)
- params: dict[str, Any] = {}
- if config.paginated:
- params["perPage"] = PER_PAGE
+ params: dict[str, Any] = dict(config.params)
+ if config.paginated and config.per_page is not None:
+ params["perPage"] = config.per_page
if config.order_by:
params["orderBy"] = config.order_by
if start_date_time:
params["startDateTime"] = start_date_time
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": BASE_URL,
- "headers": {"Accept": "application/json"},
- "auth": _build_auth(api_key, api_secret, config.requires_oauth),
- "paginator": TrustpilotPaginator() if config.paginated else "single_page",
- # capture=False: private review rows carry consumer emails, order references and
- # review text, which must stay out of HTTP sample capture (still metered and logged).
- "session": make_tracked_session(redact_values=(api_key, api_secret), capture=False),
- },
+ "client": _client_config(
+ api_key,
+ api_secret,
+ config.requires_oauth,
+ TrustpilotPaginator(config.max_pages) if config.paginated else "single_page",
+ ),
"resource_defaults": {},
"resources": [
{