Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the Etsy source (#125224)

Tom Owerse2c0a767a17a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 160c8cf4b6d8..a9b78701287a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -4334,11 +4334,11 @@ Diffed against: <https://www.etsy.com/openapi/generated/oas/3.0.0.json>
- [x] `Seller taxonomy nodes (GET /v3/application/seller-taxonomy/nodes)` — resolve taxonomy_id on listings to product categories and their hierarchy (high)
- [x] `Listing inventory (GET /v3/application/listings/batch/inventory)` — resolve transaction product_id values and analyze variant SKUs, prices, and stock quantities (high)
-- [ ] `Shop shipping profiles (GET /v3/application/shops/{shop_id}/shipping-profiles)` — resolve shipping_profile_id on listings and transactions for shipping cost and service comparisons (high)
-- [ ] `Shop readiness state definitions (GET /v3/application/shops/{shop_id}/readiness-state-definitions)` — resolve readiness_state_id on listings into preparation states and processing times (high)
+- [x] `Shop shipping profiles (GET /v3/application/shops/{shop_id}/shipping-profiles)` — resolve shipping_profile_id on listings and transactions for shipping cost and service comparisons (high)
+- [x] `Shop readiness state definitions (GET /v3/application/shops/{shop_id}/readiness-state-definitions)` — resolve readiness_state_id on listings into preparation states and processing times (high)
- [x] `Shop return policies (GET /v3/application/shops/{shop_id}/policies/return)` — resolve return_policy_id on listings for returns and catalog analysis (high)
- [x] `Payment account ledger entry payments (GET /v3/application/shops/{shop_id}/payment-account/ledger-entries/payments)` — reconcile gross sales, fees, net payments, and payment adjustments with receipts and ledger entries (high)
-- [ ] `Listing properties (GET /v3/application/shops/{shop_id}/listings/{listing_id}/properties)` — analyze sales and assortment by product attributes such as size, material, and color (medium)
+- [x] `Listing properties (GET /v3/application/shops/{shop_id}/listings/{listing_id}/properties)` — analyze sales and assortment by product attributes such as size, material, and color (medium)
Note: The official OpenAPI spec supplied GET paths, parameters, and listing and transaction fields. Transactions already expand from receipts, so alternate transaction routes are excluded. Inventory and payment reads require parent ID batches; properties require one request per listing. Media and authentication are excluded; business lookups are retained because existing rows reference their IDs.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/canonical_descriptions.py
index 162c7ee20667..8c02bc3feb29 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/canonical_descriptions.py
@@ -205,6 +205,48 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"return_deadline": "Days the buyer has to return an item.",
},
},
+ "shipping_profiles": {
+ "description": "The shop's shipping profiles, with their destinations and upgrades. Listings reference them by shipping_profile_id.",
+ "docs_url": f"{_DOCS}#operation/getShopShippingProfiles",
+ "columns": {
+ "shipping_profile_id": "Unique identifier for the shipping profile.",
+ "title": "Name of the shipping profile.",
+ "user_id": "Numeric id of the Etsy user who owns the shipping profile.",
+ "origin_country_iso": "ISO code of the country the listing ships from.",
+ "origin_postal_code": "Postal code of the location the listing ships from.",
+ "is_deleted": "Whether the shipping profile was deleted.",
+ "profile_type": "Whether the profile uses manual or calculated shipping costs.",
+ "shipping_profile_destinations": "Destinations the profile ships to, each with its costs, carrier, mail class and delivery days.",
+ "shipping_profile_upgrades": "Paid shipping upgrades the buyer can choose, each with its price and delivery days.",
+ "domestic_handling_fee": "Handling fee added to a domestic buyer's shipping total. Calculated profiles only.",
+ "international_handling_fee": "Handling fee added to an international buyer's shipping total. Calculated profiles only.",
+ },
+ },
+ "readiness_state_definitions": {
+ "description": "The shop's processing profiles, each a readiness state with its processing time. Listings reference them by readiness_state_id.",
+ "docs_url": f"{_DOCS}#operation/getShopReadinessStateDefinitions",
+ "columns": {
+ "readiness_state_id": "Unique identifier for the processing profile.",
+ "shop_id": "Numeric id of the shop that owns the processing profile.",
+ "readiness_state": "Readiness of the product: ready_to_ship or made_to_order.",
+ "min_processing_days": "Minimum number of days to process the product.",
+ "max_processing_days": "Maximum number of days to process the product.",
+ "processing_days_display_label": "Translated label for the processing time, e.g. 3 - 5 days.",
+ },
+ },
+ "listing_properties": {
+ "description": "One row per structured property set on each listing, such as its color, size or material.",
+ "docs_url": f"{_DOCS}#operation/getListingProperties",
+ "columns": {
+ "listing_id": "Listing the property value belongs to.",
+ "property_id": "Identifier of the property, e.g. the id for color. It repeats across listings.",
+ "property_name": "Name of the property.",
+ "scale_id": "Identifier of the scale the values use, if any.",
+ "scale_name": "Label of the scale the values use, if any.",
+ "value_ids": "Ids of the property values set on the listing.",
+ "values": "Property values set on the listing.",
+ },
+ },
"seller_taxonomy_nodes": {
"description": "Etsy's seller taxonomy, one row per category node. Listings reference it by taxonomy_id.",
"docs_url": f"{_DOCS}#operation/getSellerTaxonomyNodes",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/etsy.py b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/etsy.py
index 3925d6c5a88b..be875abdfd48 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/etsy.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/etsy.py
@@ -16,6 +16,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.etsy.setti
LISTING_STATES,
EtsyBatchLookup,
EtsyEndpointConfig,
+ EtsyFanOut,
)
ETSY_API_BASE = "https://api.etsy.com/v3/application"
@@ -455,6 +456,33 @@ def _batch_lookup_pages(
manager.safe_point()
+def _fan_out_pages(
+ client: EtsyClient,
+ fan_out: EtsyFanOut,
+ shop_id: str,
+ parent_pages: Iterator[list[dict[str, Any]]],
+ manager: ResumableSourceManager[EtsyResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ for parent_rows in parent_pages:
+ for parent_row in parent_rows:
+ parent_id = parent_row.get(fan_out.id_field)
+ if parent_id is None:
+ continue
+ try:
+ body = client.request(f"/shops/{shop_id}{fan_out.path.format(id=parent_id)}")
+ except HTTPError as error:
+ # A listing deleted after the parent page was read is gone, not a failed sync.
+ if error.response is None or error.response.status_code != 404:
+ raise
+ body = {}
+ rows = [{**row, fan_out.id_field: parent_id} for row in body.get("results") or [] if isinstance(row, dict)]
+ if rows:
+ yield rows
+ else:
+ # Many listings carry no properties, and each one still costs a request.
+ manager.safe_point()
+
+
def _paged_rows(
client: EtsyClient,
shop_id: str,
@@ -517,6 +545,21 @@ def get_rows(
rows = _rows_from_results(client.request(path).get("results") or [], config)
if rows:
yield rows
+ elif config.fan_out is not None:
+ parent = ETSY_ENDPOINTS[config.fan_out.parent]
+ parent_pages = _paged_rows(
+ client,
+ resolved_shop_id,
+ f"/shops/{resolved_shop_id}{parent.path}",
+ parent,
+ resumable_source_manager,
+ resume,
+ logger,
+ should_use_incremental_field=False,
+ db_incremental_field_last_value=None,
+ incremental_field=None,
+ )
+ yield from _fan_out_pages(client, config.fan_out, resolved_shop_id, parent_pages, resumable_source_manager)
elif config.batch_lookup is not None:
parent = ETSY_ENDPOINTS[config.batch_lookup.parent]
# The set of rows already yielded does not survive to a resumed attempt, and a resumed full
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/settings.py
index 3e11ac0c8e8c..939c7bd4cae3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/settings.py
@@ -40,6 +40,18 @@ class EtsyBatchLookup:
dedupe_key: str | None = None
+@dataclass(frozen=True)
+class EtsyFanOut:
+ """Rows fetched with one request per row of a parent endpoint."""
+
+ # Endpoint whose pages supply the ids. Its walk and resume state drive the fan-out.
+ parent: str
+ # Appended to /shops/{shop_id}. `{id}` is filled in with each parent row's id.
+ path: str
+ # Field on each parent row that holds the id. Each child row carries it under the same name.
+ id_field: str
+
+
@dataclass(frozen=True)
class EtsyEndpointConfig:
name: str
@@ -75,6 +87,8 @@ class EtsyEndpointConfig:
flatten_key: str | None = None
# Rows come from a batch endpoint keyed on ids the parent endpoint pages through.
batch_lookup: EtsyBatchLookup | None = None
+ # Rows come from one request per row the parent endpoint pages through.
+ fan_out: EtsyFanOut | None = None
ETSY_ENDPOINTS: dict[str, EtsyEndpointConfig] = {
@@ -176,6 +190,26 @@ ETSY_ENDPOINTS: dict[str, EtsyEndpointConfig] = {
primary_keys=["return_policy_id"],
paginated=False,
),
+ "shipping_profiles": EtsyEndpointConfig(
+ name="shipping_profiles",
+ path="/shipping-profiles",
+ primary_keys=["shipping_profile_id"],
+ paginated=False,
+ ),
+ "readiness_state_definitions": EtsyEndpointConfig(
+ name="readiness_state_definitions",
+ path="/readiness-state-definitions",
+ primary_keys=["readiness_state_id"],
+ ),
+ "listing_properties": EtsyEndpointConfig(
+ name="listing_properties",
+ path="",
+ # Property ids name an attribute such as color, so they repeat across listings.
+ primary_keys=["listing_id", "property_id"],
+ # Etsy has no batch read and no listings include for properties, so this costs one request
+ # per listing.
+ fan_out=EtsyFanOut(parent="listings", path="/listings/{id}/properties", id_field="listing_id"),
+ ),
"seller_taxonomy_nodes": EtsyEndpointConfig(
name="seller_taxonomy_nodes",
path="/seller-taxonomy/nodes",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/tests/test_etsy.py b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/tests/test_etsy.py
index 909ce4232682..67008a64ef47 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/tests/test_etsy.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/etsy/tests/test_etsy.py
@@ -456,6 +456,72 @@ class TestEtsyTransport:
if request.path.endswith("/listings/batch/inventory")
] == ["10,11", "10", "11"]
+ @parameterized.expand(
+ [
+ # A listing deleted after the listings page was read must not fail the sync.
+ ("deleted listing is skipped", 404, None),
+ ("missing scope still fails", 403, HTTPError),
+ ]
+ )
+ def test_listing_properties_fan_out_per_listing(
+ self, _name: str, middle_status: int, expected_error: Optional[type[Exception]]
+ ) -> None:
+ result = _driver().run(
+ "listing_properties",
+ route(
+ {
+ "/oauth/token": [_scripted_token()],
+ "/shops/1/listings": [
+ _scripted_page(_rows(3, start=10, key="listing_id"), 3),
+ *[_scripted_page([], 0) for _ in range(len(LISTING_STATES) - 1)],
+ ],
+ "/shops/1/listings/10/properties": [
+ _scripted_page(
+ [
+ {"property_id": 200, "property_name": "Primary color", "values": ["Red"]},
+ {"property_id": 46, "property_name": "Material", "values": ["Oak"]},
+ ],
+ 2,
+ )
+ ],
+ "/shops/1/listings/11/properties": [ScriptedResponse(status=middle_status, json={"error": "x"})],
+ "/shops/1/listings/12/properties": [_scripted_page([], 0)],
+ }
+ ),
+ )
+
+ if expected_error is not None:
+ assert isinstance(result.raised, expected_error)
+ return
+ assert result.raised is None
+ # Etsy's response omits the listing id, and property ids repeat across listings, so the
+ # rows need it to stay unique.
+ assert result.rows == [
+ {"property_id": 200, "property_name": "Primary color", "values": ["Red"], "listing_id": 10},
+ {"property_id": 46, "property_name": "Material", "values": ["Oak"], "listing_id": 10},
+ ]
+
+ def test_listing_properties_resume_the_listings_walk(self) -> None:
+ result = _driver().run(
+ "listing_properties",
+ route(
+ {
+ "/oauth/token": [_scripted_token()],
+ "/shops/1/listings": [
+ _scripted_page(_rows(1, start=10, key="listing_id"), PAGE_SIZE + 1),
+ _scripted_page([], 0),
+ ],
+ "/shops/1/listings/10/properties": [_scripted_page([{"property_id": 200}], 1)],
+ }
+ ),
+ resume_state=EtsyResumeConfig(offset=PAGE_SIZE, listing_state="draft"),
+ )
+
+ assert result.raised is None
+ listing_pages = [request for request in _get_requests(result) if request.path.endswith("/listings")]
+ assert (listing_pages[0].param("state"), listing_pages[0].param("offset")) == ("draft", str(PAGE_SIZE))
+ assert result.rows == [{"property_id": 200, "listing_id": 10}]
+
def test_seller_taxonomy_flattens_the_tree_without_looking_up_a_shop(self) -> None:
tree: list[dict[str, Any]] = [
{