Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add printify catalog variant, shipping, and gpsr tables (#126782)

Tom Owers5763774feb96
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index db58865432c9..82f325437c4d 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -871,8 +871,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/prefect_cloud/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pretix/pretix.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pretix/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 7cf66a363a04..78412f5fa43d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9763,10 +9763,10 @@ Today (7): `blueprints`, `orders`, `print_providers`, `products`, `shops`, `uplo
Diffed against: <https://developers.printify.com/>
-- [ ] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants` — the variant lookup table that resolves the variant_id carried on every synced product and order line item (size/color/SKU/price) (high)
-- [ ] `catalog/blueprints/{blueprint_id}/print_providers` — blueprint-to-print-provider mapping table; joins the two catalog tables already synced (medium)
-- [ ] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping (and the v2 shipping/standard|express|priority|economy variants)` — shipping cost and handling time per variant/country, needed for order margin analysis (medium)
-- [ ] `shops/{shop_id}/products/{product_id}/gpsr` — product safety/compliance attributes per product (low)
+- [x] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants` — the variant lookup table that resolves the variant_id carried on every synced product and order line item (size/color/SKU/price) (high)
+- [x] `catalog/blueprints/{blueprint_id}/print_providers` — blueprint-to-print-provider mapping table; joins the two catalog tables already synced (medium)
+- [x] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping (and the v2 shipping/standard|express|priority|economy variants)` — shipping cost and handling time per variant/country, needed for order margin analysis (medium)
+- [x] `shops/{shop_id}/products/{product_id}/gpsr` — product safety/compliance attributes per product (low)
Note: Single-page reference; v1 covers catalog/shops/products/orders/uploads/webhooks, and a v2 catalog shipping family (shipping/standard, /express, /priority, /economy) exists alongside the v1 shipping endpoint. Order line items are nested inside the orders payload, so they are not a separate gap.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py
index bc1c1b668848..62aa3900476e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py
@@ -100,4 +100,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"location": "The print provider's location address.",
},
},
+ "blueprint_print_providers": {
+ "description": "Which print providers fulfill each catalog blueprint, with their decoration methods and production time.",
+ "docs_url": "https://developers.printify.com/#catalog",
+ "columns": {
+ "id": "The ID of the print provider.",
+ "blueprint_id": "The ID of the catalog blueprint the print provider fulfills.",
+ "title": "The print provider's name.",
+ "decoration_methods": "Decoration methods the print provider offers for the blueprint (e.g. dtg, embroidery).",
+ "production_time": "The print provider's average production time in business days.",
+ },
+ },
+ "catalog_variants": {
+ "description": "Catalog variants (e.g. color and size combinations) for each blueprint and print provider used by a product in a connected shop. Includes out-of-stock variants.",
+ "docs_url": "https://developers.printify.com/#catalog",
+ "columns": {
+ "id": "The variant ID, referenced as variant_id on product variants and order line items.",
+ "blueprint_id": "The ID of the catalog blueprint.",
+ "print_provider_id": "The ID of the print provider.",
+ "title": "The variant's name (e.g. Heather Grey / XS).",
+ "options": "The option values of the variant, such as color and size.",
+ "placeholders": "Print areas available on the variant, with their dimensions and decoration method.",
+ "decoration_methods": "Decoration methods available for the variant.",
+ },
+ },
+ "catalog_shipping_rates": {
+ "description": "Shipping cost and handling time for each variant and destination country, for each blueprint and print provider used by a product in a connected shop.",
+ "docs_url": "https://developers.printify.com/#catalog",
+ "columns": {
+ "blueprint_id": "The ID of the catalog blueprint.",
+ "print_provider_id": "The ID of the print provider.",
+ "variant_id": "The ID of the catalog variant.",
+ "country": "The destination country code, or REST_OF_THE_WORLD.",
+ "handling_time": "The print provider's handling time for the blueprint.",
+ "first_item": "The shipping cost of the first item, in the smallest currency unit, and its currency.",
+ "additional_items": "The shipping cost of each additional item, in the smallest currency unit, and its currency.",
+ },
+ },
+ "product_gpsr": {
+ "description": "General Product Safety Regulation (GPSR) information for each product in a connected shop.",
+ "docs_url": "https://developers.printify.com/#products",
+ "columns": {
+ "shop_id": "The ID of the shop the product belongs to.",
+ "product_id": "The ID of the product.",
+ "sections": "GPSR sections, each with a title (e.g. EU representative, Care instructions) and text.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
index e44cc2d8e983..00d4da28ff71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
@@ -11,11 +11,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
make_tracked_session,
retry_with_deadline,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.printify.settings import (
PRINTIFY_ENDPOINTS,
PrintifyEndpointConfig,
+ PrintifyFanOut,
)
PRINTIFY_BASE_URL = "https://api.printify.com/v1"
@@ -23,13 +27,17 @@ REQUEST_TIMEOUT_SECONDS = 60
# Cheap endpoint used to confirm a token is genuine. Every shop-scoped stream needs the shop list
# anyway, so `shops.read` access is effectively required for the source to work at all.
DEFAULT_PROBE_PATH = "/shops.json"
+# Catalog endpoints share a separate 100 requests/minute limit per account. Catalog fan-out makes
+# hundreds of requests, so it paces itself rather than relying on 429 backoff. Each of the three
+# catalog fan-out tables takes a third of the limit, so they stay under it when they sync at once.
+CATALOG_REQUEST_INTERVAL_SECONDS = 1.8
class PrintifyRetryableError(Exception):
pass
[email protected](frozen=True)
class PrintifyResumeConfig:
# Shop currently being iterated for shop-scoped endpoints; None for account-level endpoints.
# A resumed sync re-fetches this shop from `page`; merge dedupes re-pulled rows on the
@@ -37,6 +45,17 @@ class PrintifyResumeConfig:
shop_id: int | None = None
# Next page to fetch (1-based, Laravel-style page numbers). Ignored by non-paginated endpoints.
page: int = 1
+ # Last catalog parent whose child rows were yielded, for catalog fan-out endpoints.
+ blueprint_id: int | None = None
+ print_provider_id: int | None = None
+ # Product fan-out walks each page in product id order; this is the last id on `page` it finished.
+ product_id: str | None = None
+
+
[email protected](frozen=True, order=True)
+class CatalogPair:
+ blueprint_id: int
+ print_provider_id: int
def _headers(api_key: str) -> dict[str, str]:
@@ -48,25 +67,21 @@ def _headers(api_key: str) -> dict[str, str]:
}
-@retry_with_deadline(
+_retry = retry_with_deadline(
retry=retry_if_exception_type((PrintifyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
reraise=True,
)
-def _fetch_page(
+
+
+def _request(
session: requests.Session,
path: str,
- page: int | None,
- limit: int | None,
+ params: dict[str, Any],
logger: FilteringBoundLogger,
-) -> tuple[list[dict[str, Any]], bool]:
- params: dict[str, Any] = {}
- if page is not None:
- params["page"] = page
- if limit is not None:
- params["limit"] = limit
-
+ allow_not_found: bool = False,
+) -> Any:
response = session.get(
f"{PRINTIFY_BASE_URL}{path}",
params=params or None,
@@ -77,6 +92,9 @@ def _fetch_page(
if response.status_code == 429 or response.status_code >= 500:
raise PrintifyRetryableError(f"Printify API error (retryable): status={response.status_code}, path={path}")
+ if allow_not_found and response.status_code == 404:
+ return None
+
if not response.ok:
logger.error(f"Printify API error: status={response.status_code}, body={response.text}, path={path}")
# raise_for_status() would embed the full request URL in the exception, which is surfaced
@@ -91,7 +109,24 @@ def _fetch_page(
response=response,
)
- data = response.json()
+ return response.json()
+
+
+@_retry
+def _fetch_page(
+ session: requests.Session,
+ path: str,
+ page: int | None,
+ limit: int | None,
+ logger: FilteringBoundLogger,
+) -> tuple[list[dict[str, Any]], bool]:
+ params: dict[str, Any] = {}
+ if page is not None:
+ params["page"] = page
+ if limit is not None:
+ params["limit"] = limit
+
+ data = _request(session, path, params, logger)
# Non-paginated endpoints (shops, webhooks, catalog) return a bare JSON array.
if isinstance(data, list):
@@ -160,11 +195,219 @@ def _iter_endpoint_pages(
resumable_source_manager.save_state(PrintifyResumeConfig(shop_id=shop_id, page=page))
+@_retry
+def _fetch_child(
+ session: requests.Session, path: str, params: dict[str, str], logger: FilteringBoundLogger
+) -> Any | None:
+ # A product can be deleted after the parent listing, and a product can keep a catalog
+ # blueprint/provider pair that Printify has since retired. Both answer 404; skip that parent.
+ return _request(session, path, params, logger, allow_not_found=True)
+
+
+def _shipping_rate_rows(payload: Any, parent: dict[str, Any]) -> list[dict[str, Any]]:
+ if not isinstance(payload, dict):
+ return []
+ rows: list[dict[str, Any]] = []
+ seen: set[tuple[Any, Any]] = set()
+ for profile in payload.get("profiles") or []:
+ for variant_id in profile.get("variant_ids") or []:
+ for country in profile.get("countries") or []:
+ # The primary key is (pair, variant, country); keep the first profile if two overlap.
+ if (variant_id, country) in seen:
+ continue
+ seen.add((variant_id, country))
+ rows.append(
+ {
+ **parent,
+ "variant_id": variant_id,
+ "country": country,
+ "handling_time": payload.get("handling_time"),
+ "first_item": profile.get("first_item"),
+ "additional_items": profile.get("additional_items"),
+ }
+ )
+ return rows
+
+
+def _shape_child_rows(endpoint: str, payload: Any, parent: dict[str, Any]) -> list[dict[str, Any]]:
+ if endpoint == "catalog_variants":
+ variants = payload.get("variants") if isinstance(payload, dict) else None
+ return [{**variant, **parent} for variant in variants or []]
+ if endpoint == "catalog_shipping_rates":
+ return _shipping_rate_rows(payload, parent)
+ if endpoint == "product_gpsr":
+ return [{**parent, "sections": payload}] if isinstance(payload, list) and payload else []
+ return [{**row, **parent} for row in payload] if isinstance(payload, list) else []
+
+
+def _iter_child_rows(
+ session: requests.Session,
+ config: PrintifyEndpointConfig,
+ parent: dict[str, Any],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+ state: PrintifyResumeConfig,
+) -> Iterator[list[dict[str, Any]]]:
+ path = config.path.format(**parent)
+ payload = _fetch_child(session, path, config.query_params, logger)
+ if payload is None:
+ logger.debug(f"Printify: {path} returned 404, skipping")
+ rows = _shape_child_rows(config.name, payload, parent) if payload is not None else []
+
+ resumable_source_manager.save_state(state)
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
+
+
+def _pace_catalog_request(resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig]) -> None:
+ interruptible_wait(CATALOG_REQUEST_INTERVAL_SECONDS, safe_point=resumable_source_manager.safe_point)
+
+
def _list_shops(session: requests.Session, logger: FilteringBoundLogger) -> list[dict[str, Any]]:
shops, _ = _fetch_page(session, PRINTIFY_ENDPOINTS["shops"].path, None, None, logger)
return shops
+def _iter_shop_product_pages(
+ session: requests.Session,
+ shop_id: int,
+ start_page: int,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+) -> Iterator[tuple[int, list[dict[str, Any]]]]:
+ products_config = PRINTIFY_ENDPOINTS["products"]
+ path = products_config.path.format(shop_id=shop_id)
+ page = start_page
+ while True:
+ products, has_more = _fetch_page(session, path, page, products_config.page_size, logger)
+ # The product listing yields no rows itself; every child row of earlier products is yielded.
+ resumable_source_manager.safe_point()
+ yield page, products
+ if not has_more or not products:
+ return
+ page += 1
+
+
+def _iter_blueprint_fan_out(
+ session: requests.Session,
+ config: PrintifyEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+ resume: PrintifyResumeConfig | None,
+) -> Iterator[list[dict[str, Any]]]:
+ blueprints, _ = _fetch_page(session, PRINTIFY_ENDPOINTS["blueprints"].path, None, None, logger)
+ resume_after = resume.blueprint_id if resume else None
+ # Sorted ids let a resumed sync skip completed blueprints even when the catalog changed.
+ for blueprint_id in sorted({blueprint["id"] for blueprint in blueprints}):
+ if resume_after is not None and blueprint_id <= resume_after:
+ continue
+ _pace_catalog_request(resumable_source_manager)
+ yield from _iter_child_rows(
+ session,
+ config,
+ {"blueprint_id": blueprint_id},
+ logger,
+ resumable_source_manager,
+ PrintifyResumeConfig(blueprint_id=blueprint_id),
+ )
+
+
+def _list_product_catalog_pairs(
+ session: requests.Session,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+) -> list[CatalogPair]:
+ pairs: set[CatalogPair] = set()
+ for shop in _list_shops(session, logger):
+ for _, products in _iter_shop_product_pages(session, shop["id"], 1, logger, resumable_source_manager):
+ for product in products:
+ blueprint_id = product.get("blueprint_id")
+ print_provider_id = product.get("print_provider_id")
+ if isinstance(blueprint_id, int) and isinstance(print_provider_id, int):
+ pairs.add(CatalogPair(blueprint_id, print_provider_id))
+ return sorted(pairs)
+
+
+def _iter_product_catalog_pair_fan_out(
+ session: requests.Session,
+ config: PrintifyEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+ resume: PrintifyResumeConfig | None,
+) -> Iterator[list[dict[str, Any]]]:
+ resume_after: CatalogPair | None = None
+ if resume and resume.blueprint_id is not None and resume.print_provider_id is not None:
+ resume_after = CatalogPair(resume.blueprint_id, resume.print_provider_id)
+
+ for pair in _list_product_catalog_pairs(session, logger, resumable_source_manager):
+ if resume_after is not None and pair <= resume_after:
+ continue
+ _pace_catalog_request(resumable_source_manager)
+ yield from _iter_child_rows(
+ session,
+ config,
+ dataclasses.asdict(pair),
+ logger,
+ resumable_source_manager,
+ PrintifyResumeConfig(blueprint_id=pair.blueprint_id, print_provider_id=pair.print_provider_id),
+ )
+
+
+def _iter_product_fan_out(
+ session: requests.Session,
+ config: PrintifyEndpointConfig,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],
+ resume: PrintifyResumeConfig | None,
+) -> Iterator[list[dict[str, Any]]]:
+ shops = _list_shops(session, logger)
+ resume_shop_id = resume.shop_id if resume else None
+ resume_page = resume.page if resume else 1
+ resume_product_id = resume.product_id if resume else None
+ if resume_shop_id is not None and all(shop["id"] != resume_shop_id for shop in shops):
+ logger.debug(f"Printify: saved shop {resume_shop_id} not found, restarting {config.name} from the first shop")
+ resume_shop_id = None
+
+ skipping = resume_shop_id is not None
+ for shop in shops:
+ shop_id = shop["id"]
+ skip_through_product_id: str | None = None
+ if skipping:
+ if shop_id != resume_shop_id:
+ continue
+ skipping = False
+ start_page = resume_page
+ skip_through_product_id = resume_product_id
+ else:
+ start_page = 1
+
+ for page, products in _iter_shop_product_pages(session, shop_id, start_page, logger, resumable_source_manager):
+ # Id order, not position, so a resume skips finished products even when the checkpoint
+ # product was deleted in the meantime.
+ products = sorted(products, key=lambda product: str(product["id"]))
+ if skip_through_product_id is not None:
+ products = [product for product in products if str(product["id"]) > skip_through_product_id]
+ skip_through_product_id = None
+ for product in products:
+ yield from _iter_child_rows(
+ session,
+ config,
+ {"shop_id": shop_id, "product_id": product["id"]},
+ logger,
+ resumable_source_manager,
+ PrintifyResumeConfig(shop_id=shop_id, page=page, product_id=str(product["id"])),
+ )
+
+
+_FAN_OUT_ITERATORS = {
+ PrintifyFanOut.BLUEPRINT: _iter_blueprint_fan_out,
+ PrintifyFanOut.PRODUCT_CATALOG_PAIR: _iter_product_catalog_pair_fan_out,
+ PrintifyFanOut.PRODUCT: _iter_product_fan_out,
+}
+
+
def get_rows(
api_key: str,
endpoint: str,
@@ -176,6 +419,12 @@ def get_rows(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if config.fan_out is not None:
+ if resume:
+ logger.debug(f"Printify: resuming {endpoint} from {resume}")
+ yield from _FAN_OUT_ITERATORS[config.fan_out](session, config, logger, resumable_source_manager, resume)
+ return
+
if not config.shop_scoped:
start_page = resume.page if resume else 1
if resume:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py
index ee289d15e3e4..bb5fa6223868 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py
@@ -1,7 +1,19 @@
from dataclasses import dataclass, field
+from enum import StrEnum
-@dataclass
+class PrintifyFanOut(StrEnum):
+ # One request per catalog blueprint.
+ BLUEPRINT = "blueprint"
+ # One request per (blueprint, print provider) pair used by a product in a connected shop. The full
+ # catalog has thousands of pairs behind a 100 requests/minute limit, and these pairs are the ones
+ # that resolve the variant ids on synced products and order line items.
+ PRODUCT_CATALOG_PAIR = "product_catalog_pair"
+ # One request per product in each connected shop.
+ PRODUCT = "product"
+
+
+@dataclass(frozen=True)
class PrintifyEndpointConfig:
name: str
# Path relative to the API base URL; `{shop_id}` is filled per shop for shop-scoped endpoints.
@@ -18,6 +30,9 @@ class PrintifyEndpointConfig:
# Fields stripped from rows before they're yielded (credentials that must not land in a
# warehouse table).
redact_fields: list[str] = field(default_factory=list)
+ # Child endpoints request one path per parent object; the path holds the parent's id placeholders.
+ fan_out: PrintifyFanOut | None = None
+ query_params: dict[str, str] = field(default_factory=dict)
# Printify REST API v1 list endpoints (https://developers.printify.com). All are full-refresh only:
@@ -59,6 +74,35 @@ PRINTIFY_ENDPOINTS: dict[str, PrintifyEndpointConfig] = {
),
"blueprints": PrintifyEndpointConfig(name="blueprints", path="/catalog/blueprints.json"),
"print_providers": PrintifyEndpointConfig(name="print_providers", path="/catalog/print_providers.json"),
+ "blueprint_print_providers": PrintifyEndpointConfig(
+ name="blueprint_print_providers",
+ path="/catalog/blueprints/{blueprint_id}/print_providers.json",
+ fan_out=PrintifyFanOut.BLUEPRINT,
+ primary_keys=["blueprint_id", "id"],
+ ),
+ # Variant ids are shared by every print provider of a blueprint, so the key needs both parents.
+ "catalog_variants": PrintifyEndpointConfig(
+ name="catalog_variants",
+ path="/catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants.json",
+ fan_out=PrintifyFanOut.PRODUCT_CATALOG_PAIR,
+ # Without it the API omits out-of-stock variants, which products and orders still reference.
+ query_params={"show-out-of-stock": "1"},
+ primary_keys=["blueprint_id", "print_provider_id", "id"],
+ ),
+ # v1 shipping profiles, exploded to one row per variant and destination country.
+ "catalog_shipping_rates": PrintifyEndpointConfig(
+ name="catalog_shipping_rates",
+ path="/catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping.json",
+ fan_out=PrintifyFanOut.PRODUCT_CATALOG_PAIR,
+ primary_keys=["blueprint_id", "print_provider_id", "variant_id", "country"],
+ ),
+ "product_gpsr": PrintifyEndpointConfig(
+ name="product_gpsr",
+ path="/shops/{shop_id}/products/{product_id}/gpsr.json",
+ shop_scoped=True,
+ fan_out=PrintifyFanOut.PRODUCT,
+ primary_keys=["shop_id", "product_id"],
+ ),
}
ENDPOINTS = tuple(PRINTIFY_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py
index 9e379d393606..4c987218787f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py
@@ -32,8 +32,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.printify.s
# Call the undecorated function so the tenacity retry/backoff wrapper doesn't slow failure-path tests.
_fetch_page_unwrapped = printify._fetch_page.__wrapped__ # type: ignore[attr-defined]
-# (path, page) -> (items, has_more), the shape the vendor answers per request.
-_Pages = dict[tuple[str, int | None], tuple[list[dict], bool]]
+# (path, page) -> (body, has_more), the shape the vendor answers per request. Unpaginated bodies are
+# returned as-is; a body of None answers 404.
+_Pages = dict[tuple[str, int | None], tuple[Any, bool]]
def _script(pages: _Pages) -> Callable[[RecordedRequest], ScriptedResponse]:
@@ -45,6 +46,8 @@ def _script(pages: _Pages) -> Callable[[RecordedRequest], ScriptedResponse]:
if key not in pages:
raise UnexpectedRequest(f"the script has no page for {key}")
items, has_more = pages[key]
+ if items is None:
+ return ScriptedResponse(status=404, json={"error": "Not found"})
if key[1] is None:
return ScriptedResponse(json=items)
current = key[1]
@@ -229,6 +232,146 @@ class TestGetRows:
assert [(s.shop_id, s.page) for s in result.saved_states] == [(111, 1), (222, 1)]
+class TestFanOut:
+ @staticmethod
+ def _run(pages: _Pages, endpoint: str, resume_state: PrintifyResumeConfig | None = None) -> DriveResult:
+ result = _driver().run(endpoint, _script(pages), resume_state=resume_state)
+ assert result.raised is None
+ return result
+
+ _SHOPS_WITH_PRODUCTS: _Pages = {
+ ("/shops.json", None): ([{"id": 111}, {"id": 222}], False),
+ ("/shops/111/products.json", 1): (
+ [
+ {"id": "p1", "blueprint_id": 6, "print_provider_id": 3},
+ {"id": "p2", "blueprint_id": 5, "print_provider_id": 9},
+ ],
+ True,
+ ),
+ ("/shops/111/products.json", 2): ([{"id": "p3", "blueprint_id": 6, "print_provider_id": 3}], False),
+ ("/shops/222/products.json", 1): ([{"id": "p4", "blueprint_id": 6, "print_provider_id": None}], False),
+ }
+
+ @parameterized.expand(
+ [
+ ("from_the_start", None, [3, 7]),
+ ("resume_after_saved_blueprint", PrintifyResumeConfig(blueprint_id=3), [7]),
+ ]
+ )
+ def test_blueprint_fan_out_walks_sorted_blueprints_and_skips_404(
+ self, _name: str, resume: PrintifyResumeConfig | None, expected_blueprints: list[int]
+ ) -> None:
+ pages: _Pages = {
+ ("/catalog/blueprints.json", None): ([{"id": 7}, {"id": 3}, {"id": 5}], False),
+ ("/catalog/blueprints/3/print_providers.json", None): ([{"id": 1, "title": "DJ"}], False),
+ ("/catalog/blueprints/5/print_providers.json", None): (None, False),
+ ("/catalog/blueprints/7/print_providers.json", None): ([{"id": 1, "title": "DJ"}], False),
+ }
+ result = self._run(pages, "blueprint_print_providers", resume)
+ assert [(r["blueprint_id"], r["id"]) for r in result.rows] == [(b, 1) for b in expected_blueprints]
+ if resume is None:
+ assert [s.blueprint_id for s in result.saved_states] == [3, 5, 7]
+
+ def test_catalog_pair_fan_out_requests_each_product_pair_once(self) -> None:
+ pages: _Pages = {
+ **self._SHOPS_WITH_PRODUCTS,
+ ("/catalog/blueprints/5/print_providers/9/variants.json", None): (
+ {"id": 9, "variants": [{"id": 501, "title": "Black / S"}]},
+ False,
+ ),
+ ("/catalog/blueprints/6/print_providers/3/variants.json", None): (
+ {"id": 3, "variants": [{"id": 601, "title": "White / M"}, {"id": 602, "title": "White / L"}]},
+ False,
+ ),
+ }
+ result = self._run(pages, "catalog_variants")
+ assert [(r["blueprint_id"], r["print_provider_id"], r["id"]) for r in result.rows] == [
+ (5, 9, 501),
+ (6, 3, 601),
+ (6, 3, 602),
+ ]
+ variant_requests = [r for r in result.requests if r.path.endswith("variants.json")]
+ assert [r.param("show-out-of-stock") for r in variant_requests] == ["1", "1"]
+
+ def test_catalog_pair_fan_out_resumes_after_saved_pair(self) -> None:
+ pages: _Pages = {
+ **self._SHOPS_WITH_PRODUCTS,
+ ("/catalog/blueprints/6/print_providers/3/variants.json", None): (
+ {"id": 3, "variants": [{"id": 601}]},
+ False,
+ ),
+ }
+ result = self._run(pages, "catalog_variants", PrintifyResumeConfig(blueprint_id=5, print_provider_id=9))
+ assert [r["id"] for r in result.rows] == [601]
+
+ def test_shipping_profiles_explode_to_one_row_per_variant_and_country(self) -> None:
+ pages: _Pages = {
+ ("/shops.json", None): ([{"id": 111}], False),
+ ("/shops/111/products.json", 1): ([{"id": "p1", "blueprint_id": 6, "print_provider_id": 3}], False),
+ ("/catalog/blueprints/6/print_providers/3/shipping.json", None): (
+ {
+ "handling_time": {"value": 30, "unit": "day"},
+ "profiles": [
+ {
+ "variant_ids": [601, 602],
+ "first_item": {"cost": 450, "currency": "USD"},
+ "additional_items": {"cost": 0, "currency": "USD"},
+ "countries": ["US", "CA"],
+ },
+ {
+ "variant_ids": [601],
+ "first_item": {"cost": 999, "currency": "USD"},
+ "additional_items": {"cost": 0, "currency": "USD"},
+ "countries": ["US", "REST_OF_THE_WORLD"],
+ },
+ ],
+ },
+ False,
+ ),
+ }
+ result = self._run(pages, "catalog_shipping_rates")
+ assert [(r["variant_id"], r["country"], r["first_item"]["cost"]) for r in result.rows] == [
+ (601, "US", 450),
+ (601, "CA", 450),
+ (602, "US", 450),
+ (602, "CA", 450),
+ (601, "REST_OF_THE_WORLD", 999),
+ ]
+ assert all(r["handling_time"] == {"value": 30, "unit": "day"} for r in result.rows)
+
+ def test_product_gpsr_yields_one_row_per_product_with_sections(self) -> None:
+ pages: _Pages = {
+ ("/shops.json", None): ([{"id": 111}], False),
+ ("/shops/111/products.json", 1): ([{"id": "p1"}, {"id": "p2"}], True),
+ ("/shops/111/products.json", 2): ([{"id": "p3"}], False),
+ ("/shops/111/products/p1/gpsr.json", None): ([{"title": "Care instructions", "text": "Wash"}], False),
+ ("/shops/111/products/p2/gpsr.json", None): ([], False),
+ ("/shops/111/products/p3/gpsr.json", None): (None, False),
+ }
+ result = self._run(pages, "product_gpsr")
+ assert result.rows == [
+ {"shop_id": 111, "product_id": "p1", "sections": [{"title": "Care instructions", "text": "Wash"}]}
+ ]
+ assert [(s.shop_id, s.page, s.product_id) for s in result.saved_states] == [
+ (111, 1, "p1"),
+ (111, 1, "p2"),
+ (111, 2, "p3"),
+ ]
+
+ @parameterized.expand(
+ [("checkpoint_product_present", ["p3", "p1", "p2"]), ("checkpoint_product_deleted", ["p3", "p1"])]
+ )
+ def test_product_gpsr_resumes_after_saved_product_on_saved_page(self, _name: str, page_ids: list[str]) -> None:
+ # p1's and p2's GPSR pages are absent, so fetching either again would fail the run.
+ pages: _Pages = {
+ ("/shops.json", None): ([{"id": 111}], False),
+ ("/shops/111/products.json", 1): ([{"id": product_id} for product_id in page_ids], False),
+ ("/shops/111/products/p3/gpsr.json", None): ([{"title": "Warnings", "text": "None"}], False),
+ }
+ result = self._run(pages, "product_gpsr", PrintifyResumeConfig(shop_id=111, page=1, product_id="p2"))
+ assert [r["product_id"] for r in result.rows] == ["p3"]
+
+
class TestPrintifySourceResponse:
@parameterized.expand([(e,) for e in ENDPOINTS])
def test_source_response_shape(self, endpoint: str) -> None:
@@ -240,5 +383,7 @@ class TestPrintifySourceResponse:
def test_shop_scoped_endpoints_use_composite_primary_key(self) -> None:
for config in PRINTIFY_ENDPOINTS.values():
+ if config.fan_out is not None:
+ continue
expected = ["shop_id", "id"] if config.shop_scoped else ["id"]
assert config.primary_keys == expected, config.name