:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add printify catalog variant, shipping, and gpsr tables (#126782)
Tom Owers5763774feb96
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex db58865432c9..82f325437c4d 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -871,8 +871,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/prefect_cloud/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pretix/pretix.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pretix/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.py1 products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7cf66a363a04..78412f5fa43d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9763,10 +9763,10 @@ Today (7): `blueprints`, `orders`, `print_providers`, `products`, `shops`, `uploDiffed against: <https://developers.printify.com/>-- [ ] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants` — the variant lookup table that resolves the variant_id carried on every synced product and order line item (size/color/SKU/price) (high)-- [ ] `catalog/blueprints/{blueprint_id}/print_providers` — blueprint-to-print-provider mapping table; joins the two catalog tables already synced (medium)-- [ ] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping (and the v2 shipping/standard|express|priority|economy variants)` — shipping cost and handling time per variant/country, needed for order margin analysis (medium)-- [ ] `shops/{shop_id}/products/{product_id}/gpsr` — product safety/compliance attributes per product (low)+- [x] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants` — the variant lookup table that resolves the variant_id carried on every synced product and order line item (size/color/SKU/price) (high)+- [x] `catalog/blueprints/{blueprint_id}/print_providers` — blueprint-to-print-provider mapping table; joins the two catalog tables already synced (medium)+- [x] `catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping (and the v2 shipping/standard|express|priority|economy variants)` — shipping cost and handling time per variant/country, needed for order margin analysis (medium)+- [x] `shops/{shop_id}/products/{product_id}/gpsr` — product safety/compliance attributes per product (low)Note: Single-page reference; v1 covers catalog/shops/products/orders/uploads/webhooks, and a v2 catalog shipping family (shipping/standard, /express, /priority, /economy) exists alongside the v1 shipping endpoint. Order line items are nested inside the orders payload, so they are not a separate gap.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.pyindex bc1c1b668848..62aa3900476e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/canonical_descriptions.py@@ -100,4 +100,50 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"location": "The print provider's location address.",},},+ "blueprint_print_providers": {+ "description": "Which print providers fulfill each catalog blueprint, with their decoration methods and production time.",+ "docs_url": "https://developers.printify.com/#catalog",+ "columns": {+ "id": "The ID of the print provider.",+ "blueprint_id": "The ID of the catalog blueprint the print provider fulfills.",+ "title": "The print provider's name.",+ "decoration_methods": "Decoration methods the print provider offers for the blueprint (e.g. dtg, embroidery).",+ "production_time": "The print provider's average production time in business days.",+ },+ },+ "catalog_variants": {+ "description": "Catalog variants (e.g. color and size combinations) for each blueprint and print provider used by a product in a connected shop. Includes out-of-stock variants.",+ "docs_url": "https://developers.printify.com/#catalog",+ "columns": {+ "id": "The variant ID, referenced as variant_id on product variants and order line items.",+ "blueprint_id": "The ID of the catalog blueprint.",+ "print_provider_id": "The ID of the print provider.",+ "title": "The variant's name (e.g. Heather Grey / XS).",+ "options": "The option values of the variant, such as color and size.",+ "placeholders": "Print areas available on the variant, with their dimensions and decoration method.",+ "decoration_methods": "Decoration methods available for the variant.",+ },+ },+ "catalog_shipping_rates": {+ "description": "Shipping cost and handling time for each variant and destination country, for each blueprint and print provider used by a product in a connected shop.",+ "docs_url": "https://developers.printify.com/#catalog",+ "columns": {+ "blueprint_id": "The ID of the catalog blueprint.",+ "print_provider_id": "The ID of the print provider.",+ "variant_id": "The ID of the catalog variant.",+ "country": "The destination country code, or REST_OF_THE_WORLD.",+ "handling_time": "The print provider's handling time for the blueprint.",+ "first_item": "The shipping cost of the first item, in the smallest currency unit, and its currency.",+ "additional_items": "The shipping cost of each additional item, in the smallest currency unit, and its currency.",+ },+ },+ "product_gpsr": {+ "description": "General Product Safety Regulation (GPSR) information for each product in a connected shop.",+ "docs_url": "https://developers.printify.com/#products",+ "columns": {+ "shop_id": "The ID of the shop the product belongs to.",+ "product_id": "The ID of the product.",+ "sections": "GPSR sections, each with a title (e.g. EU representative, Care instructions) and text.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.pyindex e44cc2d8e983..00d4da28ff71 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py@@ -11,11 +11,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httmake_tracked_session,retry_with_deadline,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.printify.settings import (PRINTIFY_ENDPOINTS,PrintifyEndpointConfig,+ PrintifyFanOut,)PRINTIFY_BASE_URL = "https://api.printify.com/v1"@@ -23,13 +27,17 @@ REQUEST_TIMEOUT_SECONDS = 60# Cheap endpoint used to confirm a token is genuine. Every shop-scoped stream needs the shop list# anyway, so `shops.read` access is effectively required for the source to work at all.DEFAULT_PROBE_PATH = "/shops.json"+# Catalog endpoints share a separate 100 requests/minute limit per account. Catalog fan-out makes+# hundreds of requests, so it paces itself rather than relying on 429 backoff. Each of the three+# catalog fan-out tables takes a third of the limit, so they stay under it when they sync at once.+CATALOG_REQUEST_INTERVAL_SECONDS = 1.8class PrintifyRetryableError(Exception):pass[email protected](frozen=True)class PrintifyResumeConfig:# Shop currently being iterated for shop-scoped endpoints; None for account-level endpoints.# A resumed sync re-fetches this shop from `page`; merge dedupes re-pulled rows on the@@ -37,6 +45,17 @@ class PrintifyResumeConfig:shop_id: int | None = None# Next page to fetch (1-based, Laravel-style page numbers). Ignored by non-paginated endpoints.page: int = 1+ # Last catalog parent whose child rows were yielded, for catalog fan-out endpoints.+ blueprint_id: int | None = None+ print_provider_id: int | None = None+ # Product fan-out walks each page in product id order; this is the last id on `page` it finished.+ product_id: str | None = None++[email protected](frozen=True, order=True)+class CatalogPair:+ blueprint_id: int+ print_provider_id: intdef _headers(api_key: str) -> dict[str, str]:@@ -48,25 +67,21 @@ def _headers(api_key: str) -> dict[str, str]:}-@retry_with_deadline(+_retry = retry_with_deadline(retry=retry_if_exception_type((PrintifyRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),reraise=True,)-def _fetch_page(+++def _request(session: requests.Session,path: str,- page: int | None,- limit: int | None,+ params: dict[str, Any],logger: FilteringBoundLogger,-) -> tuple[list[dict[str, Any]], bool]:- params: dict[str, Any] = {}- if page is not None:- params["page"] = page- if limit is not None:- params["limit"] = limit-+ allow_not_found: bool = False,+) -> Any:response = session.get(f"{PRINTIFY_BASE_URL}{path}",params=params or None,@@ -77,6 +92,9 @@ def _fetch_page(if response.status_code == 429 or response.status_code >= 500:raise PrintifyRetryableError(f"Printify API error (retryable): status={response.status_code}, path={path}")+ if allow_not_found and response.status_code == 404:+ return None+if not response.ok:logger.error(f"Printify API error: status={response.status_code}, body={response.text}, path={path}")# raise_for_status() would embed the full request URL in the exception, which is surfaced@@ -91,7 +109,24 @@ def _fetch_page(response=response,)- data = response.json()+ return response.json()+++@_retry+def _fetch_page(+ session: requests.Session,+ path: str,+ page: int | None,+ limit: int | None,+ logger: FilteringBoundLogger,+) -> tuple[list[dict[str, Any]], bool]:+ params: dict[str, Any] = {}+ if page is not None:+ params["page"] = page+ if limit is not None:+ params["limit"] = limit++ data = _request(session, path, params, logger)# Non-paginated endpoints (shops, webhooks, catalog) return a bare JSON array.if isinstance(data, list):@@ -160,11 +195,219 @@ def _iter_endpoint_pages(resumable_source_manager.save_state(PrintifyResumeConfig(shop_id=shop_id, page=page))+@_retry+def _fetch_child(+ session: requests.Session, path: str, params: dict[str, str], logger: FilteringBoundLogger+) -> Any | None:+ # A product can be deleted after the parent listing, and a product can keep a catalog+ # blueprint/provider pair that Printify has since retired. Both answer 404; skip that parent.+ return _request(session, path, params, logger, allow_not_found=True)+++def _shipping_rate_rows(payload: Any, parent: dict[str, Any]) -> list[dict[str, Any]]:+ if not isinstance(payload, dict):+ return []+ rows: list[dict[str, Any]] = []+ seen: set[tuple[Any, Any]] = set()+ for profile in payload.get("profiles") or []:+ for variant_id in profile.get("variant_ids") or []:+ for country in profile.get("countries") or []:+ # The primary key is (pair, variant, country); keep the first profile if two overlap.+ if (variant_id, country) in seen:+ continue+ seen.add((variant_id, country))+ rows.append(+ {+ **parent,+ "variant_id": variant_id,+ "country": country,+ "handling_time": payload.get("handling_time"),+ "first_item": profile.get("first_item"),+ "additional_items": profile.get("additional_items"),+ }+ )+ return rows+++def _shape_child_rows(endpoint: str, payload: Any, parent: dict[str, Any]) -> list[dict[str, Any]]:+ if endpoint == "catalog_variants":+ variants = payload.get("variants") if isinstance(payload, dict) else None+ return [{**variant, **parent} for variant in variants or []]+ if endpoint == "catalog_shipping_rates":+ return _shipping_rate_rows(payload, parent)+ if endpoint == "product_gpsr":+ return [{**parent, "sections": payload}] if isinstance(payload, list) and payload else []+ return [{**row, **parent} for row in payload] if isinstance(payload, list) else []+++def _iter_child_rows(+ session: requests.Session,+ config: PrintifyEndpointConfig,+ parent: dict[str, Any],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+ state: PrintifyResumeConfig,+) -> Iterator[list[dict[str, Any]]]:+ path = config.path.format(**parent)+ payload = _fetch_child(session, path, config.query_params, logger)+ if payload is None:+ logger.debug(f"Printify: {path} returned 404, skipping")+ rows = _shape_child_rows(config.name, payload, parent) if payload is not None else []++ resumable_source_manager.save_state(state)+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()+++def _pace_catalog_request(resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig]) -> None:+ interruptible_wait(CATALOG_REQUEST_INTERVAL_SECONDS, safe_point=resumable_source_manager.safe_point)++def _list_shops(session: requests.Session, logger: FilteringBoundLogger) -> list[dict[str, Any]]:shops, _ = _fetch_page(session, PRINTIFY_ENDPOINTS["shops"].path, None, None, logger)return shops+def _iter_shop_product_pages(+ session: requests.Session,+ shop_id: int,+ start_page: int,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+) -> Iterator[tuple[int, list[dict[str, Any]]]]:+ products_config = PRINTIFY_ENDPOINTS["products"]+ path = products_config.path.format(shop_id=shop_id)+ page = start_page+ while True:+ products, has_more = _fetch_page(session, path, page, products_config.page_size, logger)+ # The product listing yields no rows itself; every child row of earlier products is yielded.+ resumable_source_manager.safe_point()+ yield page, products+ if not has_more or not products:+ return+ page += 1+++def _iter_blueprint_fan_out(+ session: requests.Session,+ config: PrintifyEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+ resume: PrintifyResumeConfig | None,+) -> Iterator[list[dict[str, Any]]]:+ blueprints, _ = _fetch_page(session, PRINTIFY_ENDPOINTS["blueprints"].path, None, None, logger)+ resume_after = resume.blueprint_id if resume else None+ # Sorted ids let a resumed sync skip completed blueprints even when the catalog changed.+ for blueprint_id in sorted({blueprint["id"] for blueprint in blueprints}):+ if resume_after is not None and blueprint_id <= resume_after:+ continue+ _pace_catalog_request(resumable_source_manager)+ yield from _iter_child_rows(+ session,+ config,+ {"blueprint_id": blueprint_id},+ logger,+ resumable_source_manager,+ PrintifyResumeConfig(blueprint_id=blueprint_id),+ )+++def _list_product_catalog_pairs(+ session: requests.Session,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+) -> list[CatalogPair]:+ pairs: set[CatalogPair] = set()+ for shop in _list_shops(session, logger):+ for _, products in _iter_shop_product_pages(session, shop["id"], 1, logger, resumable_source_manager):+ for product in products:+ blueprint_id = product.get("blueprint_id")+ print_provider_id = product.get("print_provider_id")+ if isinstance(blueprint_id, int) and isinstance(print_provider_id, int):+ pairs.add(CatalogPair(blueprint_id, print_provider_id))+ return sorted(pairs)+++def _iter_product_catalog_pair_fan_out(+ session: requests.Session,+ config: PrintifyEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+ resume: PrintifyResumeConfig | None,+) -> Iterator[list[dict[str, Any]]]:+ resume_after: CatalogPair | None = None+ if resume and resume.blueprint_id is not None and resume.print_provider_id is not None:+ resume_after = CatalogPair(resume.blueprint_id, resume.print_provider_id)++ for pair in _list_product_catalog_pairs(session, logger, resumable_source_manager):+ if resume_after is not None and pair <= resume_after:+ continue+ _pace_catalog_request(resumable_source_manager)+ yield from _iter_child_rows(+ session,+ config,+ dataclasses.asdict(pair),+ logger,+ resumable_source_manager,+ PrintifyResumeConfig(blueprint_id=pair.blueprint_id, print_provider_id=pair.print_provider_id),+ )+++def _iter_product_fan_out(+ session: requests.Session,+ config: PrintifyEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[PrintifyResumeConfig],+ resume: PrintifyResumeConfig | None,+) -> Iterator[list[dict[str, Any]]]:+ shops = _list_shops(session, logger)+ resume_shop_id = resume.shop_id if resume else None+ resume_page = resume.page if resume else 1+ resume_product_id = resume.product_id if resume else None+ if resume_shop_id is not None and all(shop["id"] != resume_shop_id for shop in shops):+ logger.debug(f"Printify: saved shop {resume_shop_id} not found, restarting {config.name} from the first shop")+ resume_shop_id = None++ skipping = resume_shop_id is not None+ for shop in shops:+ shop_id = shop["id"]+ skip_through_product_id: str | None = None+ if skipping:+ if shop_id != resume_shop_id:+ continue+ skipping = False+ start_page = resume_page+ skip_through_product_id = resume_product_id+ else:+ start_page = 1++ for page, products in _iter_shop_product_pages(session, shop_id, start_page, logger, resumable_source_manager):+ # Id order, not position, so a resume skips finished products even when the checkpoint+ # product was deleted in the meantime.+ products = sorted(products, key=lambda product: str(product["id"]))+ if skip_through_product_id is not None:+ products = [product for product in products if str(product["id"]) > skip_through_product_id]+ skip_through_product_id = None+ for product in products:+ yield from _iter_child_rows(+ session,+ config,+ {"shop_id": shop_id, "product_id": product["id"]},+ logger,+ resumable_source_manager,+ PrintifyResumeConfig(shop_id=shop_id, page=page, product_id=str(product["id"])),+ )+++_FAN_OUT_ITERATORS = {+ PrintifyFanOut.BLUEPRINT: _iter_blueprint_fan_out,+ PrintifyFanOut.PRODUCT_CATALOG_PAIR: _iter_product_catalog_pair_fan_out,+ PrintifyFanOut.PRODUCT: _iter_product_fan_out,+}++def get_rows(api_key: str,endpoint: str,@@ -176,6 +419,12 @@ def get_rows(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if config.fan_out is not None:+ if resume:+ logger.debug(f"Printify: resuming {endpoint} from {resume}")+ yield from _FAN_OUT_ITERATORS[config.fan_out](session, config, logger, resumable_source_manager, resume)+ return+if not config.shop_scoped:start_page = resume.page if resume else 1if resume:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.pyindex ee289d15e3e4..bb5fa6223868 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/settings.py@@ -1,7 +1,19 @@from dataclasses import dataclass, field+from enum import StrEnum-@dataclass+class PrintifyFanOut(StrEnum):+ # One request per catalog blueprint.+ BLUEPRINT = "blueprint"+ # One request per (blueprint, print provider) pair used by a product in a connected shop. The full+ # catalog has thousands of pairs behind a 100 requests/minute limit, and these pairs are the ones+ # that resolve the variant ids on synced products and order line items.+ PRODUCT_CATALOG_PAIR = "product_catalog_pair"+ # One request per product in each connected shop.+ PRODUCT = "product"+++@dataclass(frozen=True)class PrintifyEndpointConfig:name: str# Path relative to the API base URL; `{shop_id}` is filled per shop for shop-scoped endpoints.@@ -18,6 +30,9 @@ class PrintifyEndpointConfig:# Fields stripped from rows before they're yielded (credentials that must not land in a# warehouse table).redact_fields: list[str] = field(default_factory=list)+ # Child endpoints request one path per parent object; the path holds the parent's id placeholders.+ fan_out: PrintifyFanOut | None = None+ query_params: dict[str, str] = field(default_factory=dict)# Printify REST API v1 list endpoints (https://developers.printify.com). All are full-refresh only:@@ -59,6 +74,35 @@ PRINTIFY_ENDPOINTS: dict[str, PrintifyEndpointConfig] = {),"blueprints": PrintifyEndpointConfig(name="blueprints", path="/catalog/blueprints.json"),"print_providers": PrintifyEndpointConfig(name="print_providers", path="/catalog/print_providers.json"),+ "blueprint_print_providers": PrintifyEndpointConfig(+ name="blueprint_print_providers",+ path="/catalog/blueprints/{blueprint_id}/print_providers.json",+ fan_out=PrintifyFanOut.BLUEPRINT,+ primary_keys=["blueprint_id", "id"],+ ),+ # Variant ids are shared by every print provider of a blueprint, so the key needs both parents.+ "catalog_variants": PrintifyEndpointConfig(+ name="catalog_variants",+ path="/catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/variants.json",+ fan_out=PrintifyFanOut.PRODUCT_CATALOG_PAIR,+ # Without it the API omits out-of-stock variants, which products and orders still reference.+ query_params={"show-out-of-stock": "1"},+ primary_keys=["blueprint_id", "print_provider_id", "id"],+ ),+ # v1 shipping profiles, exploded to one row per variant and destination country.+ "catalog_shipping_rates": PrintifyEndpointConfig(+ name="catalog_shipping_rates",+ path="/catalog/blueprints/{blueprint_id}/print_providers/{print_provider_id}/shipping.json",+ fan_out=PrintifyFanOut.PRODUCT_CATALOG_PAIR,+ primary_keys=["blueprint_id", "print_provider_id", "variant_id", "country"],+ ),+ "product_gpsr": PrintifyEndpointConfig(+ name="product_gpsr",+ path="/shops/{shop_id}/products/{product_id}/gpsr.json",+ shop_scoped=True,+ fan_out=PrintifyFanOut.PRODUCT,+ primary_keys=["shop_id", "product_id"],+ ),}ENDPOINTS = tuple(PRINTIFY_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.pyindex 9e379d393606..4c987218787f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/tests/test_printify.py@@ -32,8 +32,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.printify.s# Call the undecorated function so the tenacity retry/backoff wrapper doesn't slow failure-path tests._fetch_page_unwrapped = printify._fetch_page.__wrapped__ # type: ignore[attr-defined]-# (path, page) -> (items, has_more), the shape the vendor answers per request.-_Pages = dict[tuple[str, int | None], tuple[list[dict], bool]]+# (path, page) -> (body, has_more), the shape the vendor answers per request. Unpaginated bodies are+# returned as-is; a body of None answers 404.+_Pages = dict[tuple[str, int | None], tuple[Any, bool]]def _script(pages: _Pages) -> Callable[[RecordedRequest], ScriptedResponse]:@@ -45,6 +46,8 @@ def _script(pages: _Pages) -> Callable[[RecordedRequest], ScriptedResponse]:if key not in pages:raise UnexpectedRequest(f"the script has no page for {key}")items, has_more = pages[key]+ if items is None:+ return ScriptedResponse(status=404, json={"error": "Not found"})if key[1] is None:return ScriptedResponse(json=items)current = key[1]@@ -229,6 +232,146 @@ class TestGetRows:assert [(s.shop_id, s.page) for s in result.saved_states] == [(111, 1), (222, 1)]+class TestFanOut:+ @staticmethod+ def _run(pages: _Pages, endpoint: str, resume_state: PrintifyResumeConfig | None = None) -> DriveResult:+ result = _driver().run(endpoint, _script(pages), resume_state=resume_state)+ assert result.raised is None+ return result++ _SHOPS_WITH_PRODUCTS: _Pages = {+ ("/shops.json", None): ([{"id": 111}, {"id": 222}], False),+ ("/shops/111/products.json", 1): (+ [+ {"id": "p1", "blueprint_id": 6, "print_provider_id": 3},+ {"id": "p2", "blueprint_id": 5, "print_provider_id": 9},+ ],+ True,+ ),+ ("/shops/111/products.json", 2): ([{"id": "p3", "blueprint_id": 6, "print_provider_id": 3}], False),+ ("/shops/222/products.json", 1): ([{"id": "p4", "blueprint_id": 6, "print_provider_id": None}], False),+ }++ @parameterized.expand(+ [+ ("from_the_start", None, [3, 7]),+ ("resume_after_saved_blueprint", PrintifyResumeConfig(blueprint_id=3), [7]),+ ]+ )+ def test_blueprint_fan_out_walks_sorted_blueprints_and_skips_404(+ self, _name: str, resume: PrintifyResumeConfig | None, expected_blueprints: list[int]+ ) -> None:+ pages: _Pages = {+ ("/catalog/blueprints.json", None): ([{"id": 7}, {"id": 3}, {"id": 5}], False),+ ("/catalog/blueprints/3/print_providers.json", None): ([{"id": 1, "title": "DJ"}], False),+ ("/catalog/blueprints/5/print_providers.json", None): (None, False),+ ("/catalog/blueprints/7/print_providers.json", None): ([{"id": 1, "title": "DJ"}], False),+ }+ result = self._run(pages, "blueprint_print_providers", resume)+ assert [(r["blueprint_id"], r["id"]) for r in result.rows] == [(b, 1) for b in expected_blueprints]+ if resume is None:+ assert [s.blueprint_id for s in result.saved_states] == [3, 5, 7]++ def test_catalog_pair_fan_out_requests_each_product_pair_once(self) -> None:+ pages: _Pages = {+ **self._SHOPS_WITH_PRODUCTS,+ ("/catalog/blueprints/5/print_providers/9/variants.json", None): (+ {"id": 9, "variants": [{"id": 501, "title": "Black / S"}]},+ False,+ ),+ ("/catalog/blueprints/6/print_providers/3/variants.json", None): (+ {"id": 3, "variants": [{"id": 601, "title": "White / M"}, {"id": 602, "title": "White / L"}]},+ False,+ ),+ }+ result = self._run(pages, "catalog_variants")+ assert [(r["blueprint_id"], r["print_provider_id"], r["id"]) for r in result.rows] == [+ (5, 9, 501),+ (6, 3, 601),+ (6, 3, 602),+ ]+ variant_requests = [r for r in result.requests if r.path.endswith("variants.json")]+ assert [r.param("show-out-of-stock") for r in variant_requests] == ["1", "1"]++ def test_catalog_pair_fan_out_resumes_after_saved_pair(self) -> None:+ pages: _Pages = {+ **self._SHOPS_WITH_PRODUCTS,+ ("/catalog/blueprints/6/print_providers/3/variants.json", None): (+ {"id": 3, "variants": [{"id": 601}]},+ False,+ ),+ }+ result = self._run(pages, "catalog_variants", PrintifyResumeConfig(blueprint_id=5, print_provider_id=9))+ assert [r["id"] for r in result.rows] == [601]++ def test_shipping_profiles_explode_to_one_row_per_variant_and_country(self) -> None:+ pages: _Pages = {+ ("/shops.json", None): ([{"id": 111}], False),+ ("/shops/111/products.json", 1): ([{"id": "p1", "blueprint_id": 6, "print_provider_id": 3}], False),+ ("/catalog/blueprints/6/print_providers/3/shipping.json", None): (+ {+ "handling_time": {"value": 30, "unit": "day"},+ "profiles": [+ {+ "variant_ids": [601, 602],+ "first_item": {"cost": 450, "currency": "USD"},+ "additional_items": {"cost": 0, "currency": "USD"},+ "countries": ["US", "CA"],+ },+ {+ "variant_ids": [601],+ "first_item": {"cost": 999, "currency": "USD"},+ "additional_items": {"cost": 0, "currency": "USD"},+ "countries": ["US", "REST_OF_THE_WORLD"],+ },+ ],+ },+ False,+ ),+ }+ result = self._run(pages, "catalog_shipping_rates")+ assert [(r["variant_id"], r["country"], r["first_item"]["cost"]) for r in result.rows] == [+ (601, "US", 450),+ (601, "CA", 450),+ (602, "US", 450),+ (602, "CA", 450),+ (601, "REST_OF_THE_WORLD", 999),+ ]+ assert all(r["handling_time"] == {"value": 30, "unit": "day"} for r in result.rows)++ def test_product_gpsr_yields_one_row_per_product_with_sections(self) -> None:+ pages: _Pages = {+ ("/shops.json", None): ([{"id": 111}], False),+ ("/shops/111/products.json", 1): ([{"id": "p1"}, {"id": "p2"}], True),+ ("/shops/111/products.json", 2): ([{"id": "p3"}], False),+ ("/shops/111/products/p1/gpsr.json", None): ([{"title": "Care instructions", "text": "Wash"}], False),+ ("/shops/111/products/p2/gpsr.json", None): ([], False),+ ("/shops/111/products/p3/gpsr.json", None): (None, False),+ }+ result = self._run(pages, "product_gpsr")+ assert result.rows == [+ {"shop_id": 111, "product_id": "p1", "sections": [{"title": "Care instructions", "text": "Wash"}]}+ ]+ assert [(s.shop_id, s.page, s.product_id) for s in result.saved_states] == [+ (111, 1, "p1"),+ (111, 1, "p2"),+ (111, 2, "p3"),+ ]++ @parameterized.expand(+ [("checkpoint_product_present", ["p3", "p1", "p2"]), ("checkpoint_product_deleted", ["p3", "p1"])]+ )+ def test_product_gpsr_resumes_after_saved_product_on_saved_page(self, _name: str, page_ids: list[str]) -> None:+ # p1's and p2's GPSR pages are absent, so fetching either again would fail the run.+ pages: _Pages = {+ ("/shops.json", None): ([{"id": 111}], False),+ ("/shops/111/products.json", 1): ([{"id": product_id} for product_id in page_ids], False),+ ("/shops/111/products/p3/gpsr.json", None): ([{"title": "Warnings", "text": "None"}], False),+ }+ result = self._run(pages, "product_gpsr", PrintifyResumeConfig(shop_id=111, page=1, product_id="p2"))+ assert [r["product_id"] for r in result.rows] == ["p3"]++class TestPrintifySourceResponse:@parameterized.expand([(e,) for e in ENDPOINTS])def test_source_response_shape(self, endpoint: str) -> None:@@ -240,5 +383,7 @@ class TestPrintifySourceResponse:def test_shop_scoped_endpoints_use_composite_primary_key(self) -> None:for config in PRINTIFY_ENDPOINTS.values():+ if config.fan_out is not None:+ continueexpected = ["shop_id", "id"] if config.shop_scoped else ["id"]assert config.primary_keys == expected, config.name