:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): implement amazon selling partner source (#118147)
Daniel RC9d3a799a75c8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.mdindex 580e4affbe4a..94558929360b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md@@ -77,6 +77,7 @@ the row lists both.| alpha_vantage | HTTP | requests | ✅ || amazon_ads | HTTP | requests | ✅ || amazon_s3 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ |+| amazon_selling_partner | HTTP | requests + `rest_source.RESTClient` | ✅ || amplemarket | HTTP | requests + `rest_source.RESTClient` | ✅ || amplitude | HTTP | requests | ✅ || anthropic | HTTP | requests | ✅ |@@ -1038,7 +1039,6 @@ doesn't conflict with concurrent PRs.- amazon_cloudwatch- amazon_eventbridge- amazon_kinesis-- amazon_selling_partner- amazon_sns- amazon_sqs- anodot_costdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.pynew file mode 100644index 000000000000..2e9c3fd64e2f--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.py@@ -0,0 +1,561 @@+import json+import sqlite3+import tempfile+from collections.abc import Callable, Iterable, Iterator+from dataclasses import field+from datetime import UTC, datetime, timedelta+from pathlib import Path+from typing import Any, Optional++import requests+import structlog+from requests import PreparedRequest++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (+ DEFERRED_RELEASE_SWEEP_DAYS,+ EARLIEST_START,+ INVENTORY_SUMMARIES,+ INVENTORY_SUMMARIES_PATH,+ LWA_TOKEN_URL,+ MARKETPLACE_PARTICIPATIONS_PATH,+ ORDER_ITEMS,+ ORDERS,+ ORDERS_INCLUDED_DATA,+ ORDERS_PAGE_SIZE,+ ORDERS_PATH,+ PARTITION_KEYS,+ PRIMARY_KEYS,+ REQUEST_TIME_MARGIN_SECONDS,+ SP_API_HOSTS,+ SP_API_MAX_RETRY_ATTEMPTS,+ SP_API_RETRY_BACKOFF_MAX_SECONDS,+ TRANSACTIONS,+ TRANSACTIONS_PATH,+ TRANSACTIONS_WINDOW_DAYS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (+ RESTAPIConfig,+ rest_api_resource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAuth2Auth,+ OAuth2AuthRequestError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse++logger = structlog.get_logger(__name__)++LWA_REJECTED_MESSAGE = (+ "Amazon rejected the Login with Amazon credentials. "+ "Check the client ID, client secret and refresh token, then try again."+)+ACCESS_DENIED_MESSAGE = (+ "Amazon accepted the login but refused the Selling Partner API call. "+ "Check that the app is authorized for this seller and has the roles the tables need."+)+UNAVAILABLE_MESSAGE = "Couldn't reach Amazon to check the credentials. Try again in a few minutes."++REQUEST_TIMEOUT = (10.0, 30.0)+INVENTORY_ROWS_PER_BATCH = 2000+ResumeHook = Callable[[Optional[dict[str, Any]]], None]+++@frozen+class AmazonSellingPartnerResumeConfig:+ # The page token of the endpoint being walked, when the vendor's token outlives a hand-off.+ next_token: str | None = None+ # The transactions window being walked. Amazon accepts a page token only with the arguments that produced it.+ window_start: str | None = None+ window_end: str | None = None+ transaction_status: str | None = None+ # Marketplaces whose inventory walk finished. Inventory page tokens expire in 30 seconds, so a+ # marketplace restarts from its first page.+ completed_marketplaces: list[str] = field(default_factory=list)+ marketplace_id: str | None = None+ last_seller_sku: str | None = None+++class LwaAuth(OAuth2Auth):+ """Login with Amazon refresh-token grant. SP-API reads the token from `x-amz-access-token`."""++ def __init__(self, client_id: str, client_secret: str, refresh_token: str) -> None:+ super().__init__(+ token_url=LWA_TOKEN_URL,+ client_id=client_id,+ client_secret=client_secret,+ grant_type="refresh_token",+ refresh_token=refresh_token,+ )++ def __call__(self, request: PreparedRequest) -> PreparedRequest:+ if self.token is None or self._is_token_expired():+ self._obtain_token()+ assert self.token is not None+ request.headers["x-amz-access-token"] = self.token+ return request+++def to_iso8601(value: Any) -> str:+ parsed = parse_datetime_value(value)+ if parsed is None:+ return str(value)+ return parsed.strftime("%Y-%m-%dT%H:%M:%SZ")+++def parse_marketplace_ids(value: str | None) -> list[str]:+ return [part.strip() for part in (value or "").split(",") if part.strip()]+++@frozen+class TransactionWindow:+ posted_after: datetime+ posted_before: datetime+ transaction_status: str | None = None+++def transaction_windows(start: datetime, now: datetime) -> list[TransactionWindow]:+ """Contiguous [postedAfter, postedBefore) windows from `start` up to the latest time Amazon accepts."""+ cutoff = now - timedelta(seconds=REQUEST_TIME_MARGIN_SECONDS)+ windows: list[TransactionWindow] = []+ while start < cutoff:+ end = min(start + timedelta(days=TRANSACTIONS_WINDOW_DAYS), cutoff)+ windows.append(TransactionWindow(posted_after=start, posted_before=end))+ start = end+ return windows+++def _base_url(region: str) -> str:+ host = SP_API_HOSTS.get(region)+ if host is None:+ raise ValueError(f"Invalid Amazon Selling Partner region: {region}")+ return host+++def _write_disposition(merge: bool) -> Any:+ return {"disposition": "merge", "strategy": "upsert"} if merge else "replace"+++def _build_resource(+ *,+ region: str,+ auth: LwaAuth,+ name: str,+ path: str,+ data_selector: str,+ cursor_path: str,+ cursor_param: str,+ params: dict[str, Any],+ merge: bool,+ team_id: int,+ job_id: str,+ db_incremental_field_last_value: Any,+ resume_hook: ResumeHook | None = None,+ initial_paginator_state: dict[str, Any] | None = None,+) -> Resource:+ config: RESTAPIConfig = {+ "client": {+ "base_url": _base_url(region),+ "auth": auth,+ "max_retries": SP_API_MAX_RETRY_ATTEMPTS,+ "retry_backoff_max_seconds": SP_API_RETRY_BACKOFF_MAX_SECONDS,+ # Without this a stalled SP-API host holds the whole retry budget in one try, and the+ # waits above never run. The seller's other calls use the same bound.+ "request_timeout": REQUEST_TIMEOUT,+ # Order bodies carry buyer data that only the row maps remove, after any sample is taken.+ "capture": False,+ },+ "resource_defaults": {},+ "resources": [+ {+ "name": name,+ "table_name": name,+ "write_disposition": _write_disposition(merge),+ "endpoint": {+ "path": path,+ "data_selector": data_selector,+ "data_selector_empty_ok": True,+ "paginator": {"type": "cursor", "cursor_path": cursor_path, "cursor_param": cursor_param},+ "params": params,+ },+ "table_format": "delta",+ }+ ],+ }+ return rest_api_resource(+ config,+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=resume_hook,+ initial_paginator_state=initial_paginator_state,+ )+++def _wrapper_hints(name: str, merge: bool) -> dict[str, Any]:+ return {"table_name": name, "write_disposition": _write_disposition(merge), "table_format": "delta"}+++def drop_buyer_personal_data(order: dict[str, Any]) -> dict[str, Any]:+ """Remove the buyer's personal data that rides inside the TAX and FULFILLMENT datasets."""+ tax = order.get("tax")+ if isinstance(tax, dict):+ tax.pop("taxRegistrations", None)+ for item in order.get("orderItems") or []:+ product = item.get("product")+ if isinstance(product, dict):+ product.pop("customization", None)+ gift_option = ((item.get("fulfillment") or {}).get("packing") or {}).get("giftOption")+ if isinstance(gift_option, dict):+ gift_option.pop("giftMessage", None)+ return order+++def explode_order_items(order: dict[str, Any]) -> list[dict[str, Any]]:+ parent = {key: order.get(key) for key in ("orderId", "createdTime", "lastUpdatedTime")}+ parent["marketplaceId"] = (order.get("salesChannel") or {}).get("marketplaceId")+ return [{**parent, **item} for item in order.get("orderItems") or []]+++def _orders_resource(+ *,+ region: str,+ auth: LwaAuth,+ marketplace_ids: list[str],+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> Resource:+ initial_paginator_state: dict[str, Any] | None = None+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None and resume.next_token:+ initial_paginator_state = {"cursor": resume.next_token}++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state and state.get("cursor"):+ resumable_source_manager.save_state(AmazonSellingPartnerResumeConfig(next_token=str(state["cursor"])))++ resource = _build_resource(+ region=region,+ auth=auth,+ name=endpoint,+ path=ORDERS_PATH,+ data_selector="orders",+ cursor_path="pagination.nextToken",+ cursor_param="paginationToken",+ params={+ "maxResultsPerPage": ORDERS_PAGE_SIZE,+ "includedData": ORDERS_INCLUDED_DATA,+ "marketplaceIds": ",".join(marketplace_ids) or None,+ # Exactly one of createdAfter and lastUpdatedAfter is required; the latter also+ # covers a full refresh, because an order's last update is never before its creation.+ "lastUpdatedAfter": {+ "type": "incremental",+ "cursor_path": "lastUpdatedTime",+ "initial_value": EARLIEST_START,+ "convert": to_iso8601,+ },+ },+ merge=should_use_incremental_field,+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=db_incremental_field_last_value,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )+ resource.add_map(drop_buyer_personal_data)+ if endpoint == ORDER_ITEMS:+ resource.add_map(explode_order_items)+ return resource+++def deferred_release_sweep(now: datetime) -> TransactionWindow:+ cutoff = now - timedelta(seconds=REQUEST_TIME_MARGIN_SECONDS)+ return TransactionWindow(+ posted_after=cutoff - timedelta(days=DEFERRED_RELEASE_SWEEP_DAYS),+ posted_before=cutoff,+ transaction_status="DEFERRED_RELEASED",+ )+++def _transactions_resource(+ *,+ region: str,+ auth: LwaAuth,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> Resource:+ now = datetime.now(UTC)+ start = parse_datetime_value(EARLIEST_START)+ if should_use_incremental_field and db_incremental_field_last_value is not None:+ start = parse_datetime_value(db_incremental_field_last_value) or start+ assert start is not None+ windows = transaction_windows(start, now)+ initial_paginator_state: dict[str, Any] | None = None+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ if resume is not None and resume.window_start and resume.window_end:+ after, before = parse_datetime_value(resume.window_start), parse_datetime_value(resume.window_end)+ if after is not None and before is not None:+ current = TransactionWindow(+ posted_after=after, posted_before=before, transaction_status=resume.transaction_status+ )+ windows = [current]+ if current.transaction_status is None:+ windows += transaction_windows(before, now)+ if resume.next_token:+ initial_paginator_state = {"cursor": resume.next_token}+ # A first sync reads the whole history already, so the sweep would only repeat rows.+ has_watermark = should_use_incremental_field and db_incremental_field_last_value is not None+ if has_watermark and (not windows or windows[-1].transaction_status is None):+ windows.append(deferred_release_sweep(now))++ def state_for(window: TransactionWindow, next_token: str | None = None) -> AmazonSellingPartnerResumeConfig:+ return AmazonSellingPartnerResumeConfig(+ window_start=to_iso8601(window.posted_after),+ window_end=to_iso8601(window.posted_before),+ transaction_status=window.transaction_status,+ next_token=next_token,+ )++ def pages() -> Iterator[Any]:+ for index, window in enumerate(windows):+ following = windows[index + 1] if index + 1 < len(windows) else None++ def save_checkpoint(+ state: dict[str, Any] | None,+ window: TransactionWindow = window,+ following: TransactionWindow | None = following,+ ) -> None:+ if state and state.get("cursor"):+ resumable_source_manager.save_state(state_for(window, str(state["cursor"])))+ elif following is not None:+ # Staged before the window's last page is handed on, so a resume skips the window.+ resumable_source_manager.save_state(state_for(following))++ params: dict[str, Any] = {+ "postedAfter": to_iso8601(window.posted_after),+ "postedBefore": to_iso8601(window.posted_before),+ }+ if window.transaction_status is not None:+ params["transactionStatus"] = window.transaction_status+ yield from _build_resource(+ region=region,+ auth=auth,+ name=TRANSACTIONS,+ path=TRANSACTIONS_PATH,+ data_selector="payload.transactions",+ cursor_path="payload.nextToken",+ cursor_param="nextToken",+ params=params,+ merge=should_use_incremental_field,+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state if index == 0 else None,+ )++ return Resource(pages, name=TRANSACTIONS, hints=_wrapper_hints(TRANSACTIONS, should_use_incremental_field))+++def list_marketplace_ids(region: str, auth: LwaAuth) -> list[str]:+ session = make_tracked_session(redact_values=auth.secret_values())+ response = session.get(f"{_base_url(region)}{MARKETPLACE_PARTICIPATIONS_PATH}", auth=auth, timeout=REQUEST_TIMEOUT)+ response.raise_for_status()+ return [+ entry["marketplace"]["id"]+ for entry in response.json().get("payload") or []+ if (entry.get("participation") or {}).get("isParticipating", True)+ ]+++def _spool_inventory_pages(+ session: requests.Session,+ auth: LwaAuth,+ url: str,+ marketplace_id: str,+ spool: sqlite3.Connection,+ manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+ skip_if_not_covered: bool,+) -> bool:+ """Spool every page of one marketplace. False when Amazon rejects the marketplace on its first page."""+ next_token: str | None = None+ while True:+ params: dict[str, Any] = {+ "granularityType": "Marketplace",+ "granularityId": marketplace_id,+ "marketplaceIds": marketplace_id,+ "details": "true",+ }+ if next_token:+ params["nextToken"] = next_token+ response = session.get(url, params=params, auth=auth, timeout=REQUEST_TIMEOUT)+ # A discovered marketplace can be a channel FBA inventory does not cover. A later page's 400+ # is a real failure, such as an expired page token.+ if skip_if_not_covered and next_token is None and response.status_code == 400:+ return False+ response.raise_for_status()+ body = response.json()+ spool.executemany(+ "INSERT OR REPLACE INTO inventory VALUES (?, ?)",+ [+ (str(summary.get("sellerSku") or ""), json.dumps({"marketplaceId": marketplace_id, **summary}))+ for summary in (body.get("payload") or {}).get("inventorySummaries") or []+ ],+ )+ next_token = (body.get("pagination") or {}).get("nextToken")+ # The staged cursor does not cover this marketplace, so a hand-off here reads it again.+ manager.safe_point()+ if not next_token:+ return True+++def _inventory_rows(+ *,+ region: str,+ auth: LwaAuth,+ marketplace_ids: list[str],+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ manager = resumable_source_manager+ resume = manager.load_state() if manager.can_resume() else None+ completed = list(resume.completed_marketplaces) if resume is not None else []+ session = make_tracked_session(redact_values=auth.secret_values())+ url = f"{_base_url(region)}{INVENTORY_SUMMARIES_PATH}"+ discovered = not marketplace_ids++ for marketplace_id in marketplace_ids or list_marketplace_ids(region, auth):+ if marketplace_id in completed:+ continue+ after_sku = resume.last_seller_sku if resume is not None and resume.marketplace_id == marketplace_id else None+ done_state = AmazonSellingPartnerResumeConfig(completed_marketplaces=[*completed, marketplace_id])+ # Page tokens expire in 30 seconds and Amazon documents no SKU order, so rows wait on disk+ # sorted by SKU and a resume continues after the last SKU handed on.+ with tempfile.TemporaryDirectory() as spool_dir:+ # The pipeline may resume this generator on another thread.+ spool = sqlite3.connect(Path(spool_dir) / "inventory.sqlite", check_same_thread=False)+ try:+ spool.execute("CREATE TABLE inventory (seller_sku TEXT PRIMARY KEY, row TEXT NOT NULL)")+ if not _spool_inventory_pages(+ session, auth, url, marketplace_id, spool, manager, skip_if_not_covered=discovered+ ):+ logger.warning(+ "amazon_selling_partner_inventory_marketplace_skipped", marketplace_id=marketplace_id+ )+ manager.save_state(done_state)+ manager.safe_point()+ completed = done_state.completed_marketplaces+ continue+ cursor = spool.execute(+ "SELECT seller_sku, row FROM inventory WHERE seller_sku > ? ORDER BY seller_sku", (after_sku or "",)+ )+ batch = cursor.fetchmany(INVENTORY_ROWS_PER_BATCH)+ if not batch:+ manager.save_state(done_state)+ manager.safe_point()+ while batch:+ following = cursor.fetchmany(INVENTORY_ROWS_PER_BATCH)+ manager.save_state(+ done_state+ if not following+ else AmazonSellingPartnerResumeConfig(+ completed_marketplaces=completed,+ marketplace_id=marketplace_id,+ last_seller_sku=batch[-1][0],+ )+ )+ yield [json.loads(row) for _, row in batch]+ batch = following+ finally:+ spool.close()+ completed = done_state.completed_marketplaces+++def amazon_selling_partner_source(+ *,+ region: str,+ client_id: str,+ client_secret: str,+ refresh_token: str,+ marketplace_ids: list[str],+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Any,+) -> SourceResponse:+ auth = LwaAuth(client_id, client_secret, refresh_token)+ common = {+ "region": region,+ "auth": auth,+ "team_id": team_id,+ "job_id": job_id,+ "resumable_source_manager": resumable_source_manager,+ "should_use_incremental_field": should_use_incremental_field,+ "db_incremental_field_last_value": db_incremental_field_last_value,+ }+ items: Callable[[], Iterable[Any]]+ column_hints: dict[str, Any] | None = None+ if endpoint == INVENTORY_SUMMARIES:+ items = lambda: _inventory_rows(+ region=region, auth=auth, marketplace_ids=marketplace_ids, resumable_source_manager=resumable_source_manager+ )+ else:+ resource: Resource+ if endpoint in (ORDERS, ORDER_ITEMS):+ resource = _orders_resource(marketplace_ids=marketplace_ids, endpoint=endpoint, **common)+ elif endpoint == TRANSACTIONS:+ resource = _transactions_resource(**common)+ else:+ raise ValueError(f"Unknown Amazon Selling Partner endpoint: {endpoint}")+ items = lambda: resource+ column_hints = resource.column_hints++ partition_key = PARTITION_KEYS.get(endpoint)+ return SourceResponse(+ name=endpoint,+ items=items,+ primary_keys=PRIMARY_KEYS[endpoint],+ # Amazon documents no order for any of these listings. Desc stores the watermark when the+ # run completes, which is correct whichever order the rows arrive in.+ sort_mode="desc",+ partition_count=1 if partition_key else None,+ partition_size=1 if partition_key else None,+ partition_mode="datetime" if partition_key else None,+ partition_format="month" if partition_key else None,+ partition_keys=[partition_key] if partition_key else None,+ column_hints=column_hints,+ )+++def validate_credentials(+ region: str, client_id: str, client_secret: str, refresh_token: str+) -> tuple[bool, str | None]:+ auth = LwaAuth(client_id, client_secret, refresh_token)+ session = make_tracked_session(redact_values=auth.secret_values(), capture=False)+ try:+ response = session.get(+ f"{_base_url(region)}{MARKETPLACE_PARTICIPATIONS_PATH}", auth=auth, timeout=REQUEST_TIMEOUT+ )+ except OAuth2AuthRequestError as e:+ return False, LWA_REJECTED_MESSAGE if e.is_permanent else UNAVAILABLE_MESSAGE+ except requests.RequestException:+ return False, UNAVAILABLE_MESSAGE+ if response.status_code == 200:+ return True, None+ if response.status_code in (401, 403):+ return False, ACCESS_DENIED_MESSAGE+ return False, UNAVAILABLE_MESSAGEdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.pynew file mode 100644index 000000000000..371d1ba285a0--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.py@@ -0,0 +1,78 @@+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)++CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {+ "orders": {+ "description": "A customer order placed with the seller, with its items nested in orderItems.",+ "docs_url": "https://developer-docs.amazon.com/sp-api/reference/orders-v2026-01-01",+ "columns": {+ "orderId": "Amazon-defined order identifier.",+ "orderAliases": "Alternative identifiers for the order, such as a seller-defined order number.",+ "createdTime": "Time the customer placed the order, in ISO 8601 format.",+ "lastUpdatedTime": "Most recent time any aspect of the order changed, by Amazon or the seller.",+ "programs": "Special programs the order belongs to, such as PRIME or AMAZON_BUSINESS.",+ "associatedOrders": "Orders directly related to this one, such as a replacement or exchange.",+ "salesChannel": "Where the order was placed, including the marketplace id and name.",+ "proceeds": "Revenue and financial breakdown for the order.",+ "payment": "Payment information for the order.",+ "tax": "Tax information for the order.",+ "fulfillment": "Fulfillment status, who fulfills the order and the ship and delivery windows.",+ "orderItems": "The items in the order.",+ "packages": "Shipping packages and tracking for the order.",+ },+ },+ "order_items": {+ "description": "One row per item in an order, with the order's id, creation time and last update time.",+ "docs_url": "https://developer-docs.amazon.com/sp-api/reference/orders-v2026-01-01",+ "columns": {+ "orderId": "Amazon-defined identifier of the order the item belongs to.",+ "createdTime": "Time the customer placed the order, in ISO 8601 format.",+ "lastUpdatedTime": "Most recent time any aspect of the order changed.",+ "marketplaceId": "Marketplace the order was placed in.",+ "orderItemId": "Amazon-defined identifier of the order item.",+ "quantityOrdered": "Number of units the customer ordered.",+ "product": "The product ordered, including its ASIN, seller SKU and title.",+ "proceeds": "Revenue breakdown for the item.",+ "expense": "Cost information for the item.",+ "promotion": "Discounts and promotional offers applied to the item.",+ "cancellation": "Cancellation information for the item.",+ "fulfillment": "Fulfillment status and quantities for the item.",+ "tax": "Tax information for the item.",+ },+ },+ "transactions": {+ "description": "A financial transaction on the seller account, such as an order payment, refund or fee.",+ "docs_url": "https://developer-docs.amazon.com/sp-api/docs/finances-api-v2024-06-19-reference",+ "columns": {+ "transactionId": "Unique identifier of the transaction.",+ "transactionType": "Type of transaction, such as Shipment.",+ "transactionStatus": "DEFERRED, RELEASED or DEFERRED_RELEASED.",+ "description": "Reason for the transaction, such as Order Payment or Refund Order.",+ "postedDate": "Date and time the transaction was posted.",+ "totalAmount": "Total amount of money in the transaction, with its currency code.",+ "marketplaceDetails": "Marketplace where the transaction occurred.",+ "relatedIdentifiers": "Identifiers related to the transaction, such as the order and shipment ids.",+ "sellingPartnerMetadata": "The seller's id, account type and marketplace.",+ "items": "The items in the transaction with their amounts.",+ "contexts": "Additional information about the transaction.",+ "breakdowns": "How the total amount is calculated.",+ },+ },+ "inventory_summaries": {+ "description": "FBA inventory levels per seller SKU and marketplace.",+ "docs_url": "https://developer-docs.amazon.com/sp-api/docs/fba-inventory-api-v1-reference",+ "columns": {+ "marketplaceId": "Marketplace the inventory summary was requested for.",+ "asin": "Amazon Standard Identification Number of the item.",+ "fnSku": "Amazon fulfillment network SKU of the item.",+ "sellerSku": "Seller SKU of the item.",+ "condition": "Condition of the item as described by the seller.",+ "inventoryDetails": "Fulfillable, inbound, reserved, unfulfillable and researching quantities.",+ "lastUpdatedTime": "Date and time any quantity was last updated.",+ "productName": "Localized product title in the marketplace.",+ "totalQuantity": "Total units in inbound shipments or in Amazon fulfillment centers.",+ "stores": "Seller-enrolled stores that apply to the SKU.",+ },+ },+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.pynew file mode 100644index 000000000000..354d430dd063--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.py@@ -0,0 +1,63 @@+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field+from products.warehouse_sources.backend.types import IncrementalField++SP_API_HOSTS = {+ "na": "https://sellingpartnerapi-na.amazon.com",+ "eu": "https://sellingpartnerapi-eu.amazon.com",+ "fe": "https://sellingpartnerapi-fe.amazon.com",+}+LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"++ORDERS_PATH = "/orders/2026-01-01/orders"+TRANSACTIONS_PATH = "/finances/2024-06-19/transactions"+INVENTORY_SUMMARIES_PATH = "/fba/inventory/v1/summaries"+MARKETPLACE_PARTICIPATIONS_PATH = "/sellers/v1/marketplaceParticipations"++ORDERS_PAGE_SIZE = 100+# BUYER and RECIPIENT are left out: they carry PII and need restricted roles on the app.+ORDERS_INCLUDED_DATA = "FULFILLMENT,PROCEEDS,EXPENSE,PROMOTION,CANCELLATION,PACKAGES,TAX,PAYMENT"+# listTransactions answers with an empty page when postedAfter and postedBefore are over 180 days apart.+TRANSACTIONS_WINDOW_DAYS = 179+# Amazon rejects a window end closer than two minutes to the request time.+REQUEST_TIME_MARGIN_SECONDS = 180+# The oldest order data any marketplace keeps (JP, AU and SG keep orders from 2016).+EARLIEST_START = "2016-01-01T00:00:00Z"+# Financial events can post up to 48 hours late.+TRANSACTIONS_LOOKBACK_SECONDS = 3 * 24 * 60 * 60+# A deferred transaction keeps its postedDate when Amazon releases it, so each incremental run also+# re-reads the transactions released in this many days.+DEFERRED_RELEASE_SWEEP_DAYS = 60+# searchOrders refills at one request every 179 seconds once its burst is spent, and SP-API sends+# no Retry-After. Nine attempts wait 1, 2, 4 ... 128 seconds, 255 in all, past that interval and+# inside the shared retry budget, which ends a request 300 seconds after its first try.+SP_API_MAX_RETRY_ATTEMPTS = 9+SP_API_RETRY_BACKOFF_MAX_SECONDS = 128.0++ORDERS = "orders"+ORDER_ITEMS = "order_items"+TRANSACTIONS = "transactions"+INVENTORY_SUMMARIES = "inventory_summaries"++ENDPOINTS = (ORDERS, ORDER_ITEMS, TRANSACTIONS, INVENTORY_SUMMARIES)++PRIMARY_KEYS: dict[str, list[str]] = {+ ORDERS: ["orderId"],+ ORDER_ITEMS: ["orderId", "orderItemId"],+ TRANSACTIONS: ["transactionId"],+ INVENTORY_SUMMARIES: ["marketplaceId", "sellerSku"],+}++PARTITION_KEYS: dict[str, str] = {+ ORDERS: "createdTime",+ ORDER_ITEMS: "createdTime",+ TRANSACTIONS: "postedDate",+}+++INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ ORDERS: [incremental_field("lastUpdatedTime")],+ ORDER_ITEMS: [incremental_field("lastUpdatedTime")],+ TRANSACTIONS: [incremental_field("postedDate")],+}++DEFAULT_INCREMENTAL_LOOKBACK_SECONDS: dict[str, int] = {TRANSACTIONS: TRANSACTIONS_LOOKBACK_SECONDS}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.pyindex 4ca298a89030..377799a9ce54 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py@@ -1,8 +1,40 @@-from typing import cast+from typing import Optional, cast-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.facade.codebase_detection import detect+from products.warehouse_sources.backend.facade.source_config import (+ DataWarehouseSourceCategory,+ ReleaseStatus,+ SourceConfig,+ SourceFieldInputConfig,+ SourceFieldInputConfigType,+ SourceFieldSelectConfig,+ SourceFieldSelectConfigOption,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.amazon_selling_partner import (+ AmazonSellingPartnerResumeConfig,+ amazon_selling_partner_source,+ parse_marketplace_ids,+ validate_credentials as validate_amazon_selling_partner_credentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (+ DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,+ ENDPOINTS,+ INCREMENTAL_FIELDS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (+ CanonicalDescriptions,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAUTH2_PERMANENT_ERROR_MARKER,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (+ SourceSchema,+ build_endpoint_schemas,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazonsellingpartner import (AmazonSellingPartnerSourceConfig,)@@ -10,18 +42,158 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class AmazonSellingPartnerSource(SimpleSource[AmazonSellingPartnerSourceConfig]):+class AmazonSellingPartnerSource(ResumableSource[AmazonSellingPartnerSourceConfig, AmazonSellingPartnerResumeConfig]):+ codebase_detection = detect(+ env=("^SP_API_",),+ npm=("amazon-sp-api", "@sp-api-sdk/orders-api-v0", "@sp-api-sdk/auth"),+ pypi=("python-amazon-sp-api",),+ rubygems=("amz_sp_api",),+ composer=("jlevers/selling-partner-api", "highsidelabs/laravel-spapi"),+ go=("github.com/amzapi/selling-partner-api-sdk",),+ )+ # The refresh token comes from a seller-side authorization flow, so users set this up in the app.+ setup_requires_ui = True++ lists_tables_without_credentials = True # static endpoint catalog — safe for public docs+ # Each SP-API section carries its own dated version, so there is no single pin for the source.+ api_docs_url = "https://developer-docs.amazon.com/sp-api/changelog"+@propertydef source_type(self) -> ExternalDataSourceType:return ExternalDataSourceType.AMAZONSELLINGPARTNER+ def get_non_retryable_errors(self) -> dict[str, str | None]:+ return {+ "HTTP 400 from the OAuth2 token endpoint": "Amazon rejected the refresh token. Re-authorize the app for this seller and update the refresh token.",+ "HTTP 401 from the OAuth2 token endpoint": "Amazon rejected the Login with Amazon client ID or client secret. Check them in the Amazon developer console and update the source.",+ "401 Client Error: Unauthorized for url: https://sellingpartnerapi": "Amazon rejected the access token. Re-authorize the app for this seller and update the refresh token.",+ "403 Client Error: Forbidden for url: https://sellingpartnerapi": "Amazon refused access to this table. Check that the app has the role the table needs and that the seller authorized it.",+ OAUTH2_PERMANENT_ERROR_MARKER: "Amazon rejected the Login with Amazon credentials. Check the client ID, client secret and refresh token, then update the source.",+ }++ def get_canonical_descriptions(self) -> CanonicalDescriptions:+ from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.canonical_descriptions import (+ CANONICAL_DESCRIPTIONS,+ )++ return CANONICAL_DESCRIPTIONS++ def get_schemas(+ self,+ config: AmazonSellingPartnerSourceConfig,+ team_id: int,+ with_counts: bool = False,+ names: list[str] | None = None,+ force_refresh: bool = False,+ api_version: str | None = None,+ ) -> list[SourceSchema]:+ return build_endpoint_schemas(+ ENDPOINTS,+ INCREMENTAL_FIELDS,+ names,+ default_incremental_lookback_seconds=DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,+ )++ def validate_credentials(+ self,+ config: AmazonSellingPartnerSourceConfig,+ team_id: int,+ schema_name: Optional[str] = None,+ api_version: str | None = None,+ ) -> tuple[bool, str | None]:+ return validate_amazon_selling_partner_credentials(+ config.region, config.client_id, config.client_secret, config.refresh_token+ )++ def get_resumable_source_manager(+ self, inputs: SourceInputs+ ) -> ResumableSourceManager[AmazonSellingPartnerResumeConfig]:+ return ResumableSourceManager[AmazonSellingPartnerResumeConfig](inputs, AmazonSellingPartnerResumeConfig)++ def source_for_pipeline(+ self,+ config: AmazonSellingPartnerSourceConfig,+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:+ return amazon_selling_partner_source(+ region=config.region,+ client_id=config.client_id,+ client_secret=config.client_secret,+ refresh_token=config.refresh_token,+ marketplace_ids=parse_marketplace_ids(config.marketplace_ids),+ endpoint=inputs.schema_name,+ team_id=inputs.team_id,+ job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,+ )+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(name=ExternalDataSourceType.AMAZONSELLINGPARTNER,category=DataWarehouseSourceCategory.E_COMMERCE,label="Amazon Selling Partner",+ keywords=["sp-api", "seller central", "amazon seller", "fba"],+ caption=(+ "Syncs orders, order items, financial transactions and FBA inventory from your Amazon seller account. "+ "The app needs the Inventory and Order Tracking, Finance and Accounting and Amazon Fulfillment roles. "+ "Orders and order items read the same Amazon endpoint, so syncing both takes twice as long."+ ),iconPath="/static/services/amazon_selling_partner.png",- fields=cast(list[FieldType], []),- unreleasedSource=True,+ docsUrl="https://posthog.com/docs/cdp/sources/amazon-selling-partner",+ releaseStatus=ReleaseStatus.ALPHA,+ fields=cast(+ list[FieldType],+ [+ SourceFieldSelectConfig(+ name="region",+ label="Region",+ required=True,+ defaultValue="na",+ options=[+ SourceFieldSelectConfigOption(label="North America", value="na"),+ SourceFieldSelectConfigOption(label="Europe", value="eu"),+ SourceFieldSelectConfigOption(label="Far East", value="fe"),+ ],+ ),+ SourceFieldInputConfig(+ name="client_id",+ label="LWA client ID",+ type=SourceFieldInputConfigType.TEXT,+ required=True,+ placeholder="amzn1.application-oa2-client...",+ secret=False,+ ),+ SourceFieldInputConfig(+ name="client_secret",+ label="LWA client secret",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="refresh_token",+ label="Refresh token",+ type=SourceFieldInputConfigType.PASSWORD,+ required=True,+ placeholder="Atzr|...",+ secret=True,+ ),+ SourceFieldInputConfig(+ name="marketplace_ids",+ label="Marketplace IDs",+ caption="Comma-separated. Applies to orders, order items and inventory, while transactions always cover every marketplace. Leave blank to use every marketplace the seller sells in.",+ type=SourceFieldInputConfigType.TEXT,+ required=False,+ placeholder="ATVPDKIKX0DER, A1F83G8C2ARO7P",+ secret=False,+ ),+ ],+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.pynew file mode 100644index 000000000000..d8f254d30e9e--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.py@@ -0,0 +1,559 @@+import json+from collections.abc import Iterator+from datetime import UTC, datetime, timedelta+from typing import Any+from urllib.parse import parse_qs++import pytest+import time_machine++import requests++from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.amazon_selling_partner import (+ ACCESS_DENIED_MESSAGE,+ LWA_REJECTED_MESSAGE,+ UNAVAILABLE_MESSAGE,+ AmazonSellingPartnerResumeConfig,+ to_iso8601,+ transaction_windows,+ validate_credentials,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (+ EARLIEST_START,+ ORDERS_INCLUDED_DATA,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.source import (+ AmazonSellingPartnerSource,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAuth2AuthRequestError,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ DriveResult,+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ always,+ route,+ scripted_network,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazonsellingpartner import (+ AmazonSellingPartnerSourceConfig,+)++TOKEN_PATH = "/auth/o2/token"+ORDERS_PATH = "/orders/2026-01-01/orders"+TRANSACTIONS_PATH = "/finances/2024-06-19/transactions"+SUMMARIES_PATH = "/fba/inventory/v1/summaries"+PARTICIPATIONS_PATH = "/sellers/v1/marketplaceParticipations"++TOKEN = ScriptedResponse(+ json={"access_token": "Atza|access", "refresh_token": "Atzr|refresh", "token_type": "bearer", "expires_in": 3600}+)+LWA_INVALID_GRANT = ScriptedResponse(+ status=400,+ json={"error": "invalid_grant", "error_description": "The request has an invalid grant parameter : refresh_token"},+)+SP_API_FORBIDDEN = ScriptedResponse(+ status=403,+ json={"errors": [{"code": "Unauthorized", "message": "Access to requested resource is denied.", "details": ""}]},+)+NOW = datetime(2026, 10, 15, 12, 0, 0, tzinfo=UTC)+++def _driver(marketplace_ids: str | None = None) -> SourceDriver:+ config = AmazonSellingPartnerSourceConfig(+ client_id="amzn1.application-oa2-client.test",+ client_secret="client-secret",+ refresh_token="Atzr|refresh",+ region="na",+ marketplace_ids=marketplace_ids,+ )+ return SourceDriver(AmazonSellingPartnerSource(), config)+++def _api_requests(result: DriveResult) -> list[RecordedRequest]:+ return [request for request in result.requests if request.path != TOKEN_PATH]+++def _order(order_id: str, items: list[dict[str, Any]] | None = None) -> dict[str, Any]:+ return {+ "orderId": order_id,+ "createdTime": "2026-01-02T03:04:05Z",+ "lastUpdatedTime": "2026-01-03T00:00:00Z",+ "salesChannel": {"marketplaceId": "ATVPDKIKX0DER", "marketplaceName": "Amazon.com"},+ "orderItems": items or [],+ }+++def _orders_page(orders: list[dict[str, Any]], next_token: str | None = None) -> ScriptedResponse:+ return ScriptedResponse(json={"orders": orders, "pagination": {"nextToken": next_token} if next_token else {}})+++def _transactions_page(ids: list[str], next_token: str | None = None) -> ScriptedResponse:+ payload: dict[str, Any] = {+ "transactions": [{"transactionId": id, "postedDate": "2026-09-02T00:00:00Z"} for id in ids]+ }+ if next_token:+ payload["nextToken"] = next_token+ return ScriptedResponse(json={"payload": payload})+++def _summaries_page(skus: list[str], next_token: str | None = None) -> ScriptedResponse:+ body: dict[str, Any] = {"payload": {"inventorySummaries": [{"sellerSku": sku, "totalQuantity": 1} for sku in skus]}}+ if next_token:+ body["pagination"] = {"nextToken": next_token}+ return ScriptedResponse(json=body)+++class TestOrders:+ def test_walks_pages_with_the_lwa_token(self) -> None:+ result = _driver("ATVPDKIKX0DER").run(+ "orders",+ route(+ {+ TOKEN_PATH: [TOKEN],+ ORDERS_PATH: [_orders_page([_order("1"), _order("2")], "tok2"), _orders_page([_order("3")])],+ }+ ),+ )++ assert result.raised is None+ token_request = result.requests[0]+ assert token_request.path == TOKEN_PATH+ assert parse_qs((token_request.body or b"").decode()) == {+ "grant_type": ["refresh_token"],+ "refresh_token": ["Atzr|refresh"],+ "client_id": ["amzn1.application-oa2-client.test"],+ "client_secret": ["client-secret"],+ }+ api_requests = _api_requests(result)+ assert [request.path for request in api_requests] == [ORDERS_PATH, ORDERS_PATH]+ assert all(request.headers["x-amz-access-token"] == "Atza|access" for request in api_requests)+ assert [request.param("paginationToken") for request in api_requests] == [None, "tok2"]+ assert api_requests[0].param("maxResultsPerPage") == "100"+ assert api_requests[0].param("includedData") == ORDERS_INCLUDED_DATA+ assert [row["orderId"] for row in result.rows] == ["1", "2", "3"]+ assert result.saved_states == [AmazonSellingPartnerResumeConfig(next_token="tok2")]+ # Order bodies carry buyer data, so no session may store HTTP samples of them.+ assert result.session_options and all(options.get("capture") is False for options in result.session_options)++ @pytest.mark.parametrize(+ ("incremental_field", "last_value", "marketplace_ids", "expected_after", "expected_marketplaces"),+ [+ (None, None, None, EARLIEST_START, None),+ (+ "lastUpdatedTime",+ datetime(2026, 3, 4, 5, 6, 7, tzinfo=UTC),+ "ATVPDKIKX0DER, A1F83G8C2ARO7P",+ "2026-03-04T05:06:07Z",+ "ATVPDKIKX0DER,A1F83G8C2ARO7P",+ ),+ ("lastUpdatedTime", "2026-03-04 05:06:07", None, "2026-03-04T05:06:07Z", None),+ ],+ )+ def test_request_shaping(+ self,+ incremental_field: str | None,+ last_value: Any,+ marketplace_ids: str | None,+ expected_after: str,+ expected_marketplaces: str | None,+ ) -> None:+ result = _driver(marketplace_ids).run(+ "orders",+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("1")])]}),+ incremental_field=incremental_field,+ db_incremental_field_last_value=last_value,+ )++ request = _api_requests(result)[0]+ assert request.param("lastUpdatedAfter") == expected_after+ assert request.param("marketplaceIds") == expected_marketplaces+ assert request.param("createdAfter") is None++ def test_order_items_explode_each_order_with_its_parent_fields(self) -> None:+ items = [{"orderItemId": "i1", "quantityOrdered": 2}, {"orderItemId": "i2", "quantityOrdered": 1}]+ result = _driver().run(+ "order_items",+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("A", items), _order("B")])]}),+ )++ assert result.raised is None+ assert result.rows == [+ {+ "orderId": "A",+ "createdTime": "2026-01-02T03:04:05Z",+ "lastUpdatedTime": "2026-01-03T00:00:00Z",+ "marketplaceId": "ATVPDKIKX0DER",+ "orderItemId": "i1",+ "quantityOrdered": 2,+ },+ {+ "orderId": "A",+ "createdTime": "2026-01-02T03:04:05Z",+ "lastUpdatedTime": "2026-01-03T00:00:00Z",+ "marketplaceId": "ATVPDKIKX0DER",+ "orderItemId": "i2",+ "quantityOrdered": 1,+ },+ ]++ def test_buyer_personal_data_is_dropped_from_orders_and_items(self) -> None:+ item = {+ "orderItemId": "i1",+ "product": {"asin": "B01", "customization": {"customizedUrl": "https://example.com/c"}},+ "fulfillment": {"packing": {"giftOption": {"giftMessage": "Happy birthday", "giftWrapLevel": "CLASSIC"}}},+ }+ order = {**_order("A", [item]), "tax": {"taxRegistrations": [{"legalName": "Jane Example"}], "taxRate": 0.2}}++ for endpoint in ("orders", "order_items"):+ result = _driver().run(+ endpoint, route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([json.loads(json.dumps(order))])]})+ )++ assert result.raised is None+ serialized = json.dumps(result.rows)+ assert "Jane Example" not in serialized+ assert "Happy birthday" not in serialized+ assert "customizedUrl" not in serialized+ assert "CLASSIC" in serialized++ def test_orders_keep_retrying_through_amazons_refill_interval(self) -> None:+ throttled = ScriptedResponse(status=429, json={"errors": [{"code": "QuotaExceeded"}]})+ result = _driver().run(+ "orders",+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [throttled] * 8 + [_orders_page([_order("1")])]}),+ )++ assert result.raised is None+ assert [row["orderId"] for row in result.rows] == ["1"]++ def test_resumes_from_the_saved_page_token(self) -> None:+ result = _driver().run(+ "orders",+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("9")])]}),+ resume_state=AmazonSellingPartnerResumeConfig(next_token="tok9"),+ )++ assert _api_requests(result)[0].param("paginationToken") == "tok9"+ assert [row["orderId"] for row in result.rows] == ["9"]+++class TestTransactions:+ @pytest.fixture(autouse=True)+ def frozen_clock(self) -> Iterator[None]:+ with time_machine.travel(NOW, tick=False):+ yield++ def test_full_refresh_walks_contiguous_windows_from_the_earliest_start(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return TOKEN+ return _transactions_page([request.param("postedAfter") or ""])++ result = _driver().run("transactions", respond)++ assert result.raised is None+ windows = transaction_windows(datetime(2016, 1, 1, tzinfo=UTC), NOW)+ api_requests = _api_requests(result)+ assert [(r.param("postedAfter"), r.param("postedBefore")) for r in api_requests] == [+ (to_iso8601(w.posted_after), to_iso8601(w.posted_before)) for w in windows+ ]+ assert api_requests[0].param("postedAfter") == EARLIEST_START+ assert api_requests[-1].param("postedBefore") == "2026-10-15T11:57:00Z"+ assert all(r.param("nextToken") is None and r.param("transactionStatus") is None for r in api_requests)+ assert result.saved_states == [+ AmazonSellingPartnerResumeConfig(+ window_start=to_iso8601(w.posted_after), window_end=to_iso8601(w.posted_before)+ )+ for w in windows[1:]+ ]+ assert len(result.rows) == len(windows)++ def test_incremental_walks_from_the_watermark_then_sweeps_released_deferred_transactions(self) -> None:+ result = _driver().run(+ "transactions",+ route(+ {+ TOKEN_PATH: [TOKEN],+ TRANSACTIONS_PATH: [+ _transactions_page(["t1"], "n2"),+ _transactions_page(["t2"]),+ _transactions_page(["t-released"]),+ ],+ }+ ),+ incremental_field="postedDate",+ db_incremental_field_last_value=datetime(2026, 9, 1, tzinfo=UTC),+ )++ assert result.raised is None+ assert [+ (r.param("postedAfter"), r.param("postedBefore"), r.param("transactionStatus"), r.param("nextToken"))+ for r in _api_requests(result)+ ] == [+ ("2026-09-01T00:00:00Z", "2026-10-15T11:57:00Z", None, None),+ ("2026-09-01T00:00:00Z", "2026-10-15T11:57:00Z", None, "n2"),+ ("2026-08-16T11:57:00Z", "2026-10-15T11:57:00Z", "DEFERRED_RELEASED", None),+ ]+ assert [row["transactionId"] for row in result.rows] == ["t1", "t2", "t-released"]+ first_page_state = AmazonSellingPartnerResumeConfig(+ window_start="2026-09-01T00:00:00Z", window_end="2026-10-15T11:57:00Z", next_token="n2"+ )+ assert result.committed_states[0] == first_page_state++ def test_a_first_incremental_sync_skips_the_deferred_sweep(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ return TOKEN if request.path == TOKEN_PATH else _transactions_page([])++ result = _driver().run("transactions", respond, incremental_field="postedDate")++ assert result.raised is None+ assert all(request.param("transactionStatus") is None for request in _api_requests(result))++ @pytest.mark.parametrize(+ ("saved_window", "expected"),+ [+ (+ ("2026-10-01T00:00:00Z", "2026-10-14T00:00:00Z", "DEFERRED_RELEASED"),+ [("2026-10-01T00:00:00Z", "2026-10-14T00:00:00Z", "DEFERRED_RELEASED", "n2")],+ ),+ (+ ("2026-09-01T00:00:00Z", "2026-10-01T00:00:00Z", None),+ [+ ("2026-09-01T00:00:00Z", "2026-10-01T00:00:00Z", None, "n2"),+ ("2026-10-01T00:00:00Z", "2026-10-15T11:57:00Z", None, None),+ ("2026-08-16T11:57:00Z", "2026-10-15T11:57:00Z", "DEFERRED_RELEASED", None),+ ],+ ),+ ],+ )+ def test_resume_reuses_the_saved_window_with_its_token(+ self, saved_window: tuple[str, str, str | None], expected: list[tuple[str, str, str | None, str | None]]+ ) -> None:+ window_start, window_end, transaction_status = saved_window++ def respond(request: RecordedRequest) -> ScriptedResponse:+ return TOKEN if request.path == TOKEN_PATH else _transactions_page(["t"])++ result = _driver().run(+ "transactions",+ respond,+ resume_state=AmazonSellingPartnerResumeConfig(+ window_start=window_start,+ window_end=window_end,+ transaction_status=transaction_status,+ next_token="n2",+ ),+ incremental_field="postedDate",+ db_incremental_field_last_value=datetime(2026, 9, 1, tzinfo=UTC),+ )++ assert [+ (r.param("postedAfter"), r.param("postedBefore"), r.param("transactionStatus"), r.param("nextToken"))+ for r in _api_requests(result)+ ] == expected+++class TestInventorySummaries:+ def test_discovers_marketplaces_reads_each_one_whole_and_tags_rows(self) -> None:+ participations = ScriptedResponse(+ json={+ "payload": [+ {"marketplace": {"id": "A"}, "participation": {"isParticipating": True}},+ {"marketplace": {"id": "X"}, "participation": {"isParticipating": False}},+ {"marketplace": {"id": "B"}, "participation": {"isParticipating": True}},+ ]+ }+ )+ result = _driver().run(+ "inventory_summaries",+ route(+ {+ TOKEN_PATH: [TOKEN],+ PARTICIPATIONS_PATH: [participations],+ SUMMARIES_PATH: [+ _summaries_page(["sku1"], "next"),+ _summaries_page(["sku2"]),+ _summaries_page(["sku3"]),+ ],+ }+ ),+ )++ assert result.raised is None+ summaries = [request for request in _api_requests(result) if request.path == SUMMARIES_PATH]+ assert [(r.param("granularityId"), r.param("marketplaceIds"), r.param("nextToken")) for r in summaries] == [+ ("A", "A", None),+ ("A", "A", "next"),+ ("B", "B", None),+ ]+ assert summaries[0].param("details") == "true"+ assert all(r.param("startDateTime") is None for r in summaries)+ # Both pages of a marketplace arrive in one batch, so no page token waits on the pipeline.+ assert [[(row["marketplaceId"], row["sellerSku"]) for row in item] for item in result.items] == [+ [("A", "sku1"), ("A", "sku2")],+ [("B", "sku3")],+ ]+ assert result.committed_states == [+ AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"]),+ AmazonSellingPartnerResumeConfig(completed_marketplaces=["A", "B"]),+ ]++ def test_resume_skips_finished_marketplaces(self) -> None:+ result = _driver("A,B").run(+ "inventory_summaries",+ route({TOKEN_PATH: [TOKEN], SUMMARIES_PATH: [_summaries_page(["sku9"])]}),+ resume_state=AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"]),+ )++ assert result.raised is None+ assert [request.param("granularityId") for request in _api_requests(result)] == ["B"]++ def test_a_discovered_marketplace_inventory_does_not_cover_is_skipped(self) -> None:+ participations = ScriptedResponse(+ json={+ "payload": [+ {"marketplace": {"id": "X"}, "participation": {"isParticipating": True}},+ {"marketplace": {"id": "A"}, "participation": {"isParticipating": True}},+ ]+ }+ )+ invalid_marketplace = ScriptedResponse(+ status=400, json={"errors": [{"code": "InvalidInput", "message": "Invalid granularityId"}]}+ )++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return TOKEN+ if request.path == PARTICIPATIONS_PATH:+ return participations+ return invalid_marketplace if request.param("granularityId") == "X" else _summaries_page(["sku1"])++ result = _driver().run("inventory_summaries", respond)++ assert result.raised is None+ assert [(row["marketplaceId"], row["sellerSku"]) for row in result.rows] == [("A", "sku1")]+ assert result.final_state == AmazonSellingPartnerResumeConfig(completed_marketplaces=["X", "A"])++ def test_a_400_after_the_first_page_of_a_discovered_marketplace_fails_the_sync(self) -> None:+ participations = ScriptedResponse(+ json={"payload": [{"marketplace": {"id": "A"}, "participation": {"isParticipating": True}}]}+ )+ expired_token = ScriptedResponse(+ status=400, json={"errors": [{"code": "InvalidInput", "message": "The next token has expired"}]}+ )++ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return TOKEN+ if request.path == PARTICIPATIONS_PATH:+ return participations+ return expired_token if request.param("nextToken") else _summaries_page(["sku1"], "next")++ result = _driver().run("inventory_summaries", respond)++ assert isinstance(result.raised, requests.HTTPError)+ assert result.final_state is None++ def test_a_configured_marketplace_that_amazon_rejects_fails_the_sync(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ return TOKEN if request.path == TOKEN_PATH else ScriptedResponse(status=400, json={"errors": []})++ result = _driver("X").run("inventory_summaries", respond)++ assert isinstance(result.raised, requests.HTTPError)++ def test_resume_inside_a_marketplace_skips_rows_already_handed_on(self) -> None:+ result = _driver("A").run(+ "inventory_summaries",+ route(+ {+ TOKEN_PATH: [TOKEN],+ SUMMARIES_PATH: [_summaries_page(["sku1", "sku2"], "next"), _summaries_page(["sku3"])],+ }+ ),+ resume_state=AmazonSellingPartnerResumeConfig(marketplace_id="A", last_seller_sku="sku2"),+ )++ assert result.raised is None+ assert [row["sellerSku"] for row in result.rows] == ["sku3"]+ assert result.final_state == AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"])+++class TestErrors:+ def test_a_rejected_refresh_token_is_a_permanent_auth_error(self) -> None:+ result = _driver().run("orders", always(LWA_INVALID_GRANT))++ assert isinstance(result.raised, OAuth2AuthRequestError)+ assert result.raised.is_permanent+ assert "invalid_grant" in str(result.raised)+ assert result.rows == []++ def test_a_forbidden_api_response_keeps_the_vendor_error_code(self) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ return TOKEN if request.path == TOKEN_PATH else SP_API_FORBIDDEN++ result = _driver().run("orders", respond)++ assert isinstance(result.raised, requests.HTTPError)+ message = str(result.raised)+ assert (+ "403 Client Error: Forbidden for url: https://sellingpartnerapi-na.amazon.com/orders/2026-01-01/orders"+ in message+ )+ assert message.endswith("| api error: code=Unauthorized")+++class TestValidateCredentials:+ @pytest.mark.parametrize(+ ("token_answer", "probe_answer", "expected"),+ [+ (LWA_INVALID_GRANT, None, (False, LWA_REJECTED_MESSAGE)),+ (ScriptedResponse(status=503, json={}), None, (False, UNAVAILABLE_MESSAGE)),+ (TOKEN, ScriptedResponse(json={"payload": []}), (True, None)),+ (TOKEN, SP_API_FORBIDDEN, (False, ACCESS_DENIED_MESSAGE)),+ ],+ )+ def test_maps_the_vendor_answers(+ self, token_answer: ScriptedResponse, probe_answer: ScriptedResponse | None, expected: tuple[bool, str | None]+ ) -> None:+ def respond(request: RecordedRequest) -> ScriptedResponse:+ if request.path == TOKEN_PATH:+ return token_answer+ assert probe_answer is not None+ return probe_answer++ with scripted_network(respond) as network:+ assert validate_credentials("na", "client", "secret", "Atzr|refresh") == expected+ assert network.requests_log[-1].path == (TOKEN_PATH if probe_answer is None else PARTICIPATIONS_PATH)++ def test_a_network_failure_is_reported_as_unavailable(self) -> None:+ with scripted_network(always(requests.ConnectionError("boom"))):+ assert validate_credentials("na", "client", "secret", "Atzr|refresh") == (False, UNAVAILABLE_MESSAGE)+++ ("value", "expected"),+ [+ (datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC), "2026-01-02T03:04:05Z"),+ ("2026-01-02T03:04:05+02:00", "2026-01-02T01:04:05Z"),+ ("2026-01-02", "2026-01-02T00:00:00Z"),+ ("not-a-date", "not-a-date"),+ ],+)+def test_to_iso8601(value: Any, expected: str) -> None:+ assert to_iso8601(value) == expected+++def test_transaction_windows_are_contiguous_and_end_before_the_request_margin() -> None:+ start = NOW - timedelta(days=400)++ windows = transaction_windows(start, NOW)++ assert [window.posted_before - window.posted_after for window in windows[:-1]] == [timedelta(days=179)] * 2+ assert [window.posted_before for window in windows[:-1]] == [window.posted_after for window in windows[1:]]+ assert windows[0].posted_after == start+ assert windows[-1].posted_before == NOW - timedelta(seconds=180)+ assert transaction_windows(NOW - timedelta(seconds=60), NOW) == []diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.pyindex ab29ea90d58c..08d39843cb9b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py@@ -79,6 +79,7 @@ _AUTH_HEADER_NAMES: frozenset[str] = frozenset("x-sn-apikey","x-ck-api-key","x-auth-token",+ "x-amz-access-token","x-metabase-session","ob-token-v1","kalshi-access-signature",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.pyindex b336910cd587..fcf1f942e00e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py@@ -1,9 +1,15 @@# This file is automatically generated from `SourceRegistry.get_all_sources()`# Do not edit manually - run `pnpm generate:source-configs` to regenerate.+from typing import Literal+from products.warehouse_sources.backend.temporal.data_imports.sources.common import config@config.configclass AmazonSellingPartnerSourceConfig(config.Config):- pass+ client_id: str+ client_secret: str+ refresh_token: str+ region: Literal["na", "eu", "fe"] = config.value(default="na")+ marketplace_ids: str | None = None