Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): implement amazon selling partner source (#118147)

Daniel RC9d3a799a75c8
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
index 580e4affbe4a..94558929360b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md
@@ -77,6 +77,7 @@ the row lists both.
| alpha_vantage | HTTP | requests | ✅ |
| amazon_ads | HTTP | requests | ✅ |
| amazon_s3 | HTTP (vendor SDK) | boto3 (listing), s3fs (reads), deltalake (Delta tables) | ⚠️ |
+| amazon_selling_partner | HTTP | requests + `rest_source.RESTClient` | ✅ |
| amplemarket | HTTP | requests + `rest_source.RESTClient` | ✅ |
| amplitude | HTTP | requests | ✅ |
| anthropic | HTTP | requests | ✅ |
@@ -1038,7 +1039,6 @@ doesn't conflict with concurrent PRs.
- amazon_cloudwatch
- amazon_eventbridge
- amazon_kinesis
-- amazon_selling_partner
- amazon_sns
- amazon_sqs
- anodot_cost
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.py
new file mode 100644
index 000000000000..2e9c3fd64e2f
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/amazon_selling_partner.py
@@ -0,0 +1,561 @@
+import json
+import sqlite3
+import tempfile
+from collections.abc import Callable, Iterable, Iterator
+from dataclasses import field
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any, Optional
+
+import requests
+import structlog
+from requests import PreparedRequest
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (
+ DEFERRED_RELEASE_SWEEP_DAYS,
+ EARLIEST_START,
+ INVENTORY_SUMMARIES,
+ INVENTORY_SUMMARIES_PATH,
+ LWA_TOKEN_URL,
+ MARKETPLACE_PARTICIPATIONS_PATH,
+ ORDER_ITEMS,
+ ORDERS,
+ ORDERS_INCLUDED_DATA,
+ ORDERS_PAGE_SIZE,
+ ORDERS_PATH,
+ PARTITION_KEYS,
+ PRIMARY_KEYS,
+ REQUEST_TIME_MARGIN_SECONDS,
+ SP_API_HOSTS,
+ SP_API_MAX_RETRY_ATTEMPTS,
+ SP_API_RETRY_BACKOFF_MAX_SECONDS,
+ TRANSACTIONS,
+ TRANSACTIONS_PATH,
+ TRANSACTIONS_WINDOW_DAYS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
+ RESTAPIConfig,
+ rest_api_resource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
+ OAuth2Auth,
+ OAuth2AuthRequestError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+
+logger = structlog.get_logger(__name__)
+
+LWA_REJECTED_MESSAGE = (
+ "Amazon rejected the Login with Amazon credentials. "
+ "Check the client ID, client secret and refresh token, then try again."
+)
+ACCESS_DENIED_MESSAGE = (
+ "Amazon accepted the login but refused the Selling Partner API call. "
+ "Check that the app is authorized for this seller and has the roles the tables need."
+)
+UNAVAILABLE_MESSAGE = "Couldn't reach Amazon to check the credentials. Try again in a few minutes."
+
+REQUEST_TIMEOUT = (10.0, 30.0)
+INVENTORY_ROWS_PER_BATCH = 2000
+ResumeHook = Callable[[Optional[dict[str, Any]]], None]
+
+
+@frozen
+class AmazonSellingPartnerResumeConfig:
+ # The page token of the endpoint being walked, when the vendor's token outlives a hand-off.
+ next_token: str | None = None
+ # The transactions window being walked. Amazon accepts a page token only with the arguments that produced it.
+ window_start: str | None = None
+ window_end: str | None = None
+ transaction_status: str | None = None
+ # Marketplaces whose inventory walk finished. Inventory page tokens expire in 30 seconds, so a
+ # marketplace restarts from its first page.
+ completed_marketplaces: list[str] = field(default_factory=list)
+ marketplace_id: str | None = None
+ last_seller_sku: str | None = None
+
+
+class LwaAuth(OAuth2Auth):
+ """Login with Amazon refresh-token grant. SP-API reads the token from `x-amz-access-token`."""
+
+ def __init__(self, client_id: str, client_secret: str, refresh_token: str) -> None:
+ super().__init__(
+ token_url=LWA_TOKEN_URL,
+ client_id=client_id,
+ client_secret=client_secret,
+ grant_type="refresh_token",
+ refresh_token=refresh_token,
+ )
+
+ def __call__(self, request: PreparedRequest) -> PreparedRequest:
+ if self.token is None or self._is_token_expired():
+ self._obtain_token()
+ assert self.token is not None
+ request.headers["x-amz-access-token"] = self.token
+ return request
+
+
+def to_iso8601(value: Any) -> str:
+ parsed = parse_datetime_value(value)
+ if parsed is None:
+ return str(value)
+ return parsed.strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def parse_marketplace_ids(value: str | None) -> list[str]:
+ return [part.strip() for part in (value or "").split(",") if part.strip()]
+
+
+@frozen
+class TransactionWindow:
+ posted_after: datetime
+ posted_before: datetime
+ transaction_status: str | None = None
+
+
+def transaction_windows(start: datetime, now: datetime) -> list[TransactionWindow]:
+ """Contiguous [postedAfter, postedBefore) windows from `start` up to the latest time Amazon accepts."""
+ cutoff = now - timedelta(seconds=REQUEST_TIME_MARGIN_SECONDS)
+ windows: list[TransactionWindow] = []
+ while start < cutoff:
+ end = min(start + timedelta(days=TRANSACTIONS_WINDOW_DAYS), cutoff)
+ windows.append(TransactionWindow(posted_after=start, posted_before=end))
+ start = end
+ return windows
+
+
+def _base_url(region: str) -> str:
+ host = SP_API_HOSTS.get(region)
+ if host is None:
+ raise ValueError(f"Invalid Amazon Selling Partner region: {region}")
+ return host
+
+
+def _write_disposition(merge: bool) -> Any:
+ return {"disposition": "merge", "strategy": "upsert"} if merge else "replace"
+
+
+def _build_resource(
+ *,
+ region: str,
+ auth: LwaAuth,
+ name: str,
+ path: str,
+ data_selector: str,
+ cursor_path: str,
+ cursor_param: str,
+ params: dict[str, Any],
+ merge: bool,
+ team_id: int,
+ job_id: str,
+ db_incremental_field_last_value: Any,
+ resume_hook: ResumeHook | None = None,
+ initial_paginator_state: dict[str, Any] | None = None,
+) -> Resource:
+ config: RESTAPIConfig = {
+ "client": {
+ "base_url": _base_url(region),
+ "auth": auth,
+ "max_retries": SP_API_MAX_RETRY_ATTEMPTS,
+ "retry_backoff_max_seconds": SP_API_RETRY_BACKOFF_MAX_SECONDS,
+ # Without this a stalled SP-API host holds the whole retry budget in one try, and the
+ # waits above never run. The seller's other calls use the same bound.
+ "request_timeout": REQUEST_TIMEOUT,
+ # Order bodies carry buyer data that only the row maps remove, after any sample is taken.
+ "capture": False,
+ },
+ "resource_defaults": {},
+ "resources": [
+ {
+ "name": name,
+ "table_name": name,
+ "write_disposition": _write_disposition(merge),
+ "endpoint": {
+ "path": path,
+ "data_selector": data_selector,
+ "data_selector_empty_ok": True,
+ "paginator": {"type": "cursor", "cursor_path": cursor_path, "cursor_param": cursor_param},
+ "params": params,
+ },
+ "table_format": "delta",
+ }
+ ],
+ }
+ return rest_api_resource(
+ config,
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=resume_hook,
+ initial_paginator_state=initial_paginator_state,
+ )
+
+
+def _wrapper_hints(name: str, merge: bool) -> dict[str, Any]:
+ return {"table_name": name, "write_disposition": _write_disposition(merge), "table_format": "delta"}
+
+
+def drop_buyer_personal_data(order: dict[str, Any]) -> dict[str, Any]:
+ """Remove the buyer's personal data that rides inside the TAX and FULFILLMENT datasets."""
+ tax = order.get("tax")
+ if isinstance(tax, dict):
+ tax.pop("taxRegistrations", None)
+ for item in order.get("orderItems") or []:
+ product = item.get("product")
+ if isinstance(product, dict):
+ product.pop("customization", None)
+ gift_option = ((item.get("fulfillment") or {}).get("packing") or {}).get("giftOption")
+ if isinstance(gift_option, dict):
+ gift_option.pop("giftMessage", None)
+ return order
+
+
+def explode_order_items(order: dict[str, Any]) -> list[dict[str, Any]]:
+ parent = {key: order.get(key) for key in ("orderId", "createdTime", "lastUpdatedTime")}
+ parent["marketplaceId"] = (order.get("salesChannel") or {}).get("marketplaceId")
+ return [{**parent, **item} for item in order.get("orderItems") or []]
+
+
+def _orders_resource(
+ *,
+ region: str,
+ auth: LwaAuth,
+ marketplace_ids: list[str],
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Resource:
+ initial_paginator_state: dict[str, Any] | None = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None and resume.next_token:
+ initial_paginator_state = {"cursor": resume.next_token}
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(AmazonSellingPartnerResumeConfig(next_token=str(state["cursor"])))
+
+ resource = _build_resource(
+ region=region,
+ auth=auth,
+ name=endpoint,
+ path=ORDERS_PATH,
+ data_selector="orders",
+ cursor_path="pagination.nextToken",
+ cursor_param="paginationToken",
+ params={
+ "maxResultsPerPage": ORDERS_PAGE_SIZE,
+ "includedData": ORDERS_INCLUDED_DATA,
+ "marketplaceIds": ",".join(marketplace_ids) or None,
+ # Exactly one of createdAfter and lastUpdatedAfter is required; the latter also
+ # covers a full refresh, because an order's last update is never before its creation.
+ "lastUpdatedAfter": {
+ "type": "incremental",
+ "cursor_path": "lastUpdatedTime",
+ "initial_value": EARLIEST_START,
+ "convert": to_iso8601,
+ },
+ },
+ merge=should_use_incremental_field,
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ resource.add_map(drop_buyer_personal_data)
+ if endpoint == ORDER_ITEMS:
+ resource.add_map(explode_order_items)
+ return resource
+
+
+def deferred_release_sweep(now: datetime) -> TransactionWindow:
+ cutoff = now - timedelta(seconds=REQUEST_TIME_MARGIN_SECONDS)
+ return TransactionWindow(
+ posted_after=cutoff - timedelta(days=DEFERRED_RELEASE_SWEEP_DAYS),
+ posted_before=cutoff,
+ transaction_status="DEFERRED_RELEASED",
+ )
+
+
+def _transactions_resource(
+ *,
+ region: str,
+ auth: LwaAuth,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> Resource:
+ now = datetime.now(UTC)
+ start = parse_datetime_value(EARLIEST_START)
+ if should_use_incremental_field and db_incremental_field_last_value is not None:
+ start = parse_datetime_value(db_incremental_field_last_value) or start
+ assert start is not None
+ windows = transaction_windows(start, now)
+ initial_paginator_state: dict[str, Any] | None = None
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ if resume is not None and resume.window_start and resume.window_end:
+ after, before = parse_datetime_value(resume.window_start), parse_datetime_value(resume.window_end)
+ if after is not None and before is not None:
+ current = TransactionWindow(
+ posted_after=after, posted_before=before, transaction_status=resume.transaction_status
+ )
+ windows = [current]
+ if current.transaction_status is None:
+ windows += transaction_windows(before, now)
+ if resume.next_token:
+ initial_paginator_state = {"cursor": resume.next_token}
+ # A first sync reads the whole history already, so the sweep would only repeat rows.
+ has_watermark = should_use_incremental_field and db_incremental_field_last_value is not None
+ if has_watermark and (not windows or windows[-1].transaction_status is None):
+ windows.append(deferred_release_sweep(now))
+
+ def state_for(window: TransactionWindow, next_token: str | None = None) -> AmazonSellingPartnerResumeConfig:
+ return AmazonSellingPartnerResumeConfig(
+ window_start=to_iso8601(window.posted_after),
+ window_end=to_iso8601(window.posted_before),
+ transaction_status=window.transaction_status,
+ next_token=next_token,
+ )
+
+ def pages() -> Iterator[Any]:
+ for index, window in enumerate(windows):
+ following = windows[index + 1] if index + 1 < len(windows) else None
+
+ def save_checkpoint(
+ state: dict[str, Any] | None,
+ window: TransactionWindow = window,
+ following: TransactionWindow | None = following,
+ ) -> None:
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(state_for(window, str(state["cursor"])))
+ elif following is not None:
+ # Staged before the window's last page is handed on, so a resume skips the window.
+ resumable_source_manager.save_state(state_for(following))
+
+ params: dict[str, Any] = {
+ "postedAfter": to_iso8601(window.posted_after),
+ "postedBefore": to_iso8601(window.posted_before),
+ }
+ if window.transaction_status is not None:
+ params["transactionStatus"] = window.transaction_status
+ yield from _build_resource(
+ region=region,
+ auth=auth,
+ name=TRANSACTIONS,
+ path=TRANSACTIONS_PATH,
+ data_selector="payload.transactions",
+ cursor_path="payload.nextToken",
+ cursor_param="nextToken",
+ params=params,
+ merge=should_use_incremental_field,
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state if index == 0 else None,
+ )
+
+ return Resource(pages, name=TRANSACTIONS, hints=_wrapper_hints(TRANSACTIONS, should_use_incremental_field))
+
+
+def list_marketplace_ids(region: str, auth: LwaAuth) -> list[str]:
+ session = make_tracked_session(redact_values=auth.secret_values())
+ response = session.get(f"{_base_url(region)}{MARKETPLACE_PARTICIPATIONS_PATH}", auth=auth, timeout=REQUEST_TIMEOUT)
+ response.raise_for_status()
+ return [
+ entry["marketplace"]["id"]
+ for entry in response.json().get("payload") or []
+ if (entry.get("participation") or {}).get("isParticipating", True)
+ ]
+
+
+def _spool_inventory_pages(
+ session: requests.Session,
+ auth: LwaAuth,
+ url: str,
+ marketplace_id: str,
+ spool: sqlite3.Connection,
+ manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+ skip_if_not_covered: bool,
+) -> bool:
+ """Spool every page of one marketplace. False when Amazon rejects the marketplace on its first page."""
+ next_token: str | None = None
+ while True:
+ params: dict[str, Any] = {
+ "granularityType": "Marketplace",
+ "granularityId": marketplace_id,
+ "marketplaceIds": marketplace_id,
+ "details": "true",
+ }
+ if next_token:
+ params["nextToken"] = next_token
+ response = session.get(url, params=params, auth=auth, timeout=REQUEST_TIMEOUT)
+ # A discovered marketplace can be a channel FBA inventory does not cover. A later page's 400
+ # is a real failure, such as an expired page token.
+ if skip_if_not_covered and next_token is None and response.status_code == 400:
+ return False
+ response.raise_for_status()
+ body = response.json()
+ spool.executemany(
+ "INSERT OR REPLACE INTO inventory VALUES (?, ?)",
+ [
+ (str(summary.get("sellerSku") or ""), json.dumps({"marketplaceId": marketplace_id, **summary}))
+ for summary in (body.get("payload") or {}).get("inventorySummaries") or []
+ ],
+ )
+ next_token = (body.get("pagination") or {}).get("nextToken")
+ # The staged cursor does not cover this marketplace, so a hand-off here reads it again.
+ manager.safe_point()
+ if not next_token:
+ return True
+
+
+def _inventory_rows(
+ *,
+ region: str,
+ auth: LwaAuth,
+ marketplace_ids: list[str],
+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
+ manager = resumable_source_manager
+ resume = manager.load_state() if manager.can_resume() else None
+ completed = list(resume.completed_marketplaces) if resume is not None else []
+ session = make_tracked_session(redact_values=auth.secret_values())
+ url = f"{_base_url(region)}{INVENTORY_SUMMARIES_PATH}"
+ discovered = not marketplace_ids
+
+ for marketplace_id in marketplace_ids or list_marketplace_ids(region, auth):
+ if marketplace_id in completed:
+ continue
+ after_sku = resume.last_seller_sku if resume is not None and resume.marketplace_id == marketplace_id else None
+ done_state = AmazonSellingPartnerResumeConfig(completed_marketplaces=[*completed, marketplace_id])
+ # Page tokens expire in 30 seconds and Amazon documents no SKU order, so rows wait on disk
+ # sorted by SKU and a resume continues after the last SKU handed on.
+ with tempfile.TemporaryDirectory() as spool_dir:
+ # The pipeline may resume this generator on another thread.
+ spool = sqlite3.connect(Path(spool_dir) / "inventory.sqlite", check_same_thread=False)
+ try:
+ spool.execute("CREATE TABLE inventory (seller_sku TEXT PRIMARY KEY, row TEXT NOT NULL)")
+ if not _spool_inventory_pages(
+ session, auth, url, marketplace_id, spool, manager, skip_if_not_covered=discovered
+ ):
+ logger.warning(
+ "amazon_selling_partner_inventory_marketplace_skipped", marketplace_id=marketplace_id
+ )
+ manager.save_state(done_state)
+ manager.safe_point()
+ completed = done_state.completed_marketplaces
+ continue
+ cursor = spool.execute(
+ "SELECT seller_sku, row FROM inventory WHERE seller_sku > ? ORDER BY seller_sku", (after_sku or "",)
+ )
+ batch = cursor.fetchmany(INVENTORY_ROWS_PER_BATCH)
+ if not batch:
+ manager.save_state(done_state)
+ manager.safe_point()
+ while batch:
+ following = cursor.fetchmany(INVENTORY_ROWS_PER_BATCH)
+ manager.save_state(
+ done_state
+ if not following
+ else AmazonSellingPartnerResumeConfig(
+ completed_marketplaces=completed,
+ marketplace_id=marketplace_id,
+ last_seller_sku=batch[-1][0],
+ )
+ )
+ yield [json.loads(row) for _, row in batch]
+ batch = following
+ finally:
+ spool.close()
+ completed = done_state.completed_marketplaces
+
+
+def amazon_selling_partner_source(
+ *,
+ region: str,
+ client_id: str,
+ client_secret: str,
+ refresh_token: str,
+ marketplace_ids: list[str],
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Any,
+) -> SourceResponse:
+ auth = LwaAuth(client_id, client_secret, refresh_token)
+ common = {
+ "region": region,
+ "auth": auth,
+ "team_id": team_id,
+ "job_id": job_id,
+ "resumable_source_manager": resumable_source_manager,
+ "should_use_incremental_field": should_use_incremental_field,
+ "db_incremental_field_last_value": db_incremental_field_last_value,
+ }
+ items: Callable[[], Iterable[Any]]
+ column_hints: dict[str, Any] | None = None
+ if endpoint == INVENTORY_SUMMARIES:
+ items = lambda: _inventory_rows(
+ region=region, auth=auth, marketplace_ids=marketplace_ids, resumable_source_manager=resumable_source_manager
+ )
+ else:
+ resource: Resource
+ if endpoint in (ORDERS, ORDER_ITEMS):
+ resource = _orders_resource(marketplace_ids=marketplace_ids, endpoint=endpoint, **common)
+ elif endpoint == TRANSACTIONS:
+ resource = _transactions_resource(**common)
+ else:
+ raise ValueError(f"Unknown Amazon Selling Partner endpoint: {endpoint}")
+ items = lambda: resource
+ column_hints = resource.column_hints
+
+ partition_key = PARTITION_KEYS.get(endpoint)
+ return SourceResponse(
+ name=endpoint,
+ items=items,
+ primary_keys=PRIMARY_KEYS[endpoint],
+ # Amazon documents no order for any of these listings. Desc stores the watermark when the
+ # run completes, which is correct whichever order the rows arrive in.
+ sort_mode="desc",
+ partition_count=1 if partition_key else None,
+ partition_size=1 if partition_key else None,
+ partition_mode="datetime" if partition_key else None,
+ partition_format="month" if partition_key else None,
+ partition_keys=[partition_key] if partition_key else None,
+ column_hints=column_hints,
+ )
+
+
+def validate_credentials(
+ region: str, client_id: str, client_secret: str, refresh_token: str
+) -> tuple[bool, str | None]:
+ auth = LwaAuth(client_id, client_secret, refresh_token)
+ session = make_tracked_session(redact_values=auth.secret_values(), capture=False)
+ try:
+ response = session.get(
+ f"{_base_url(region)}{MARKETPLACE_PARTICIPATIONS_PATH}", auth=auth, timeout=REQUEST_TIMEOUT
+ )
+ except OAuth2AuthRequestError as e:
+ return False, LWA_REJECTED_MESSAGE if e.is_permanent else UNAVAILABLE_MESSAGE
+ except requests.RequestException:
+ return False, UNAVAILABLE_MESSAGE
+ if response.status_code == 200:
+ return True, None
+ if response.status_code in (401, 403):
+ return False, ACCESS_DENIED_MESSAGE
+ return False, UNAVAILABLE_MESSAGE
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.py
new file mode 100644
index 000000000000..371d1ba285a0
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/canonical_descriptions.py
@@ -0,0 +1,78 @@
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
+
+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
+ "orders": {
+ "description": "A customer order placed with the seller, with its items nested in orderItems.",
+ "docs_url": "https://developer-docs.amazon.com/sp-api/reference/orders-v2026-01-01",
+ "columns": {
+ "orderId": "Amazon-defined order identifier.",
+ "orderAliases": "Alternative identifiers for the order, such as a seller-defined order number.",
+ "createdTime": "Time the customer placed the order, in ISO 8601 format.",
+ "lastUpdatedTime": "Most recent time any aspect of the order changed, by Amazon or the seller.",
+ "programs": "Special programs the order belongs to, such as PRIME or AMAZON_BUSINESS.",
+ "associatedOrders": "Orders directly related to this one, such as a replacement or exchange.",
+ "salesChannel": "Where the order was placed, including the marketplace id and name.",
+ "proceeds": "Revenue and financial breakdown for the order.",
+ "payment": "Payment information for the order.",
+ "tax": "Tax information for the order.",
+ "fulfillment": "Fulfillment status, who fulfills the order and the ship and delivery windows.",
+ "orderItems": "The items in the order.",
+ "packages": "Shipping packages and tracking for the order.",
+ },
+ },
+ "order_items": {
+ "description": "One row per item in an order, with the order's id, creation time and last update time.",
+ "docs_url": "https://developer-docs.amazon.com/sp-api/reference/orders-v2026-01-01",
+ "columns": {
+ "orderId": "Amazon-defined identifier of the order the item belongs to.",
+ "createdTime": "Time the customer placed the order, in ISO 8601 format.",
+ "lastUpdatedTime": "Most recent time any aspect of the order changed.",
+ "marketplaceId": "Marketplace the order was placed in.",
+ "orderItemId": "Amazon-defined identifier of the order item.",
+ "quantityOrdered": "Number of units the customer ordered.",
+ "product": "The product ordered, including its ASIN, seller SKU and title.",
+ "proceeds": "Revenue breakdown for the item.",
+ "expense": "Cost information for the item.",
+ "promotion": "Discounts and promotional offers applied to the item.",
+ "cancellation": "Cancellation information for the item.",
+ "fulfillment": "Fulfillment status and quantities for the item.",
+ "tax": "Tax information for the item.",
+ },
+ },
+ "transactions": {
+ "description": "A financial transaction on the seller account, such as an order payment, refund or fee.",
+ "docs_url": "https://developer-docs.amazon.com/sp-api/docs/finances-api-v2024-06-19-reference",
+ "columns": {
+ "transactionId": "Unique identifier of the transaction.",
+ "transactionType": "Type of transaction, such as Shipment.",
+ "transactionStatus": "DEFERRED, RELEASED or DEFERRED_RELEASED.",
+ "description": "Reason for the transaction, such as Order Payment or Refund Order.",
+ "postedDate": "Date and time the transaction was posted.",
+ "totalAmount": "Total amount of money in the transaction, with its currency code.",
+ "marketplaceDetails": "Marketplace where the transaction occurred.",
+ "relatedIdentifiers": "Identifiers related to the transaction, such as the order and shipment ids.",
+ "sellingPartnerMetadata": "The seller's id, account type and marketplace.",
+ "items": "The items in the transaction with their amounts.",
+ "contexts": "Additional information about the transaction.",
+ "breakdowns": "How the total amount is calculated.",
+ },
+ },
+ "inventory_summaries": {
+ "description": "FBA inventory levels per seller SKU and marketplace.",
+ "docs_url": "https://developer-docs.amazon.com/sp-api/docs/fba-inventory-api-v1-reference",
+ "columns": {
+ "marketplaceId": "Marketplace the inventory summary was requested for.",
+ "asin": "Amazon Standard Identification Number of the item.",
+ "fnSku": "Amazon fulfillment network SKU of the item.",
+ "sellerSku": "Seller SKU of the item.",
+ "condition": "Condition of the item as described by the seller.",
+ "inventoryDetails": "Fulfillable, inbound, reserved, unfulfillable and researching quantities.",
+ "lastUpdatedTime": "Date and time any quantity was last updated.",
+ "productName": "Localized product title in the marketplace.",
+ "totalQuantity": "Total units in inbound shipments or in Amazon fulfillment centers.",
+ "stores": "Seller-enrolled stores that apply to the SKU.",
+ },
+ },
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.py
new file mode 100644
index 000000000000..354d430dd063
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/settings.py
@@ -0,0 +1,63 @@
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
+from products.warehouse_sources.backend.types import IncrementalField
+
+SP_API_HOSTS = {
+ "na": "https://sellingpartnerapi-na.amazon.com",
+ "eu": "https://sellingpartnerapi-eu.amazon.com",
+ "fe": "https://sellingpartnerapi-fe.amazon.com",
+}
+LWA_TOKEN_URL = "https://api.amazon.com/auth/o2/token"
+
+ORDERS_PATH = "/orders/2026-01-01/orders"
+TRANSACTIONS_PATH = "/finances/2024-06-19/transactions"
+INVENTORY_SUMMARIES_PATH = "/fba/inventory/v1/summaries"
+MARKETPLACE_PARTICIPATIONS_PATH = "/sellers/v1/marketplaceParticipations"
+
+ORDERS_PAGE_SIZE = 100
+# BUYER and RECIPIENT are left out: they carry PII and need restricted roles on the app.
+ORDERS_INCLUDED_DATA = "FULFILLMENT,PROCEEDS,EXPENSE,PROMOTION,CANCELLATION,PACKAGES,TAX,PAYMENT"
+# listTransactions answers with an empty page when postedAfter and postedBefore are over 180 days apart.
+TRANSACTIONS_WINDOW_DAYS = 179
+# Amazon rejects a window end closer than two minutes to the request time.
+REQUEST_TIME_MARGIN_SECONDS = 180
+# The oldest order data any marketplace keeps (JP, AU and SG keep orders from 2016).
+EARLIEST_START = "2016-01-01T00:00:00Z"
+# Financial events can post up to 48 hours late.
+TRANSACTIONS_LOOKBACK_SECONDS = 3 * 24 * 60 * 60
+# A deferred transaction keeps its postedDate when Amazon releases it, so each incremental run also
+# re-reads the transactions released in this many days.
+DEFERRED_RELEASE_SWEEP_DAYS = 60
+# searchOrders refills at one request every 179 seconds once its burst is spent, and SP-API sends
+# no Retry-After. Nine attempts wait 1, 2, 4 ... 128 seconds, 255 in all, past that interval and
+# inside the shared retry budget, which ends a request 300 seconds after its first try.
+SP_API_MAX_RETRY_ATTEMPTS = 9
+SP_API_RETRY_BACKOFF_MAX_SECONDS = 128.0
+
+ORDERS = "orders"
+ORDER_ITEMS = "order_items"
+TRANSACTIONS = "transactions"
+INVENTORY_SUMMARIES = "inventory_summaries"
+
+ENDPOINTS = (ORDERS, ORDER_ITEMS, TRANSACTIONS, INVENTORY_SUMMARIES)
+
+PRIMARY_KEYS: dict[str, list[str]] = {
+ ORDERS: ["orderId"],
+ ORDER_ITEMS: ["orderId", "orderItemId"],
+ TRANSACTIONS: ["transactionId"],
+ INVENTORY_SUMMARIES: ["marketplaceId", "sellerSku"],
+}
+
+PARTITION_KEYS: dict[str, str] = {
+ ORDERS: "createdTime",
+ ORDER_ITEMS: "createdTime",
+ TRANSACTIONS: "postedDate",
+}
+
+
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ ORDERS: [incremental_field("lastUpdatedTime")],
+ ORDER_ITEMS: [incremental_field("lastUpdatedTime")],
+ TRANSACTIONS: [incremental_field("postedDate")],
+}
+
+DEFAULT_INCREMENTAL_LOOKBACK_SECONDS: dict[str, int] = {TRANSACTIONS: TRANSACTIONS_LOOKBACK_SECONDS}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py
index 4ca298a89030..377799a9ce54 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/source.py
@@ -1,8 +1,40 @@
-from typing import cast
+from typing import Optional, cast
-from products.warehouse_sources.backend.facade.source_config import DataWarehouseSourceCategory, SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.backend.facade.source_config import (
+ DataWarehouseSourceCategory,
+ ReleaseStatus,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigOption,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.amazon_selling_partner import (
+ AmazonSellingPartnerResumeConfig,
+ amazon_selling_partner_source,
+ parse_marketplace_ids,
+ validate_credentials as validate_amazon_selling_partner_credentials,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (
+ DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+ CanonicalDescriptions,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
+ OAUTH2_PERMANENT_ERROR_MARKER,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
+ SourceSchema,
+ build_endpoint_schemas,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazonsellingpartner import (
AmazonSellingPartnerSourceConfig,
)
@@ -10,18 +42,158 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class AmazonSellingPartnerSource(SimpleSource[AmazonSellingPartnerSourceConfig]):
+class AmazonSellingPartnerSource(ResumableSource[AmazonSellingPartnerSourceConfig, AmazonSellingPartnerResumeConfig]):
+ codebase_detection = detect(
+ env=("^SP_API_",),
+ npm=("amazon-sp-api", "@sp-api-sdk/orders-api-v0", "@sp-api-sdk/auth"),
+ pypi=("python-amazon-sp-api",),
+ rubygems=("amz_sp_api",),
+ composer=("jlevers/selling-partner-api", "highsidelabs/laravel-spapi"),
+ go=("github.com/amzapi/selling-partner-api-sdk",),
+ )
+ # The refresh token comes from a seller-side authorization flow, so users set this up in the app.
+ setup_requires_ui = True
+
+ lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
+ # Each SP-API section carries its own dated version, so there is no single pin for the source.
+ api_docs_url = "https://developer-docs.amazon.com/sp-api/changelog"
+
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.AMAZONSELLINGPARTNER
+ def get_non_retryable_errors(self) -> dict[str, str | None]:
+ return {
+ "HTTP 400 from the OAuth2 token endpoint": "Amazon rejected the refresh token. Re-authorize the app for this seller and update the refresh token.",
+ "HTTP 401 from the OAuth2 token endpoint": "Amazon rejected the Login with Amazon client ID or client secret. Check them in the Amazon developer console and update the source.",
+ "401 Client Error: Unauthorized for url: https://sellingpartnerapi": "Amazon rejected the access token. Re-authorize the app for this seller and update the refresh token.",
+ "403 Client Error: Forbidden for url: https://sellingpartnerapi": "Amazon refused access to this table. Check that the app has the role the table needs and that the seller authorized it.",
+ OAUTH2_PERMANENT_ERROR_MARKER: "Amazon rejected the Login with Amazon credentials. Check the client ID, client secret and refresh token, then update the source.",
+ }
+
+ def get_canonical_descriptions(self) -> CanonicalDescriptions:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.canonical_descriptions import (
+ CANONICAL_DESCRIPTIONS,
+ )
+
+ return CANONICAL_DESCRIPTIONS
+
+ def get_schemas(
+ self,
+ config: AmazonSellingPartnerSourceConfig,
+ team_id: int,
+ with_counts: bool = False,
+ names: list[str] | None = None,
+ force_refresh: bool = False,
+ api_version: str | None = None,
+ ) -> list[SourceSchema]:
+ return build_endpoint_schemas(
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ names,
+ default_incremental_lookback_seconds=DEFAULT_INCREMENTAL_LOOKBACK_SECONDS,
+ )
+
+ def validate_credentials(
+ self,
+ config: AmazonSellingPartnerSourceConfig,
+ team_id: int,
+ schema_name: Optional[str] = None,
+ api_version: str | None = None,
+ ) -> tuple[bool, str | None]:
+ return validate_amazon_selling_partner_credentials(
+ config.region, config.client_id, config.client_secret, config.refresh_token
+ )
+
+ def get_resumable_source_manager(
+ self, inputs: SourceInputs
+ ) -> ResumableSourceManager[AmazonSellingPartnerResumeConfig]:
+ return ResumableSourceManager[AmazonSellingPartnerResumeConfig](inputs, AmazonSellingPartnerResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: AmazonSellingPartnerSourceConfig,
+ resumable_source_manager: ResumableSourceManager[AmazonSellingPartnerResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
+ return amazon_selling_partner_source(
+ region=config.region,
+ client_id=config.client_id,
+ client_secret=config.client_secret,
+ refresh_token=config.refresh_token,
+ marketplace_ids=parse_marketplace_ids(config.marketplace_ids),
+ endpoint=inputs.schema_name,
+ team_id=inputs.team_id,
+ job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value
+ if inputs.should_use_incremental_field
+ else None,
+ )
+
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.AMAZONSELLINGPARTNER,
category=DataWarehouseSourceCategory.E_COMMERCE,
label="Amazon Selling Partner",
+ keywords=["sp-api", "seller central", "amazon seller", "fba"],
+ caption=(
+ "Syncs orders, order items, financial transactions and FBA inventory from your Amazon seller account. "
+ "The app needs the Inventory and Order Tracking, Finance and Accounting and Amazon Fulfillment roles. "
+ "Orders and order items read the same Amazon endpoint, so syncing both takes twice as long."
+ ),
iconPath="/static/services/amazon_selling_partner.png",
- fields=cast(list[FieldType], []),
- unreleasedSource=True,
+ docsUrl="https://posthog.com/docs/cdp/sources/amazon-selling-partner",
+ releaseStatus=ReleaseStatus.ALPHA,
+ fields=cast(
+ list[FieldType],
+ [
+ SourceFieldSelectConfig(
+ name="region",
+ label="Region",
+ required=True,
+ defaultValue="na",
+ options=[
+ SourceFieldSelectConfigOption(label="North America", value="na"),
+ SourceFieldSelectConfigOption(label="Europe", value="eu"),
+ SourceFieldSelectConfigOption(label="Far East", value="fe"),
+ ],
+ ),
+ SourceFieldInputConfig(
+ name="client_id",
+ label="LWA client ID",
+ type=SourceFieldInputConfigType.TEXT,
+ required=True,
+ placeholder="amzn1.application-oa2-client...",
+ secret=False,
+ ),
+ SourceFieldInputConfig(
+ name="client_secret",
+ label="LWA client secret",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="",
+ secret=True,
+ ),
+ SourceFieldInputConfig(
+ name="refresh_token",
+ label="Refresh token",
+ type=SourceFieldInputConfigType.PASSWORD,
+ required=True,
+ placeholder="Atzr|...",
+ secret=True,
+ ),
+ SourceFieldInputConfig(
+ name="marketplace_ids",
+ label="Marketplace IDs",
+ caption="Comma-separated. Applies to orders, order items and inventory, while transactions always cover every marketplace. Leave blank to use every marketplace the seller sells in.",
+ type=SourceFieldInputConfigType.TEXT,
+ required=False,
+ placeholder="ATVPDKIKX0DER, A1F83G8C2ARO7P",
+ secret=False,
+ ),
+ ],
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.py
new file mode 100644
index 000000000000..d8f254d30e9e
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_selling_partner/tests/test_amazon_selling_partner.py
@@ -0,0 +1,559 @@
+import json
+from collections.abc import Iterator
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from urllib.parse import parse_qs
+
+import pytest
+import time_machine
+
+import requests
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.amazon_selling_partner import (
+ ACCESS_DENIED_MESSAGE,
+ LWA_REJECTED_MESSAGE,
+ UNAVAILABLE_MESSAGE,
+ AmazonSellingPartnerResumeConfig,
+ to_iso8601,
+ transaction_windows,
+ validate_credentials,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.settings import (
+ EARLIEST_START,
+ ORDERS_INCLUDED_DATA,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_selling_partner.source import (
+ AmazonSellingPartnerSource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (
+ OAuth2AuthRequestError,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ DriveResult,
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ always,
+ route,
+ scripted_network,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.amazonsellingpartner import (
+ AmazonSellingPartnerSourceConfig,
+)
+
+TOKEN_PATH = "/auth/o2/token"
+ORDERS_PATH = "/orders/2026-01-01/orders"
+TRANSACTIONS_PATH = "/finances/2024-06-19/transactions"
+SUMMARIES_PATH = "/fba/inventory/v1/summaries"
+PARTICIPATIONS_PATH = "/sellers/v1/marketplaceParticipations"
+
+TOKEN = ScriptedResponse(
+ json={"access_token": "Atza|access", "refresh_token": "Atzr|refresh", "token_type": "bearer", "expires_in": 3600}
+)
+LWA_INVALID_GRANT = ScriptedResponse(
+ status=400,
+ json={"error": "invalid_grant", "error_description": "The request has an invalid grant parameter : refresh_token"},
+)
+SP_API_FORBIDDEN = ScriptedResponse(
+ status=403,
+ json={"errors": [{"code": "Unauthorized", "message": "Access to requested resource is denied.", "details": ""}]},
+)
+NOW = datetime(2026, 10, 15, 12, 0, 0, tzinfo=UTC)
+
+
+def _driver(marketplace_ids: str | None = None) -> SourceDriver:
+ config = AmazonSellingPartnerSourceConfig(
+ client_id="amzn1.application-oa2-client.test",
+ client_secret="client-secret",
+ refresh_token="Atzr|refresh",
+ region="na",
+ marketplace_ids=marketplace_ids,
+ )
+ return SourceDriver(AmazonSellingPartnerSource(), config)
+
+
+def _api_requests(result: DriveResult) -> list[RecordedRequest]:
+ return [request for request in result.requests if request.path != TOKEN_PATH]
+
+
+def _order(order_id: str, items: list[dict[str, Any]] | None = None) -> dict[str, Any]:
+ return {
+ "orderId": order_id,
+ "createdTime": "2026-01-02T03:04:05Z",
+ "lastUpdatedTime": "2026-01-03T00:00:00Z",
+ "salesChannel": {"marketplaceId": "ATVPDKIKX0DER", "marketplaceName": "Amazon.com"},
+ "orderItems": items or [],
+ }
+
+
+def _orders_page(orders: list[dict[str, Any]], next_token: str | None = None) -> ScriptedResponse:
+ return ScriptedResponse(json={"orders": orders, "pagination": {"nextToken": next_token} if next_token else {}})
+
+
+def _transactions_page(ids: list[str], next_token: str | None = None) -> ScriptedResponse:
+ payload: dict[str, Any] = {
+ "transactions": [{"transactionId": id, "postedDate": "2026-09-02T00:00:00Z"} for id in ids]
+ }
+ if next_token:
+ payload["nextToken"] = next_token
+ return ScriptedResponse(json={"payload": payload})
+
+
+def _summaries_page(skus: list[str], next_token: str | None = None) -> ScriptedResponse:
+ body: dict[str, Any] = {"payload": {"inventorySummaries": [{"sellerSku": sku, "totalQuantity": 1} for sku in skus]}}
+ if next_token:
+ body["pagination"] = {"nextToken": next_token}
+ return ScriptedResponse(json=body)
+
+
+class TestOrders:
+ def test_walks_pages_with_the_lwa_token(self) -> None:
+ result = _driver("ATVPDKIKX0DER").run(
+ "orders",
+ route(
+ {
+ TOKEN_PATH: [TOKEN],
+ ORDERS_PATH: [_orders_page([_order("1"), _order("2")], "tok2"), _orders_page([_order("3")])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ token_request = result.requests[0]
+ assert token_request.path == TOKEN_PATH
+ assert parse_qs((token_request.body or b"").decode()) == {
+ "grant_type": ["refresh_token"],
+ "refresh_token": ["Atzr|refresh"],
+ "client_id": ["amzn1.application-oa2-client.test"],
+ "client_secret": ["client-secret"],
+ }
+ api_requests = _api_requests(result)
+ assert [request.path for request in api_requests] == [ORDERS_PATH, ORDERS_PATH]
+ assert all(request.headers["x-amz-access-token"] == "Atza|access" for request in api_requests)
+ assert [request.param("paginationToken") for request in api_requests] == [None, "tok2"]
+ assert api_requests[0].param("maxResultsPerPage") == "100"
+ assert api_requests[0].param("includedData") == ORDERS_INCLUDED_DATA
+ assert [row["orderId"] for row in result.rows] == ["1", "2", "3"]
+ assert result.saved_states == [AmazonSellingPartnerResumeConfig(next_token="tok2")]
+ # Order bodies carry buyer data, so no session may store HTTP samples of them.
+ assert result.session_options and all(options.get("capture") is False for options in result.session_options)
+
+ @pytest.mark.parametrize(
+ ("incremental_field", "last_value", "marketplace_ids", "expected_after", "expected_marketplaces"),
+ [
+ (None, None, None, EARLIEST_START, None),
+ (
+ "lastUpdatedTime",
+ datetime(2026, 3, 4, 5, 6, 7, tzinfo=UTC),
+ "ATVPDKIKX0DER, A1F83G8C2ARO7P",
+ "2026-03-04T05:06:07Z",
+ "ATVPDKIKX0DER,A1F83G8C2ARO7P",
+ ),
+ ("lastUpdatedTime", "2026-03-04 05:06:07", None, "2026-03-04T05:06:07Z", None),
+ ],
+ )
+ def test_request_shaping(
+ self,
+ incremental_field: str | None,
+ last_value: Any,
+ marketplace_ids: str | None,
+ expected_after: str,
+ expected_marketplaces: str | None,
+ ) -> None:
+ result = _driver(marketplace_ids).run(
+ "orders",
+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("1")])]}),
+ incremental_field=incremental_field,
+ db_incremental_field_last_value=last_value,
+ )
+
+ request = _api_requests(result)[0]
+ assert request.param("lastUpdatedAfter") == expected_after
+ assert request.param("marketplaceIds") == expected_marketplaces
+ assert request.param("createdAfter") is None
+
+ def test_order_items_explode_each_order_with_its_parent_fields(self) -> None:
+ items = [{"orderItemId": "i1", "quantityOrdered": 2}, {"orderItemId": "i2", "quantityOrdered": 1}]
+ result = _driver().run(
+ "order_items",
+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("A", items), _order("B")])]}),
+ )
+
+ assert result.raised is None
+ assert result.rows == [
+ {
+ "orderId": "A",
+ "createdTime": "2026-01-02T03:04:05Z",
+ "lastUpdatedTime": "2026-01-03T00:00:00Z",
+ "marketplaceId": "ATVPDKIKX0DER",
+ "orderItemId": "i1",
+ "quantityOrdered": 2,
+ },
+ {
+ "orderId": "A",
+ "createdTime": "2026-01-02T03:04:05Z",
+ "lastUpdatedTime": "2026-01-03T00:00:00Z",
+ "marketplaceId": "ATVPDKIKX0DER",
+ "orderItemId": "i2",
+ "quantityOrdered": 1,
+ },
+ ]
+
+ def test_buyer_personal_data_is_dropped_from_orders_and_items(self) -> None:
+ item = {
+ "orderItemId": "i1",
+ "product": {"asin": "B01", "customization": {"customizedUrl": "https://example.com/c"}},
+ "fulfillment": {"packing": {"giftOption": {"giftMessage": "Happy birthday", "giftWrapLevel": "CLASSIC"}}},
+ }
+ order = {**_order("A", [item]), "tax": {"taxRegistrations": [{"legalName": "Jane Example"}], "taxRate": 0.2}}
+
+ for endpoint in ("orders", "order_items"):
+ result = _driver().run(
+ endpoint, route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([json.loads(json.dumps(order))])]})
+ )
+
+ assert result.raised is None
+ serialized = json.dumps(result.rows)
+ assert "Jane Example" not in serialized
+ assert "Happy birthday" not in serialized
+ assert "customizedUrl" not in serialized
+ assert "CLASSIC" in serialized
+
+ def test_orders_keep_retrying_through_amazons_refill_interval(self) -> None:
+ throttled = ScriptedResponse(status=429, json={"errors": [{"code": "QuotaExceeded"}]})
+ result = _driver().run(
+ "orders",
+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [throttled] * 8 + [_orders_page([_order("1")])]}),
+ )
+
+ assert result.raised is None
+ assert [row["orderId"] for row in result.rows] == ["1"]
+
+ def test_resumes_from_the_saved_page_token(self) -> None:
+ result = _driver().run(
+ "orders",
+ route({TOKEN_PATH: [TOKEN], ORDERS_PATH: [_orders_page([_order("9")])]}),
+ resume_state=AmazonSellingPartnerResumeConfig(next_token="tok9"),
+ )
+
+ assert _api_requests(result)[0].param("paginationToken") == "tok9"
+ assert [row["orderId"] for row in result.rows] == ["9"]
+
+
+class TestTransactions:
+ @pytest.fixture(autouse=True)
+ def frozen_clock(self) -> Iterator[None]:
+ with time_machine.travel(NOW, tick=False):
+ yield
+
+ def test_full_refresh_walks_contiguous_windows_from_the_earliest_start(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return TOKEN
+ return _transactions_page([request.param("postedAfter") or ""])
+
+ result = _driver().run("transactions", respond)
+
+ assert result.raised is None
+ windows = transaction_windows(datetime(2016, 1, 1, tzinfo=UTC), NOW)
+ api_requests = _api_requests(result)
+ assert [(r.param("postedAfter"), r.param("postedBefore")) for r in api_requests] == [
+ (to_iso8601(w.posted_after), to_iso8601(w.posted_before)) for w in windows
+ ]
+ assert api_requests[0].param("postedAfter") == EARLIEST_START
+ assert api_requests[-1].param("postedBefore") == "2026-10-15T11:57:00Z"
+ assert all(r.param("nextToken") is None and r.param("transactionStatus") is None for r in api_requests)
+ assert result.saved_states == [
+ AmazonSellingPartnerResumeConfig(
+ window_start=to_iso8601(w.posted_after), window_end=to_iso8601(w.posted_before)
+ )
+ for w in windows[1:]
+ ]
+ assert len(result.rows) == len(windows)
+
+ def test_incremental_walks_from_the_watermark_then_sweeps_released_deferred_transactions(self) -> None:
+ result = _driver().run(
+ "transactions",
+ route(
+ {
+ TOKEN_PATH: [TOKEN],
+ TRANSACTIONS_PATH: [
+ _transactions_page(["t1"], "n2"),
+ _transactions_page(["t2"]),
+ _transactions_page(["t-released"]),
+ ],
+ }
+ ),
+ incremental_field="postedDate",
+ db_incremental_field_last_value=datetime(2026, 9, 1, tzinfo=UTC),
+ )
+
+ assert result.raised is None
+ assert [
+ (r.param("postedAfter"), r.param("postedBefore"), r.param("transactionStatus"), r.param("nextToken"))
+ for r in _api_requests(result)
+ ] == [
+ ("2026-09-01T00:00:00Z", "2026-10-15T11:57:00Z", None, None),
+ ("2026-09-01T00:00:00Z", "2026-10-15T11:57:00Z", None, "n2"),
+ ("2026-08-16T11:57:00Z", "2026-10-15T11:57:00Z", "DEFERRED_RELEASED", None),
+ ]
+ assert [row["transactionId"] for row in result.rows] == ["t1", "t2", "t-released"]
+ first_page_state = AmazonSellingPartnerResumeConfig(
+ window_start="2026-09-01T00:00:00Z", window_end="2026-10-15T11:57:00Z", next_token="n2"
+ )
+ assert result.committed_states[0] == first_page_state
+
+ def test_a_first_incremental_sync_skips_the_deferred_sweep(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ return TOKEN if request.path == TOKEN_PATH else _transactions_page([])
+
+ result = _driver().run("transactions", respond, incremental_field="postedDate")
+
+ assert result.raised is None
+ assert all(request.param("transactionStatus") is None for request in _api_requests(result))
+
+ @pytest.mark.parametrize(
+ ("saved_window", "expected"),
+ [
+ (
+ ("2026-10-01T00:00:00Z", "2026-10-14T00:00:00Z", "DEFERRED_RELEASED"),
+ [("2026-10-01T00:00:00Z", "2026-10-14T00:00:00Z", "DEFERRED_RELEASED", "n2")],
+ ),
+ (
+ ("2026-09-01T00:00:00Z", "2026-10-01T00:00:00Z", None),
+ [
+ ("2026-09-01T00:00:00Z", "2026-10-01T00:00:00Z", None, "n2"),
+ ("2026-10-01T00:00:00Z", "2026-10-15T11:57:00Z", None, None),
+ ("2026-08-16T11:57:00Z", "2026-10-15T11:57:00Z", "DEFERRED_RELEASED", None),
+ ],
+ ),
+ ],
+ )
+ def test_resume_reuses_the_saved_window_with_its_token(
+ self, saved_window: tuple[str, str, str | None], expected: list[tuple[str, str, str | None, str | None]]
+ ) -> None:
+ window_start, window_end, transaction_status = saved_window
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ return TOKEN if request.path == TOKEN_PATH else _transactions_page(["t"])
+
+ result = _driver().run(
+ "transactions",
+ respond,
+ resume_state=AmazonSellingPartnerResumeConfig(
+ window_start=window_start,
+ window_end=window_end,
+ transaction_status=transaction_status,
+ next_token="n2",
+ ),
+ incremental_field="postedDate",
+ db_incremental_field_last_value=datetime(2026, 9, 1, tzinfo=UTC),
+ )
+
+ assert [
+ (r.param("postedAfter"), r.param("postedBefore"), r.param("transactionStatus"), r.param("nextToken"))
+ for r in _api_requests(result)
+ ] == expected
+
+
+class TestInventorySummaries:
+ def test_discovers_marketplaces_reads_each_one_whole_and_tags_rows(self) -> None:
+ participations = ScriptedResponse(
+ json={
+ "payload": [
+ {"marketplace": {"id": "A"}, "participation": {"isParticipating": True}},
+ {"marketplace": {"id": "X"}, "participation": {"isParticipating": False}},
+ {"marketplace": {"id": "B"}, "participation": {"isParticipating": True}},
+ ]
+ }
+ )
+ result = _driver().run(
+ "inventory_summaries",
+ route(
+ {
+ TOKEN_PATH: [TOKEN],
+ PARTICIPATIONS_PATH: [participations],
+ SUMMARIES_PATH: [
+ _summaries_page(["sku1"], "next"),
+ _summaries_page(["sku2"]),
+ _summaries_page(["sku3"]),
+ ],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ summaries = [request for request in _api_requests(result) if request.path == SUMMARIES_PATH]
+ assert [(r.param("granularityId"), r.param("marketplaceIds"), r.param("nextToken")) for r in summaries] == [
+ ("A", "A", None),
+ ("A", "A", "next"),
+ ("B", "B", None),
+ ]
+ assert summaries[0].param("details") == "true"
+ assert all(r.param("startDateTime") is None for r in summaries)
+ # Both pages of a marketplace arrive in one batch, so no page token waits on the pipeline.
+ assert [[(row["marketplaceId"], row["sellerSku"]) for row in item] for item in result.items] == [
+ [("A", "sku1"), ("A", "sku2")],
+ [("B", "sku3")],
+ ]
+ assert result.committed_states == [
+ AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"]),
+ AmazonSellingPartnerResumeConfig(completed_marketplaces=["A", "B"]),
+ ]
+
+ def test_resume_skips_finished_marketplaces(self) -> None:
+ result = _driver("A,B").run(
+ "inventory_summaries",
+ route({TOKEN_PATH: [TOKEN], SUMMARIES_PATH: [_summaries_page(["sku9"])]}),
+ resume_state=AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"]),
+ )
+
+ assert result.raised is None
+ assert [request.param("granularityId") for request in _api_requests(result)] == ["B"]
+
+ def test_a_discovered_marketplace_inventory_does_not_cover_is_skipped(self) -> None:
+ participations = ScriptedResponse(
+ json={
+ "payload": [
+ {"marketplace": {"id": "X"}, "participation": {"isParticipating": True}},
+ {"marketplace": {"id": "A"}, "participation": {"isParticipating": True}},
+ ]
+ }
+ )
+ invalid_marketplace = ScriptedResponse(
+ status=400, json={"errors": [{"code": "InvalidInput", "message": "Invalid granularityId"}]}
+ )
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return TOKEN
+ if request.path == PARTICIPATIONS_PATH:
+ return participations
+ return invalid_marketplace if request.param("granularityId") == "X" else _summaries_page(["sku1"])
+
+ result = _driver().run("inventory_summaries", respond)
+
+ assert result.raised is None
+ assert [(row["marketplaceId"], row["sellerSku"]) for row in result.rows] == [("A", "sku1")]
+ assert result.final_state == AmazonSellingPartnerResumeConfig(completed_marketplaces=["X", "A"])
+
+ def test_a_400_after_the_first_page_of_a_discovered_marketplace_fails_the_sync(self) -> None:
+ participations = ScriptedResponse(
+ json={"payload": [{"marketplace": {"id": "A"}, "participation": {"isParticipating": True}}]}
+ )
+ expired_token = ScriptedResponse(
+ status=400, json={"errors": [{"code": "InvalidInput", "message": "The next token has expired"}]}
+ )
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return TOKEN
+ if request.path == PARTICIPATIONS_PATH:
+ return participations
+ return expired_token if request.param("nextToken") else _summaries_page(["sku1"], "next")
+
+ result = _driver().run("inventory_summaries", respond)
+
+ assert isinstance(result.raised, requests.HTTPError)
+ assert result.final_state is None
+
+ def test_a_configured_marketplace_that_amazon_rejects_fails_the_sync(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ return TOKEN if request.path == TOKEN_PATH else ScriptedResponse(status=400, json={"errors": []})
+
+ result = _driver("X").run("inventory_summaries", respond)
+
+ assert isinstance(result.raised, requests.HTTPError)
+
+ def test_resume_inside_a_marketplace_skips_rows_already_handed_on(self) -> None:
+ result = _driver("A").run(
+ "inventory_summaries",
+ route(
+ {
+ TOKEN_PATH: [TOKEN],
+ SUMMARIES_PATH: [_summaries_page(["sku1", "sku2"], "next"), _summaries_page(["sku3"])],
+ }
+ ),
+ resume_state=AmazonSellingPartnerResumeConfig(marketplace_id="A", last_seller_sku="sku2"),
+ )
+
+ assert result.raised is None
+ assert [row["sellerSku"] for row in result.rows] == ["sku3"]
+ assert result.final_state == AmazonSellingPartnerResumeConfig(completed_marketplaces=["A"])
+
+
+class TestErrors:
+ def test_a_rejected_refresh_token_is_a_permanent_auth_error(self) -> None:
+ result = _driver().run("orders", always(LWA_INVALID_GRANT))
+
+ assert isinstance(result.raised, OAuth2AuthRequestError)
+ assert result.raised.is_permanent
+ assert "invalid_grant" in str(result.raised)
+ assert result.rows == []
+
+ def test_a_forbidden_api_response_keeps_the_vendor_error_code(self) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ return TOKEN if request.path == TOKEN_PATH else SP_API_FORBIDDEN
+
+ result = _driver().run("orders", respond)
+
+ assert isinstance(result.raised, requests.HTTPError)
+ message = str(result.raised)
+ assert (
+ "403 Client Error: Forbidden for url: https://sellingpartnerapi-na.amazon.com/orders/2026-01-01/orders"
+ in message
+ )
+ assert message.endswith("| api error: code=Unauthorized")
+
+
+class TestValidateCredentials:
+ @pytest.mark.parametrize(
+ ("token_answer", "probe_answer", "expected"),
+ [
+ (LWA_INVALID_GRANT, None, (False, LWA_REJECTED_MESSAGE)),
+ (ScriptedResponse(status=503, json={}), None, (False, UNAVAILABLE_MESSAGE)),
+ (TOKEN, ScriptedResponse(json={"payload": []}), (True, None)),
+ (TOKEN, SP_API_FORBIDDEN, (False, ACCESS_DENIED_MESSAGE)),
+ ],
+ )
+ def test_maps_the_vendor_answers(
+ self, token_answer: ScriptedResponse, probe_answer: ScriptedResponse | None, expected: tuple[bool, str | None]
+ ) -> None:
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ if request.path == TOKEN_PATH:
+ return token_answer
+ assert probe_answer is not None
+ return probe_answer
+
+ with scripted_network(respond) as network:
+ assert validate_credentials("na", "client", "secret", "Atzr|refresh") == expected
+ assert network.requests_log[-1].path == (TOKEN_PATH if probe_answer is None else PARTICIPATIONS_PATH)
+
+ def test_a_network_failure_is_reported_as_unavailable(self) -> None:
+ with scripted_network(always(requests.ConnectionError("boom"))):
+ assert validate_credentials("na", "client", "secret", "Atzr|refresh") == (False, UNAVAILABLE_MESSAGE)
+
+
+ ("value", "expected"),
+ [
+ (datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC), "2026-01-02T03:04:05Z"),
+ ("2026-01-02T03:04:05+02:00", "2026-01-02T01:04:05Z"),
+ ("2026-01-02", "2026-01-02T00:00:00Z"),
+ ("not-a-date", "not-a-date"),
+ ],
+)
+def test_to_iso8601(value: Any, expected: str) -> None:
+ assert to_iso8601(value) == expected
+
+
+def test_transaction_windows_are_contiguous_and_end_before_the_request_margin() -> None:
+ start = NOW - timedelta(days=400)
+
+ windows = transaction_windows(start, NOW)
+
+ assert [window.posted_before - window.posted_after for window in windows[:-1]] == [timedelta(days=179)] * 2
+ assert [window.posted_before for window in windows[:-1]] == [window.posted_after for window in windows[1:]]
+ assert windows[0].posted_after == start
+ assert windows[-1].posted_before == NOW - timedelta(seconds=180)
+ assert transaction_windows(NOW - timedelta(seconds=60), NOW) == []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py
index ab29ea90d58c..08d39843cb9b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/sampling.py
@@ -79,6 +79,7 @@ _AUTH_HEADER_NAMES: frozenset[str] = frozenset(
"x-sn-apikey",
"x-ck-api-key",
"x-auth-token",
+ "x-amz-access-token",
"x-metabase-session",
"ob-token-v1",
"kalshi-access-signature",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py
index b336910cd587..fcf1f942e00e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/amazonsellingpartner.py
@@ -1,9 +1,15 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
+from typing import Literal
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
@config.config
class AmazonSellingPartnerSourceConfig(config.Config):
- pass
+ client_id: str
+ client_secret: str
+ refresh_token: str
+ region: Literal["na", "eu", "fe"] = config.value(default="na")
+ marketplace_ids: str | None = None