Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(sendowl): add bundles and licenses tables (#130003)

Tom Owerse1fbf321e3fb
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 955181877299..a735353c9354 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -328,7 +328,6 @@
1 products/warehouse_sources/sources/scaleway/scaleway.py
1 products/warehouse_sources/sources/scaleway/settings.py
1 products/warehouse_sources/sources/semgrep/settings.py
-1 products/warehouse_sources/sources/sendowl/settings.py
1 products/warehouse_sources/sources/sentry/sentry.py
1 products/warehouse_sources/sources/sentry/settings.py
1 products/warehouse_sources/sources/servicenow/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index d43530f66ff6..76758658ac74 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -10831,12 +10831,12 @@ Note: Current coverage is essentially only the suppression family plus templates
## Sendowl — gaps
-Today (4): `discount_codes`, `orders`, `products`, `subscriptions`
+Today (6): `bundles`, `discount_codes`, `licenses`, `orders`, `products`, `subscriptions`
Diffed against: <https://dashboard.sendowl.com/developers/api/bundles (resource nav enumerated from https://dashboard.sendowl.com/developers/api/orders; per-resource endpoint lists read from /developers/api/{bundles,discounts,drip_items,licenses,products,subscriptions})>
-- [ ] `GET /api/v1/packages (Bundles)` — Product bundles — a first-class sellable object alongside products, and the lookup that resolves bundle purchases appearing on orders (high)
-- [ ] `GET /api/v1/products/{product_id}/licenses and GET /api/v1/orders/{order_id}/licenses` — Issued license keys per product and per order — the fulfillment record for license-based products (medium)
+- [x] `GET /api/v1/packages (Bundles)` — Product bundles — a first-class sellable object alongside products, and the lookup that resolves bundle purchases appearing on orders (high) → `bundles` (full refresh)
+- [x] `GET /api/v1/products/{product_id}/licenses and GET /api/v1/orders/{order_id}/licenses` — Issued license keys per product and per order — the fulfillment record for license-based products (medium) → `licenses` (full refresh, fan-out over software `products`; the per-order list is a subset of the per-product list, so it is not synced separately)
Note: SendOwl's public API is small: products, packages (bundles), orders, subscriptions, discounts, drip_items, licenses. Four of the seven are already synced. drip_items exposes no index/list endpoint (only nested POST, PUT, DELETE), so it is not syncable. Licenses are only listable nested under a product or order ID, so they need a parent-driven fan-out rather than a flat list resource.
diff --git a/products/warehouse_sources/sources/sendowl/canonical_descriptions.py b/products/warehouse_sources/sources/sendowl/canonical_descriptions.py
index 0fd787777c67..b0ab9120c6e3 100644
--- a/products/warehouse_sources/sources/sendowl/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/sendowl/canonical_descriptions.py
@@ -19,6 +19,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "The date and time the product was last updated.",
},
},
+ "bundles": {
+ "description": "A SendOwl bundle: a set of products sold together at one price. The API calls bundles packages.",
+ "docs_url": "https://dashboard.sendowl.com/developers/api/bundles",
+ "columns": {
+ "id": "The unique ID of the bundle.",
+ "name": "The display name of the bundle.",
+ "price": "The price the bundle is sold at.",
+ "currency_code": "The ISO 4217 currency code the bundle is sold in.",
+ "components": "The components of the bundle, with the IDs of the products it contains.",
+ "price_is_minimum": "Whether the bundle uses pay-what-you-want pricing, with the price as the minimum.",
+ "instant_buy_url": "The instant buy URL for the bundle.",
+ "add_to_cart_url": "The add-to-cart URL for the bundle.",
+ "sales_page_url": "The sales page URL for the bundle.",
+ "created_at": "The date and time the bundle was created (UTC).",
+ "updated_at": "The date and time the bundle was last updated (UTC).",
+ },
+ },
"orders": {
"description": "A SendOwl order — a completed purchase, including buyer, line items, and payment details.",
"docs_url": "https://dashboard.sendowl.com/developers/api/orders",
@@ -65,4 +82,15 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "The date and time the discount code was last updated.",
},
},
+ "licenses": {
+ "description": "A SendOwl license: a unique key for a software product, issued to a buyer when an order completes.",
+ "docs_url": "https://dashboard.sendowl.com/developers/api/licenses",
+ "columns": {
+ "id": "The unique ID of the license.",
+ "key": "The license key.",
+ "product_id": "The ID of the product the license belongs to.",
+ "order_id": "The ID of the order the license was issued to, if any.",
+ "order_refunded": "Whether the order the license was issued to has been refunded.",
+ },
+ },
}
diff --git a/products/warehouse_sources/sources/sendowl/sendowl.py b/products/warehouse_sources/sources/sendowl/sendowl.py
index 65ada498daf0..e3afcb96f2aa 100644
--- a/products/warehouse_sources/sources/sendowl/sendowl.py
+++ b/products/warehouse_sources/sources/sendowl/sendowl.py
@@ -1,24 +1,24 @@
import dataclasses
-from typing import Any, Optional
+from typing import Any, Optional, cast
from requests import Request, Response
from requests.auth import HTTPBasicAuth
from products.warehouse_sources.sources.sdk import (
BasePaginator,
+ ClientConfig,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
+ build_dependent_resource,
make_tracked_session,
rest_api_resource,
validate_via_probe,
)
-from products.warehouse_sources.sources.sendowl.settings import SENDOWL_ENDPOINTS
+from products.warehouse_sources.sources.sendowl.settings import PER_PAGE, SENDOWL_ENDPOINTS, SendowlEndpointConfig
SENDOWL_BASE_URL = "https://www.sendowl.com"
-# SendOwl caps `per_page` at 50 (default 10); the largest page minimises round trips while
-# staying under the documented ~1 request/second advisory.
-PER_PAGE = 50
# Cheap endpoint used to confirm the credentials are genuine. The key pair is account-wide, so
# one probe validates access to every list endpoint.
DEFAULT_PROBE_PATH = "/api/v1/products"
@@ -29,6 +29,7 @@ class SendowlResumeConfig:
# Next page to fetch (1-indexed). Page-number pagination is deterministic, so a crashed
# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.
next_page: int = 1
+ fanout_state: Optional[dict[str, Any]] = None
class SendowlPaginator(BasePaginator):
@@ -87,6 +88,78 @@ def _unwrap(wrapper_key: str):
return _map
+def _software_products(row: dict[str, Any]) -> list[dict[str, Any]]:
+ # Only software products carry license keys, so skip the licenses request for the others.
+ product = row["product"]
+ return [product] if product.get("product_type") == "software" else []
+
+
+def _client_config(api_key: str, api_secret: str) -> ClientConfig:
+ return {
+ "base_url": SENDOWL_BASE_URL,
+ "headers": {"Accept": "application/json"},
+ # HTTP Basic auth with the API key as username and the secret as password. Supplying it
+ # via the framework auth config redacts the secret from any raised error/log.
+ "auth": {"type": "http_basic", "username": api_key, "password": api_secret},
+ "paginator": SendowlPaginator(per_page=PER_PAGE),
+ }
+
+
+def _fanout_source(
+ api_key: str,
+ api_secret: str,
+ endpoint: str,
+ config: SendowlEndpointConfig,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[SendowlResumeConfig],
+) -> SourceResponse:
+ assert config.fanout is not None
+
+ initial_paginator_state: Optional[dict[str, Any]] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None and resume.fanout_state:
+ initial_paginator_state = resume.fanout_state
+
+ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ resumable_source_manager.save_state(SendowlResumeConfig(fanout_state=state))
+
+ resource = build_dependent_resource(
+ endpoint_configs=SENDOWL_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ # License keys are redeemable codes under a generic `key` field the sample scrubber does
+ # not redact, so keep these responses out of HTTP sample capture.
+ client_config={**_client_config(api_key, api_secret), "capture": False},
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=None,
+ page_size_param="per_page",
+ parent_endpoint_extra={"data_selector_malformed_retryable": True},
+ parent_data_map=_software_products,
+ child_endpoint_extra={
+ "data_selector_malformed_retryable": True,
+ # A product deleted after the products listing returns 404 here, so skip it.
+ "response_actions": [{"status_code": 404, "action": "ignore"}],
+ },
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ # Keeps the checkpoint to two pages of finished products instead of every product.
+ resume_parent_pagination=True,
+ )
+ items = cast(Resource, resource).add_map(_unwrap(config.wrapper_key))
+
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: items,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ )
+
+
def sendowl_source(
api_key: str,
api_secret: str,
@@ -98,15 +171,11 @@ def sendowl_source(
) -> SourceResponse:
config = SENDOWL_ENDPOINTS[endpoint]
+ if config.fanout is not None:
+ return _fanout_source(api_key, api_secret, endpoint, config, team_id, job_id, resumable_source_manager)
+
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": SENDOWL_BASE_URL,
- "headers": {"Accept": "application/json"},
- # HTTP Basic auth with the API key as username and the secret as password. Supplying it
- # via the framework auth config redacts the secret from any raised error/log.
- "auth": {"type": "http_basic", "username": api_key, "password": api_secret},
- "paginator": SendowlPaginator(per_page=PER_PAGE),
- },
+ "client": _client_config(api_key, api_secret),
"resource_defaults": {},
"resources": [
{
diff --git a/products/warehouse_sources/sources/sendowl/settings.py b/products/warehouse_sources/sources/sendowl/settings.py
index be0b262c3140..943213fed687 100644
--- a/products/warehouse_sources/sources/sendowl/settings.py
+++ b/products/warehouse_sources/sources/sendowl/settings.py
@@ -1,7 +1,14 @@
from dataclasses import dataclass, field
+from typing import Any, Optional
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig
-@dataclass
+# SendOwl caps `per_page` at 50 (default 10); the largest page minimises round trips while
+# staying under the documented ~1 request/second advisory.
+PER_PAGE = 50
+
+
+@dataclass(frozen=True)
class SendowlEndpointConfig:
name: str
path: str
@@ -10,12 +17,17 @@ class SendowlEndpointConfig:
wrapper_key: str
# SendOwl object IDs are unique per account, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ fanout: Optional[DependentEndpointConfig] = None
+ page_size: int = PER_PAGE
+ incremental_fields: list[Any] = field(default_factory=list)
+ default_incremental_field: Optional[str] = None
-# SendOwl list endpoints. All are full-refresh only. Resources are versioned per path: products,
-# subscriptions and discount codes live under `/api/v1`, while orders use `/api/v1_3`.
+# SendOwl list endpoints, all full refresh. Orders use `/api/v1_3` and the rest use `/api/v1`.
+# The API paths still call bundles packages.
SENDOWL_ENDPOINTS: dict[str, SendowlEndpointConfig] = {
"products": SendowlEndpointConfig(name="products", path="/api/v1/products", wrapper_key="product"),
+ "bundles": SendowlEndpointConfig(name="bundles", path="/api/v1/packages", wrapper_key="package"),
"orders": SendowlEndpointConfig(name="orders", path="/api/v1_3/orders", wrapper_key="order"),
"subscriptions": SendowlEndpointConfig(
name="subscriptions", path="/api/v1/subscriptions", wrapper_key="subscription"
@@ -23,6 +35,19 @@ SENDOWL_ENDPOINTS: dict[str, SendowlEndpointConfig] = {
"discount_codes": SendowlEndpointConfig(
name="discount_codes", path="/api/v1/discount_codes", wrapper_key="discount_code"
),
+ # The per-product list also holds keys not yet issued to an order, so the per-order list
+ # (a subset) is not synced.
+ "licenses": SendowlEndpointConfig(
+ name="licenses",
+ path="/api/v1/products/{product_id}/licenses",
+ wrapper_key="license",
+ fanout=DependentEndpointConfig(
+ parent_name="products",
+ resolve_param="product_id",
+ resolve_field="id",
+ include_from_parent=[],
+ ),
+ ),
}
ENDPOINTS = tuple(SENDOWL_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/sources/sendowl/source.py b/products/warehouse_sources/sources/sendowl/source.py
index b59666dd0215..d20e353c31fc 100644
--- a/products/warehouse_sources/sources/sendowl/source.py
+++ b/products/warehouse_sources/sources/sendowl/source.py
@@ -46,7 +46,7 @@ class SendowlSource(ResumableSource[SendowlSourceConfig, SendowlResumeConfig]):
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your SendOwl API credentials to pull your SendOwl data into the PostHog Data warehouse.
-You can create an API key and secret under **Settings → API credentials** in the [SendOwl dashboard](https://dashboard.sendowl.com/settings/api_keys). Grant the key the **Manager** permission for read access to products, orders, subscriptions, and discount codes.
+You can create an API key and secret under **Settings → API credentials** in the [SendOwl dashboard](https://dashboard.sendowl.com/settings/api_keys). Grant the key the **Manager** permission for read access to products, bundles, orders, subscriptions, discount codes, and licenses.
""",
iconPath="/static/services/sendowl.png",
docsUrl="https://posthog.com/docs/cdp/sources/sendowl",
diff --git a/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py b/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py
index 392c4a3a0cbd..6d9f17ae4406 100644
--- a/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py
+++ b/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py
@@ -1,12 +1,20 @@
+from collections.abc import Callable
from typing import Any
+from urllib.parse import parse_qsl, urlsplit
import requests
from parameterized import parameterized
from products.warehouse_sources.sources.sdk import RESTClientRetryableError
-from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always
+from products.warehouse_sources.sources.sdk.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ UnexpectedRequest,
+ always,
+)
from products.warehouse_sources.sources.sendowl._config import SendowlSourceConfig
-from products.warehouse_sources.sources.sendowl.sendowl import PER_PAGE, SendowlResumeConfig
+from products.warehouse_sources.sources.sendowl.sendowl import PER_PAGE, SENDOWL_BASE_URL, SendowlResumeConfig
from products.warehouse_sources.sources.sendowl.settings import ENDPOINTS, SENDOWL_ENDPOINTS
from products.warehouse_sources.sources.sendowl.source import SendowlSource
@@ -28,6 +36,95 @@ def _body_response(status: int, body: Any) -> ScriptedResponse:
return ScriptedResponse(status=status, json=body)
+def _by_url(responses_by_url: dict[str, list[ScriptedResponse]]) -> Callable[[RecordedRequest], ScriptedResponse]:
+ def _norm(url: str) -> tuple[str, tuple[tuple[str, str], ...]]:
+ parts = urlsplit(url)
+ return parts.path, tuple(sorted(parse_qsl(parts.query)))
+
+ queues = {_norm(url): list(queue) for url, queue in responses_by_url.items()}
+
+ def respond(request: RecordedRequest) -> ScriptedResponse:
+ queue = queues.get(_norm(request.url))
+ if not queue:
+ raise UnexpectedRequest(f"unexpected request url {request.url!r}")
+ return queue.pop(0)
+
+ return respond
+
+
+def _url(path: str, page: int = 1) -> str:
+ return f"{SENDOWL_BASE_URL}{path}?per_page={PER_PAGE}&page={page}"
+
+
+def _license(license_id: int, product_id: int) -> dict[str, Any]:
+ return {"id": license_id, "product_id": product_id, "order_id": None, "order_refunded": False, "key": "TEST-KEY"}
+
+
+class TestLicensesFanout:
+ def test_walks_every_page_of_each_software_product_and_skips_other_products(self) -> None:
+ full_page = [_license(i, 1) for i in range(PER_PAGE)]
+ responses = {
+ _url("/api/v1/products"): [
+ _page(
+ "product",
+ [
+ {"id": 1, "product_type": "software"},
+ {"id": 2, "product_type": "digital"},
+ {"id": 3, "product_type": "software"},
+ ],
+ )
+ ],
+ _url("/api/v1/products/1/licenses"): [_page("license", full_page)],
+ _url("/api/v1/products/1/licenses", page=2): [_page("license", [_license(100, 1)])],
+ _url("/api/v1/products/3/licenses"): [_page("license", [_license(200, 3)])],
+ }
+
+ result = _driver().run("licenses", _by_url(responses))
+
+ assert result.raised is None
+ assert result.rows == [*full_page, _license(100, 1), _license(200, 3)]
+ assert all("/products/2/" not in request.url for request in result.requests)
+
+ def test_product_deleted_before_its_licenses_are_fetched_is_skipped(self) -> None:
+ responses = {
+ _url("/api/v1/products"): [
+ _page("product", [{"id": 1, "product_type": "software"}, {"id": 2, "product_type": "software"}])
+ ],
+ _url("/api/v1/products/1/licenses"): [_body_response(404, {"error": "not found"})],
+ _url("/api/v1/products/2/licenses"): [_page("license", [_license(7, 2)])],
+ }
+
+ result = _driver().run("licenses", _by_url(responses))
+
+ assert result.raised is None
+ assert result.rows == [_license(7, 2)]
+
+ @parameterized.expand([("parent_listing_from_start", None, 1), ("parent_listing_from_saved_page", 2, 2)])
+ def test_resumes_mid_product_from_the_saved_page(
+ self, _name: str, saved_parent_page: int | None, expected_parent_page: int
+ ) -> None:
+ responses = {
+ _url("/api/v1/products", page=expected_parent_page): [
+ _page("product", [{"id": 1, "product_type": "software"}, {"id": 2, "product_type": "software"}])
+ ],
+ _url("/api/v1/products/2/licenses", page=3): [_page("license", [_license(9, 2)])],
+ }
+ fanout_state: dict[str, Any] = {
+ "completed": ["/api/v1/products/1/licenses"],
+ "current": "/api/v1/products/2/licenses",
+ "child_state": {"page": 3},
+ }
+ if saved_parent_page is not None:
+ fanout_state["parent"] = {"page": saved_parent_page}
+
+ result = _driver().run(
+ "licenses", _by_url(responses), resume_state=SendowlResumeConfig(fanout_state=fanout_state)
+ )
+
+ assert result.raised is None
+ assert result.rows == [_license(9, 2)]
+
+
class TestPagination:
def test_resumes_from_saved_page(self) -> None:
full_page = [{"id": i} for i in range(PER_PAGE)]
@@ -69,7 +166,9 @@ class TestErrorHandling:
class TestSendowlSourceResponse:
- @parameterized.expand([("products",), ("orders",), ("subscriptions",), ("discount_codes",)])
+ @parameterized.expand(
+ [("products",), ("bundles",), ("orders",), ("subscriptions",), ("discount_codes",), ("licenses",)]
+ )
def test_response_uses_id_primary_key(self, endpoint: str) -> None:
result = _driver().run(endpoint, [ScriptedResponse(json=[])])