:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(sendowl): add bundles and licenses tables (#130003)
Tom Owerse1fbf321e3fb
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 955181877299..a735353c9354 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -328,7 +328,6 @@1 products/warehouse_sources/sources/scaleway/scaleway.py1 products/warehouse_sources/sources/scaleway/settings.py1 products/warehouse_sources/sources/semgrep/settings.py-1 products/warehouse_sources/sources/sendowl/settings.py1 products/warehouse_sources/sources/sentry/sentry.py1 products/warehouse_sources/sources/sentry/settings.py1 products/warehouse_sources/sources/servicenow/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex d43530f66ff6..76758658ac74 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -10831,12 +10831,12 @@ Note: Current coverage is essentially only the suppression family plus templates## Sendowl — gaps-Today (4): `discount_codes`, `orders`, `products`, `subscriptions`+Today (6): `bundles`, `discount_codes`, `licenses`, `orders`, `products`, `subscriptions`Diffed against: <https://dashboard.sendowl.com/developers/api/bundles (resource nav enumerated from https://dashboard.sendowl.com/developers/api/orders; per-resource endpoint lists read from /developers/api/{bundles,discounts,drip_items,licenses,products,subscriptions})>-- [ ] `GET /api/v1/packages (Bundles)` — Product bundles — a first-class sellable object alongside products, and the lookup that resolves bundle purchases appearing on orders (high)-- [ ] `GET /api/v1/products/{product_id}/licenses and GET /api/v1/orders/{order_id}/licenses` — Issued license keys per product and per order — the fulfillment record for license-based products (medium)+- [x] `GET /api/v1/packages (Bundles)` — Product bundles — a first-class sellable object alongside products, and the lookup that resolves bundle purchases appearing on orders (high) → `bundles` (full refresh)+- [x] `GET /api/v1/products/{product_id}/licenses and GET /api/v1/orders/{order_id}/licenses` — Issued license keys per product and per order — the fulfillment record for license-based products (medium) → `licenses` (full refresh, fan-out over software `products`; the per-order list is a subset of the per-product list, so it is not synced separately)Note: SendOwl's public API is small: products, packages (bundles), orders, subscriptions, discounts, drip_items, licenses. Four of the seven are already synced. drip_items exposes no index/list endpoint (only nested POST, PUT, DELETE), so it is not syncable. Licenses are only listable nested under a product or order ID, so they need a parent-driven fan-out rather than a flat list resource.diff --git a/products/warehouse_sources/sources/sendowl/canonical_descriptions.py b/products/warehouse_sources/sources/sendowl/canonical_descriptions.pyindex 0fd787777c67..b0ab9120c6e3 100644--- a/products/warehouse_sources/sources/sendowl/canonical_descriptions.py+++ b/products/warehouse_sources/sources/sendowl/canonical_descriptions.py@@ -19,6 +19,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "The date and time the product was last updated.",},},+ "bundles": {+ "description": "A SendOwl bundle: a set of products sold together at one price. The API calls bundles packages.",+ "docs_url": "https://dashboard.sendowl.com/developers/api/bundles",+ "columns": {+ "id": "The unique ID of the bundle.",+ "name": "The display name of the bundle.",+ "price": "The price the bundle is sold at.",+ "currency_code": "The ISO 4217 currency code the bundle is sold in.",+ "components": "The components of the bundle, with the IDs of the products it contains.",+ "price_is_minimum": "Whether the bundle uses pay-what-you-want pricing, with the price as the minimum.",+ "instant_buy_url": "The instant buy URL for the bundle.",+ "add_to_cart_url": "The add-to-cart URL for the bundle.",+ "sales_page_url": "The sales page URL for the bundle.",+ "created_at": "The date and time the bundle was created (UTC).",+ "updated_at": "The date and time the bundle was last updated (UTC).",+ },+ },"orders": {"description": "A SendOwl order — a completed purchase, including buyer, line items, and payment details.","docs_url": "https://dashboard.sendowl.com/developers/api/orders",@@ -65,4 +82,15 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "The date and time the discount code was last updated.",},},+ "licenses": {+ "description": "A SendOwl license: a unique key for a software product, issued to a buyer when an order completes.",+ "docs_url": "https://dashboard.sendowl.com/developers/api/licenses",+ "columns": {+ "id": "The unique ID of the license.",+ "key": "The license key.",+ "product_id": "The ID of the product the license belongs to.",+ "order_id": "The ID of the order the license was issued to, if any.",+ "order_refunded": "Whether the order the license was issued to has been refunded.",+ },+ },}diff --git a/products/warehouse_sources/sources/sendowl/sendowl.py b/products/warehouse_sources/sources/sendowl/sendowl.pyindex 65ada498daf0..e3afcb96f2aa 100644--- a/products/warehouse_sources/sources/sendowl/sendowl.py+++ b/products/warehouse_sources/sources/sendowl/sendowl.py@@ -1,24 +1,24 @@import dataclasses-from typing import Any, Optional+from typing import Any, Optional, castfrom requests import Request, Responsefrom requests.auth import HTTPBasicAuthfrom products.warehouse_sources.sources.sdk import (BasePaginator,+ ClientConfig,+ Resource,RESTAPIConfig,ResumableSourceManager,SourceResponse,+ build_dependent_resource,make_tracked_session,rest_api_resource,validate_via_probe,)-from products.warehouse_sources.sources.sendowl.settings import SENDOWL_ENDPOINTS+from products.warehouse_sources.sources.sendowl.settings import PER_PAGE, SENDOWL_ENDPOINTS, SendowlEndpointConfigSENDOWL_BASE_URL = "https://www.sendowl.com"-# SendOwl caps `per_page` at 50 (default 10); the largest page minimises round trips while-# staying under the documented ~1 request/second advisory.-PER_PAGE = 50# Cheap endpoint used to confirm the credentials are genuine. The key pair is account-wide, so# one probe validates access to every list endpoint.DEFAULT_PROBE_PATH = "/api/v1/products"@@ -29,6 +29,7 @@ class SendowlResumeConfig:# Next page to fetch (1-indexed). Page-number pagination is deterministic, so a crashed# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.next_page: int = 1+ fanout_state: Optional[dict[str, Any]] = Noneclass SendowlPaginator(BasePaginator):@@ -87,6 +88,78 @@ def _unwrap(wrapper_key: str):return _map+def _software_products(row: dict[str, Any]) -> list[dict[str, Any]]:+ # Only software products carry license keys, so skip the licenses request for the others.+ product = row["product"]+ return [product] if product.get("product_type") == "software" else []+++def _client_config(api_key: str, api_secret: str) -> ClientConfig:+ return {+ "base_url": SENDOWL_BASE_URL,+ "headers": {"Accept": "application/json"},+ # HTTP Basic auth with the API key as username and the secret as password. Supplying it+ # via the framework auth config redacts the secret from any raised error/log.+ "auth": {"type": "http_basic", "username": api_key, "password": api_secret},+ "paginator": SendowlPaginator(per_page=PER_PAGE),+ }+++def _fanout_source(+ api_key: str,+ api_secret: str,+ endpoint: str,+ config: SendowlEndpointConfig,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[SendowlResumeConfig],+) -> SourceResponse:+ assert config.fanout is not None++ initial_paginator_state: Optional[dict[str, Any]] = None+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None and resume.fanout_state:+ initial_paginator_state = resume.fanout_state++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ resumable_source_manager.save_state(SendowlResumeConfig(fanout_state=state))++ resource = build_dependent_resource(+ endpoint_configs=SENDOWL_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=config.fanout,+ # License keys are redeemable codes under a generic `key` field the sample scrubber does+ # not redact, so keep these responses out of HTTP sample capture.+ client_config={**_client_config(api_key, api_secret), "capture": False},+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ page_size_param="per_page",+ parent_endpoint_extra={"data_selector_malformed_retryable": True},+ parent_data_map=_software_products,+ child_endpoint_extra={+ "data_selector_malformed_retryable": True,+ # A product deleted after the products listing returns 404 here, so skip it.+ "response_actions": [{"status_code": 404, "action": "ignore"}],+ },+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ # Keeps the checkpoint to two pages of finished products instead of every product.+ resume_parent_pagination=True,+ )+ items = cast(Resource, resource).add_map(_unwrap(config.wrapper_key))++ return SourceResponse(+ name=endpoint,+ items=lambda: items,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ )++def sendowl_source(api_key: str,api_secret: str,@@ -98,15 +171,11 @@ def sendowl_source() -> SourceResponse:config = SENDOWL_ENDPOINTS[endpoint]+ if config.fanout is not None:+ return _fanout_source(api_key, api_secret, endpoint, config, team_id, job_id, resumable_source_manager)+rest_config: RESTAPIConfig = {- "client": {- "base_url": SENDOWL_BASE_URL,- "headers": {"Accept": "application/json"},- # HTTP Basic auth with the API key as username and the secret as password. Supplying it- # via the framework auth config redacts the secret from any raised error/log.- "auth": {"type": "http_basic", "username": api_key, "password": api_secret},- "paginator": SendowlPaginator(per_page=PER_PAGE),- },+ "client": _client_config(api_key, api_secret),"resource_defaults": {},"resources": [{diff --git a/products/warehouse_sources/sources/sendowl/settings.py b/products/warehouse_sources/sources/sendowl/settings.pyindex be0b262c3140..943213fed687 100644--- a/products/warehouse_sources/sources/sendowl/settings.py+++ b/products/warehouse_sources/sources/sendowl/settings.py@@ -1,7 +1,14 @@from dataclasses import dataclass, field+from typing import Any, Optional+from products.warehouse_sources.sources.sdk import DependentEndpointConfig-@dataclass+# SendOwl caps `per_page` at 50 (default 10); the largest page minimises round trips while+# staying under the documented ~1 request/second advisory.+PER_PAGE = 50+++@dataclass(frozen=True)class SendowlEndpointConfig:name: strpath: str@@ -10,12 +17,17 @@ class SendowlEndpointConfig:wrapper_key: str# SendOwl object IDs are unique per account, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ fanout: Optional[DependentEndpointConfig] = None+ page_size: int = PER_PAGE+ incremental_fields: list[Any] = field(default_factory=list)+ default_incremental_field: Optional[str] = None-# SendOwl list endpoints. All are full-refresh only. Resources are versioned per path: products,-# subscriptions and discount codes live under `/api/v1`, while orders use `/api/v1_3`.+# SendOwl list endpoints, all full refresh. Orders use `/api/v1_3` and the rest use `/api/v1`.+# The API paths still call bundles packages.SENDOWL_ENDPOINTS: dict[str, SendowlEndpointConfig] = {"products": SendowlEndpointConfig(name="products", path="/api/v1/products", wrapper_key="product"),+ "bundles": SendowlEndpointConfig(name="bundles", path="/api/v1/packages", wrapper_key="package"),"orders": SendowlEndpointConfig(name="orders", path="/api/v1_3/orders", wrapper_key="order"),"subscriptions": SendowlEndpointConfig(name="subscriptions", path="/api/v1/subscriptions", wrapper_key="subscription"@@ -23,6 +35,19 @@ SENDOWL_ENDPOINTS: dict[str, SendowlEndpointConfig] = {"discount_codes": SendowlEndpointConfig(name="discount_codes", path="/api/v1/discount_codes", wrapper_key="discount_code"),+ # The per-product list also holds keys not yet issued to an order, so the per-order list+ # (a subset) is not synced.+ "licenses": SendowlEndpointConfig(+ name="licenses",+ path="/api/v1/products/{product_id}/licenses",+ wrapper_key="license",+ fanout=DependentEndpointConfig(+ parent_name="products",+ resolve_param="product_id",+ resolve_field="id",+ include_from_parent=[],+ ),+ ),}ENDPOINTS = tuple(SENDOWL_ENDPOINTS.keys())diff --git a/products/warehouse_sources/sources/sendowl/source.py b/products/warehouse_sources/sources/sendowl/source.pyindex b59666dd0215..d20e353c31fc 100644--- a/products/warehouse_sources/sources/sendowl/source.py+++ b/products/warehouse_sources/sources/sendowl/source.py@@ -46,7 +46,7 @@ class SendowlSource(ResumableSource[SendowlSourceConfig, SendowlResumeConfig]):releaseStatus=ReleaseStatus.BETA,caption="""Enter your SendOwl API credentials to pull your SendOwl data into the PostHog Data warehouse.-You can create an API key and secret under **Settings → API credentials** in the [SendOwl dashboard](https://dashboard.sendowl.com/settings/api_keys). Grant the key the **Manager** permission for read access to products, orders, subscriptions, and discount codes.+You can create an API key and secret under **Settings → API credentials** in the [SendOwl dashboard](https://dashboard.sendowl.com/settings/api_keys). Grant the key the **Manager** permission for read access to products, bundles, orders, subscriptions, discount codes, and licenses.""",iconPath="/static/services/sendowl.png",docsUrl="https://posthog.com/docs/cdp/sources/sendowl",diff --git a/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py b/products/warehouse_sources/sources/sendowl/tests/test_sendowl.pyindex 392c4a3a0cbd..6d9f17ae4406 100644--- a/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py+++ b/products/warehouse_sources/sources/sendowl/tests/test_sendowl.py@@ -1,12 +1,20 @@+from collections.abc import Callablefrom typing import Any+from urllib.parse import parse_qsl, urlsplitimport requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.sdk import RESTClientRetryableError-from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always+from products.warehouse_sources.sources.sdk.testing import (+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+ UnexpectedRequest,+ always,+)from products.warehouse_sources.sources.sendowl._config import SendowlSourceConfig-from products.warehouse_sources.sources.sendowl.sendowl import PER_PAGE, SendowlResumeConfig+from products.warehouse_sources.sources.sendowl.sendowl import PER_PAGE, SENDOWL_BASE_URL, SendowlResumeConfigfrom products.warehouse_sources.sources.sendowl.settings import ENDPOINTS, SENDOWL_ENDPOINTSfrom products.warehouse_sources.sources.sendowl.source import SendowlSource@@ -28,6 +36,95 @@ def _body_response(status: int, body: Any) -> ScriptedResponse:return ScriptedResponse(status=status, json=body)+def _by_url(responses_by_url: dict[str, list[ScriptedResponse]]) -> Callable[[RecordedRequest], ScriptedResponse]:+ def _norm(url: str) -> tuple[str, tuple[tuple[str, str], ...]]:+ parts = urlsplit(url)+ return parts.path, tuple(sorted(parse_qsl(parts.query)))++ queues = {_norm(url): list(queue) for url, queue in responses_by_url.items()}++ def respond(request: RecordedRequest) -> ScriptedResponse:+ queue = queues.get(_norm(request.url))+ if not queue:+ raise UnexpectedRequest(f"unexpected request url {request.url!r}")+ return queue.pop(0)++ return respond+++def _url(path: str, page: int = 1) -> str:+ return f"{SENDOWL_BASE_URL}{path}?per_page={PER_PAGE}&page={page}"+++def _license(license_id: int, product_id: int) -> dict[str, Any]:+ return {"id": license_id, "product_id": product_id, "order_id": None, "order_refunded": False, "key": "TEST-KEY"}+++class TestLicensesFanout:+ def test_walks_every_page_of_each_software_product_and_skips_other_products(self) -> None:+ full_page = [_license(i, 1) for i in range(PER_PAGE)]+ responses = {+ _url("/api/v1/products"): [+ _page(+ "product",+ [+ {"id": 1, "product_type": "software"},+ {"id": 2, "product_type": "digital"},+ {"id": 3, "product_type": "software"},+ ],+ )+ ],+ _url("/api/v1/products/1/licenses"): [_page("license", full_page)],+ _url("/api/v1/products/1/licenses", page=2): [_page("license", [_license(100, 1)])],+ _url("/api/v1/products/3/licenses"): [_page("license", [_license(200, 3)])],+ }++ result = _driver().run("licenses", _by_url(responses))++ assert result.raised is None+ assert result.rows == [*full_page, _license(100, 1), _license(200, 3)]+ assert all("/products/2/" not in request.url for request in result.requests)++ def test_product_deleted_before_its_licenses_are_fetched_is_skipped(self) -> None:+ responses = {+ _url("/api/v1/products"): [+ _page("product", [{"id": 1, "product_type": "software"}, {"id": 2, "product_type": "software"}])+ ],+ _url("/api/v1/products/1/licenses"): [_body_response(404, {"error": "not found"})],+ _url("/api/v1/products/2/licenses"): [_page("license", [_license(7, 2)])],+ }++ result = _driver().run("licenses", _by_url(responses))++ assert result.raised is None+ assert result.rows == [_license(7, 2)]++ @parameterized.expand([("parent_listing_from_start", None, 1), ("parent_listing_from_saved_page", 2, 2)])+ def test_resumes_mid_product_from_the_saved_page(+ self, _name: str, saved_parent_page: int | None, expected_parent_page: int+ ) -> None:+ responses = {+ _url("/api/v1/products", page=expected_parent_page): [+ _page("product", [{"id": 1, "product_type": "software"}, {"id": 2, "product_type": "software"}])+ ],+ _url("/api/v1/products/2/licenses", page=3): [_page("license", [_license(9, 2)])],+ }+ fanout_state: dict[str, Any] = {+ "completed": ["/api/v1/products/1/licenses"],+ "current": "/api/v1/products/2/licenses",+ "child_state": {"page": 3},+ }+ if saved_parent_page is not None:+ fanout_state["parent"] = {"page": saved_parent_page}++ result = _driver().run(+ "licenses", _by_url(responses), resume_state=SendowlResumeConfig(fanout_state=fanout_state)+ )++ assert result.raised is None+ assert result.rows == [_license(9, 2)]++class TestPagination:def test_resumes_from_saved_page(self) -> None:full_page = [{"id": i} for i in range(PER_PAGE)]@@ -69,7 +166,9 @@ class TestErrorHandling:class TestSendowlSourceResponse:- @parameterized.expand([("products",), ("orders",), ("subscriptions",), ("discount_codes",)])+ @parameterized.expand(+ [("products",), ("bundles",), ("orders",), ("subscriptions",), ("discount_codes",), ("licenses",)]+ )def test_response_uses_id_primary_key(self, endpoint: str) -> None:result = _driver().run(endpoint, [ScriptedResponse(json=[])])