:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add outbrain breakdown report and conversion event tables (#123994)
Tom Owersbc543d104c3f
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex f0952a12d897..a1ef91eb5708 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -833,8 +833,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/ortto/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/oura/oura.py1 products/warehouse_sources/backend/temporal/data_imports/sources/oura/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex f1733e76f020..ff0cdee83484 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9003,10 +9003,10 @@ Today (6): `budgets`, `campaign_performance`, `campaigns`, `marketer_performanceDiffed against: <https://jsapi.apiary.io/apis/amplifyv01/api-description-document>-- [ ] `/reports/marketers/{id}/promotedContent` — per-ad (promoted link) performance - we sync promoted_links metadata but no spend/clicks/conversions against them (high)-- [ ] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)-- [ ] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)-- [ ] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)+- [x] `/reports/marketers/{id}/promotedContent` — per-ad (promoted link) performance - we sync promoted_links metadata but no spend/clicks/conversions against them (high)+- [x] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)+- [x] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)+- [x] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)- [ ] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)- [ ] `/reports/marketers/{id}/geo` — geo breakdown with country/region granularity (medium)- [ ] `/reports/marketers/{id}/platforms` — device/platform breakdown (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.pyindex 9b4b5779a71d..0d5953f378ae 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py@@ -93,6 +93,25 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"cachedImageUrl": "URL of the cached version of the promoted link's image.",},},+ "conversion_events": {+ "description": "A conversion event (pixel or URL rule) defined by a marketer. Resolves the conversion ids in report conversion details.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/multiple-conversions",+ "columns": {+ "id": "Unique identifier for the conversion event.",+ "_marketer_id": "ID of the marketer that owns the conversion event (injected during sync).",+ "marketerId": "ID of the marketer that owns the conversion event.",+ "name": "The conversion event's name.",+ "type": "How the conversion is detected (e.g. URL_BASED, EVENT_BASED).",+ "category": "Conversion category (e.g. PURCHASE).",+ "enabled": "Whether the conversion event is enabled.",+ "trackingStatus": "Whether the conversion event is currently receiving data (ACTIVE or INACTIVE).",+ "conversionWindow": "Number of days after a click in which a conversion is attributed.",+ "includeInConversions": "Whether the event counts toward the reported conversions metric.",+ "url": "URL rule that triggers a URL_BASED conversion.",+ "creationTime": "Time at which the conversion event was created.",+ "lastModified": "Time at which the conversion event was last modified.",+ },+ },"marketer_performance_daily": {"description": "Daily performance metrics (impressions, clicks, spend, conversions) per marketer.","docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",@@ -112,4 +131,31 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {**_REPORT_METRICS,},},+ "promoted_link_performance": {+ "description": "Per-promoted-link performance totals for a trailing window (a snapshot, full-refreshed each sync).",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/promoted-content",+ "columns": {+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",+ "metadata": "The promoted link the metrics belong to, including its id, campaignId, title, url and status.",+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the promoted link.",+ },+ },+ "publisher_performance": {+ "description": "Per-publisher performance totals for a trailing window (a snapshot, full-refreshed each sync).",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/publishers",+ "columns": {+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",+ "metadata": "The publisher the metrics belong to, including its id, name, url and block status.",+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the publisher.",+ },+ },+ "section_performance": {+ "description": "Per-section performance totals for a trailing window (a snapshot, full-refreshed each sync). Sections with fewer than 1 click or 10 impressions are omitted.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/sections",+ "columns": {+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",+ "metadata": "The section the metrics belong to, including its id, name, publisherId, publisherName and block status.",+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the section.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.pyindex 28a43322f1af..c727e12b0257 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py@@ -25,6 +25,7 @@ REQUEST_TIMEOUT_SECONDS = 60MAX_RETRY_ATTEMPTS = 5PAGE_SIZE = 100# Periodic reports return one row per day; 500 covers the full default backfill.+# Breakdown reports page through with offset at the same size.REPORT_ROW_LIMIT = 500# Tokens are valid 30 days but /login is capped at 2 requests/hour, so the# token is cached across runs and only re-minted when it expires.@@ -35,10 +36,15 @@ class OutbrainRetryableError(Exception):pass[email protected](frozen=True)class OutbrainResumeConfig:# Index of the next fan-out parent (marketer or campaign) to process.next_index: int+ # Report offset to resume at within that marketer (paginated breakdown reports only).+ next_offset: int = 0+ # Report window the offset belongs to, so a resume after UTC midnight keeps paging the same window.+ window_from: str | None = None+ window_to: str | None = Nonedef _get_session(password: str, token: Optional[str] = None) -> requests.Session:@@ -208,38 +214,62 @@ def get_rows(# Conversions restate for ~30 days — re-pull the lookback window.window_start = watermark - timedelta(days=REPORT_LOOKBACK_DAYS)+ start_offset = 0+ if resume_config is not None and config.paginated and resume_config.next_offset:+ start_offset = resume_config.next_offset+ saved_from, saved_to = _to_date(resume_config.window_from), _to_date(resume_config.window_to)+ if saved_from is not None and saved_to is not None:+ window_start, today = saved_from, saved_tomarketer_ids = list_marketer_ids()for index in range(start_index, len(marketer_ids)):marketer_id = marketer_ids[index]- params = urlencode(- {- "from": window_start.isoformat(),- "to": today.isoformat(),- "breakdown": "daily",- "limit": REPORT_ROW_LIMIT,- }- if config.kind == "periodic_report"- else {"from": window_start.isoformat(), "to": today.isoformat(), "limit": REPORT_ROW_LIMIT}- )path = config.path.format(marketer_id=quote(marketer_id))- rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{params}"), config.data_key)-- out: list[dict[str, Any]] = []- for row in rows:- enriched = {**row, "_marketer_id": marketer_id}+ offset = start_offset if index == start_index else 0+ while True:+ params: dict[str, Any] = {"from": window_start.isoformat(), "to": today.isoformat()}if config.kind == "periodic_report":- metadata_value = row.get("metadata")- metadata = metadata_value if isinstance(metadata_value, dict) else {}- row_date = metadata.get("fromDate") or metadata.get("id")- if not row_date:- continue- enriched["_date"] = row_date- out.append(enriched)- if out:- yield out-- # Save state AFTER yielding so a crash re-yields the in-flight marketer.- resumable_source_manager.save_state(OutbrainResumeConfig(next_index=index + 1))+ params["breakdown"] = "daily"+ params["limit"] = REPORT_ROW_LIMIT+ if config.paginated:+ params["offset"] = offset+ if config.include_inactive_campaigns:+ params["includeArchivedCampaigns"] = "true"+ params["enabledCampaignsOnly"] = "false"+ rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}"), config.data_key)++ out: list[dict[str, Any]] = []+ for row in rows:+ enriched = {**row, "_marketer_id": marketer_id}+ if config.kind == "periodic_report":+ metadata_value = row.get("metadata")+ metadata = metadata_value if isinstance(metadata_value, dict) else {}+ row_date = metadata.get("fromDate") or metadata.get("id")+ if not row_date:+ continue+ enriched["_date"] = row_date+ out.append(enriched)++ has_more = config.paginated and len(rows) >= REPORT_ROW_LIMIT+ next_state = (+ OutbrainResumeConfig(+ next_index=index,+ next_offset=offset + len(rows),+ window_from=window_start.isoformat(),+ window_to=today.isoformat(),+ )+ if has_more+ else OutbrainResumeConfig(next_index=index + 1)+ )+ # Stage before the yield so a resumed full refresh never re-appends a written page.+ resumable_source_manager.save_state(next_state)+ if out:+ yield out+ else:+ resumable_source_manager.safe_point()++ if not has_more:+ break+ offset += len(rows)def _to_date(value: Any) -> Optional[date]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.pyindex 519094fcdb1d..927442e086c6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py@@ -23,7 +23,7 @@ _DATE_INCREMENTAL_FIELDS: list[IncrementalField] = []-@dataclass+@dataclass(frozen=True)class OutbrainEndpointConfig:name: strkind: OutbrainEndpointKind@@ -34,8 +34,11 @@ class OutbrainEndpointConfig:data_key: strprimary_keys: list[str] | None = field(default_factory=lambda: ["id"])incremental_fields: list[IncrementalField] = field(default_factory=list)- # Entity endpoints paginate with limit/offset.+ # Entity and breakdown report endpoints paginate with limit/offset.paginated: bool = False+ # Reports default to enabled, unarchived campaigns only, which would drop+ # spend from campaigns paused or archived inside the window.+ include_inactive_campaigns: bool = FalseOUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {@@ -67,6 +70,12 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {data_key="promotedLinks",paginated=True,),+ "conversion_events": OutbrainEndpointConfig(+ name="conversion_events",+ kind="per_marketer",+ path="/marketers/{marketer_id}/conversionEvents",+ data_key="conversionEvents",+ ),# Daily periodic performance per marketer; rows carry the day in# metadata, injected as `_date` for the incremental cursor."marketer_performance_daily": OutbrainEndpointConfig(@@ -86,6 +95,35 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {data_key="results",primary_keys=None,),+ # Breakdown reports have no date dimension either, so they are trailing+ # window snapshots too. One row per publisher/section/promoted link.+ "promoted_link_performance": OutbrainEndpointConfig(+ name="promoted_link_performance",+ kind="snapshot_report",+ path="/reports/marketers/{marketer_id}/promotedContent",+ data_key="results",+ primary_keys=None,+ paginated=True,+ include_inactive_campaigns=True,+ ),+ "publisher_performance": OutbrainEndpointConfig(+ name="publisher_performance",+ kind="snapshot_report",+ path="/reports/marketers/{marketer_id}/publishers",+ data_key="results",+ primary_keys=None,+ paginated=True,+ include_inactive_campaigns=True,+ ),+ "section_performance": OutbrainEndpointConfig(+ name="section_performance",+ kind="snapshot_report",+ path="/reports/marketers/{marketer_id}/sections",+ data_key="results",+ primary_keys=None,+ paginated=True,+ include_inactive_campaigns=True,+ ),}ENDPOINTS = tuple(OUTBRAIN_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.pyindex 389120170e83..7524a743e5f1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_)from products.warehouse_sources.backend.temporal.data_imports.sources.outbrain.outbrain import (PAGE_SIZE,+ REPORT_ROW_LIMIT,OutbrainResumeConfig,validate_credentials,)@@ -178,6 +179,62 @@ class TestReportStreams:assert "_date" not in result.rows[0]assert result.requests[1].param("breakdown") is None+ @pytest.mark.parametrize(+ "endpoint, report_path",+ [+ ("promoted_link_performance", "promotedContent"),+ ("publisher_performance", "publishers"),+ ("section_performance", "sections"),+ ],+ )+ def test_breakdown_report_pages_with_offset(+ self, mock_cache: mock.MagicMock, endpoint: str, report_path: str+ ) -> None:+ mock_cache.get.return_value = "tok"+ full_page = [{"metadata": {"id": f"r{i}"}, "metrics": {"clicks": i}} for i in range(REPORT_ROW_LIMIT)]+ result = _driver().run(+ endpoint,+ [+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),+ ScriptedResponse(json={"results": full_page, "totalResults": REPORT_ROW_LIMIT + 1}),+ ScriptedResponse(json={"results": [{"metadata": {"id": "r-last"}, "metrics": {}}]}),+ ],+ )++ assert result.raised is None+ assert [len(batch) for batch in result.items] == [REPORT_ROW_LIMIT, 1]+ assert all(row["_marketer_id"] == "m1" for row in result.rows)+ assert result.paths[1:] == [f"/amplify/v0.1/reports/marketers/m1/{report_path}"] * 2+ assert [request.param("offset") for request in result.requests[1:]] == ["0", str(REPORT_ROW_LIMIT)]+ assert all(request.param("includeArchivedCampaigns") == "true" for request in result.requests[1:])+ assert all(request.param("enabledCampaignsOnly") == "false" for request in result.requests[1:])+ window = {"window_from": result.requests[1].param("from"), "window_to": result.requests[1].param("to")}+ assert result.saved_states == [+ OutbrainResumeConfig(next_index=0, next_offset=REPORT_ROW_LIMIT, **window),+ OutbrainResumeConfig(next_index=1),+ ]++ def test_breakdown_report_resumes_at_saved_offset(self, mock_cache: mock.MagicMock) -> None:+ mock_cache.get.return_value = "tok"+ result = _driver().run(+ "publisher_performance",+ [+ ScriptedResponse(json={"marketers": [{"id": "m1"}, {"id": "m2"}]}),+ ScriptedResponse(json={"results": [{"metadata": {"id": "m1-last"}, "metrics": {}}]}),+ ScriptedResponse(json={"results": [{"metadata": {"id": "m2-first"}, "metrics": {}}]}),+ ],+ resume_state=OutbrainResumeConfig(+ next_index=0, next_offset=REPORT_ROW_LIMIT, window_from="2020-01-01", window_to="2020-01-31"+ ),+ )++ assert result.raised is None+ assert [(request.param("from"), request.param("to")) for request in result.requests[1:]] == [+ ("2020-01-01", "2020-01-31")+ ] * 2+ assert [request.param("offset") for request in result.requests[1:]] == [str(REPORT_ROW_LIMIT), "0"]+ assert [row["_marketer_id"] for row in result.rows] == ["m1", "m2"]+class TestOutbrainSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))