Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse): add outbrain breakdown report and conversion event tables (#123994)

Tom Owersbc543d104c3f
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index f0952a12d897..a1ef91eb5708 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -833,8 +833,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/ortto/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/oura/oura.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/oura/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index f1733e76f020..ff0cdee83484 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9003,10 +9003,10 @@ Today (6): `budgets`, `campaign_performance`, `campaigns`, `marketer_performance
Diffed against: <https://jsapi.apiary.io/apis/amplifyv01/api-description-document>
-- [ ] `/reports/marketers/{id}/promotedContent` — per-ad (promoted link) performance - we sync promoted_links metadata but no spend/clicks/conversions against them (high)
-- [ ] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)
-- [ ] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)
-- [ ] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)
+- [x] `/reports/marketers/{id}/promotedContent` — per-ad (promoted link) performance - we sync promoted_links metadata but no spend/clicks/conversions against them (high)
+- [x] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)
+- [x] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)
+- [x] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)
- [ ] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)
- [ ] `/reports/marketers/{id}/geo` — geo breakdown with country/region granularity (medium)
- [ ] `/reports/marketers/{id}/platforms` — device/platform breakdown (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
index 9b4b5779a71d..0d5953f378ae 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
@@ -93,6 +93,25 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"cachedImageUrl": "URL of the cached version of the promoted link's image.",
},
},
+ "conversion_events": {
+ "description": "A conversion event (pixel or URL rule) defined by a marketer. Resolves the conversion ids in report conversion details.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/multiple-conversions",
+ "columns": {
+ "id": "Unique identifier for the conversion event.",
+ "_marketer_id": "ID of the marketer that owns the conversion event (injected during sync).",
+ "marketerId": "ID of the marketer that owns the conversion event.",
+ "name": "The conversion event's name.",
+ "type": "How the conversion is detected (e.g. URL_BASED, EVENT_BASED).",
+ "category": "Conversion category (e.g. PURCHASE).",
+ "enabled": "Whether the conversion event is enabled.",
+ "trackingStatus": "Whether the conversion event is currently receiving data (ACTIVE or INACTIVE).",
+ "conversionWindow": "Number of days after a click in which a conversion is attributed.",
+ "includeInConversions": "Whether the event counts toward the reported conversions metric.",
+ "url": "URL rule that triggers a URL_BASED conversion.",
+ "creationTime": "Time at which the conversion event was created.",
+ "lastModified": "Time at which the conversion event was last modified.",
+ },
+ },
"marketer_performance_daily": {
"description": "Daily performance metrics (impressions, clicks, spend, conversions) per marketer.",
"docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",
@@ -112,4 +131,31 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
**_REPORT_METRICS,
},
},
+ "promoted_link_performance": {
+ "description": "Per-promoted-link performance totals for a trailing window (a snapshot, full-refreshed each sync).",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/promoted-content",
+ "columns": {
+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",
+ "metadata": "The promoted link the metrics belong to, including its id, campaignId, title, url and status.",
+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the promoted link.",
+ },
+ },
+ "publisher_performance": {
+ "description": "Per-publisher performance totals for a trailing window (a snapshot, full-refreshed each sync).",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/publishers",
+ "columns": {
+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",
+ "metadata": "The publisher the metrics belong to, including its id, name, url and block status.",
+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the publisher.",
+ },
+ },
+ "section_performance": {
+ "description": "Per-section performance totals for a trailing window (a snapshot, full-refreshed each sync). Sections with fewer than 1 click or 10 impressions are omitted.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/sections",
+ "columns": {
+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",
+ "metadata": "The section the metrics belong to, including its id, name, publisherId, publisherName and block status.",
+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the section.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
index 28a43322f1af..c727e12b0257 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
@@ -25,6 +25,7 @@ REQUEST_TIMEOUT_SECONDS = 60
MAX_RETRY_ATTEMPTS = 5
PAGE_SIZE = 100
# Periodic reports return one row per day; 500 covers the full default backfill.
+# Breakdown reports page through with offset at the same size.
REPORT_ROW_LIMIT = 500
# Tokens are valid 30 days but /login is capped at 2 requests/hour, so the
# token is cached across runs and only re-minted when it expires.
@@ -35,10 +36,15 @@ class OutbrainRetryableError(Exception):
pass
[email protected](frozen=True)
class OutbrainResumeConfig:
# Index of the next fan-out parent (marketer or campaign) to process.
next_index: int
+ # Report offset to resume at within that marketer (paginated breakdown reports only).
+ next_offset: int = 0
+ # Report window the offset belongs to, so a resume after UTC midnight keeps paging the same window.
+ window_from: str | None = None
+ window_to: str | None = None
def _get_session(password: str, token: Optional[str] = None) -> requests.Session:
@@ -208,38 +214,62 @@ def get_rows(
# Conversions restate for ~30 days — re-pull the lookback window.
window_start = watermark - timedelta(days=REPORT_LOOKBACK_DAYS)
+ start_offset = 0
+ if resume_config is not None and config.paginated and resume_config.next_offset:
+ start_offset = resume_config.next_offset
+ saved_from, saved_to = _to_date(resume_config.window_from), _to_date(resume_config.window_to)
+ if saved_from is not None and saved_to is not None:
+ window_start, today = saved_from, saved_to
marketer_ids = list_marketer_ids()
for index in range(start_index, len(marketer_ids)):
marketer_id = marketer_ids[index]
- params = urlencode(
- {
- "from": window_start.isoformat(),
- "to": today.isoformat(),
- "breakdown": "daily",
- "limit": REPORT_ROW_LIMIT,
- }
- if config.kind == "periodic_report"
- else {"from": window_start.isoformat(), "to": today.isoformat(), "limit": REPORT_ROW_LIMIT}
- )
path = config.path.format(marketer_id=quote(marketer_id))
- rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{params}"), config.data_key)
-
- out: list[dict[str, Any]] = []
- for row in rows:
- enriched = {**row, "_marketer_id": marketer_id}
+ offset = start_offset if index == start_index else 0
+ while True:
+ params: dict[str, Any] = {"from": window_start.isoformat(), "to": today.isoformat()}
if config.kind == "periodic_report":
- metadata_value = row.get("metadata")
- metadata = metadata_value if isinstance(metadata_value, dict) else {}
- row_date = metadata.get("fromDate") or metadata.get("id")
- if not row_date:
- continue
- enriched["_date"] = row_date
- out.append(enriched)
- if out:
- yield out
-
- # Save state AFTER yielding so a crash re-yields the in-flight marketer.
- resumable_source_manager.save_state(OutbrainResumeConfig(next_index=index + 1))
+ params["breakdown"] = "daily"
+ params["limit"] = REPORT_ROW_LIMIT
+ if config.paginated:
+ params["offset"] = offset
+ if config.include_inactive_campaigns:
+ params["includeArchivedCampaigns"] = "true"
+ params["enabledCampaignsOnly"] = "false"
+ rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}"), config.data_key)
+
+ out: list[dict[str, Any]] = []
+ for row in rows:
+ enriched = {**row, "_marketer_id": marketer_id}
+ if config.kind == "periodic_report":
+ metadata_value = row.get("metadata")
+ metadata = metadata_value if isinstance(metadata_value, dict) else {}
+ row_date = metadata.get("fromDate") or metadata.get("id")
+ if not row_date:
+ continue
+ enriched["_date"] = row_date
+ out.append(enriched)
+
+ has_more = config.paginated and len(rows) >= REPORT_ROW_LIMIT
+ next_state = (
+ OutbrainResumeConfig(
+ next_index=index,
+ next_offset=offset + len(rows),
+ window_from=window_start.isoformat(),
+ window_to=today.isoformat(),
+ )
+ if has_more
+ else OutbrainResumeConfig(next_index=index + 1)
+ )
+ # Stage before the yield so a resumed full refresh never re-appends a written page.
+ resumable_source_manager.save_state(next_state)
+ if out:
+ yield out
+ else:
+ resumable_source_manager.safe_point()
+
+ if not has_more:
+ break
+ offset += len(rows)
def _to_date(value: Any) -> Optional[date]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
index 519094fcdb1d..927442e086c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
@@ -23,7 +23,7 @@ _DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [
]
-@dataclass
+@dataclass(frozen=True)
class OutbrainEndpointConfig:
name: str
kind: OutbrainEndpointKind
@@ -34,8 +34,11 @@ class OutbrainEndpointConfig:
data_key: str
primary_keys: list[str] | None = field(default_factory=lambda: ["id"])
incremental_fields: list[IncrementalField] = field(default_factory=list)
- # Entity endpoints paginate with limit/offset.
+ # Entity and breakdown report endpoints paginate with limit/offset.
paginated: bool = False
+ # Reports default to enabled, unarchived campaigns only, which would drop
+ # spend from campaigns paused or archived inside the window.
+ include_inactive_campaigns: bool = False
OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
@@ -67,6 +70,12 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
data_key="promotedLinks",
paginated=True,
),
+ "conversion_events": OutbrainEndpointConfig(
+ name="conversion_events",
+ kind="per_marketer",
+ path="/marketers/{marketer_id}/conversionEvents",
+ data_key="conversionEvents",
+ ),
# Daily periodic performance per marketer; rows carry the day in
# metadata, injected as `_date` for the incremental cursor.
"marketer_performance_daily": OutbrainEndpointConfig(
@@ -86,6 +95,35 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
data_key="results",
primary_keys=None,
),
+ # Breakdown reports have no date dimension either, so they are trailing
+ # window snapshots too. One row per publisher/section/promoted link.
+ "promoted_link_performance": OutbrainEndpointConfig(
+ name="promoted_link_performance",
+ kind="snapshot_report",
+ path="/reports/marketers/{marketer_id}/promotedContent",
+ data_key="results",
+ primary_keys=None,
+ paginated=True,
+ include_inactive_campaigns=True,
+ ),
+ "publisher_performance": OutbrainEndpointConfig(
+ name="publisher_performance",
+ kind="snapshot_report",
+ path="/reports/marketers/{marketer_id}/publishers",
+ data_key="results",
+ primary_keys=None,
+ paginated=True,
+ include_inactive_campaigns=True,
+ ),
+ "section_performance": OutbrainEndpointConfig(
+ name="section_performance",
+ kind="snapshot_report",
+ path="/reports/marketers/{marketer_id}/sections",
+ data_key="results",
+ primary_keys=None,
+ paginated=True,
+ include_inactive_campaigns=True,
+ ),
}
ENDPOINTS = tuple(OUTBRAIN_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
index 389120170e83..7524a743e5f1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
@@ -12,6 +12,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
)
from products.warehouse_sources.backend.temporal.data_imports.sources.outbrain.outbrain import (
PAGE_SIZE,
+ REPORT_ROW_LIMIT,
OutbrainResumeConfig,
validate_credentials,
)
@@ -178,6 +179,62 @@ class TestReportStreams:
assert "_date" not in result.rows[0]
assert result.requests[1].param("breakdown") is None
+ @pytest.mark.parametrize(
+ "endpoint, report_path",
+ [
+ ("promoted_link_performance", "promotedContent"),
+ ("publisher_performance", "publishers"),
+ ("section_performance", "sections"),
+ ],
+ )
+ def test_breakdown_report_pages_with_offset(
+ self, mock_cache: mock.MagicMock, endpoint: str, report_path: str
+ ) -> None:
+ mock_cache.get.return_value = "tok"
+ full_page = [{"metadata": {"id": f"r{i}"}, "metrics": {"clicks": i}} for i in range(REPORT_ROW_LIMIT)]
+ result = _driver().run(
+ endpoint,
+ [
+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),
+ ScriptedResponse(json={"results": full_page, "totalResults": REPORT_ROW_LIMIT + 1}),
+ ScriptedResponse(json={"results": [{"metadata": {"id": "r-last"}, "metrics": {}}]}),
+ ],
+ )
+
+ assert result.raised is None
+ assert [len(batch) for batch in result.items] == [REPORT_ROW_LIMIT, 1]
+ assert all(row["_marketer_id"] == "m1" for row in result.rows)
+ assert result.paths[1:] == [f"/amplify/v0.1/reports/marketers/m1/{report_path}"] * 2
+ assert [request.param("offset") for request in result.requests[1:]] == ["0", str(REPORT_ROW_LIMIT)]
+ assert all(request.param("includeArchivedCampaigns") == "true" for request in result.requests[1:])
+ assert all(request.param("enabledCampaignsOnly") == "false" for request in result.requests[1:])
+ window = {"window_from": result.requests[1].param("from"), "window_to": result.requests[1].param("to")}
+ assert result.saved_states == [
+ OutbrainResumeConfig(next_index=0, next_offset=REPORT_ROW_LIMIT, **window),
+ OutbrainResumeConfig(next_index=1),
+ ]
+
+ def test_breakdown_report_resumes_at_saved_offset(self, mock_cache: mock.MagicMock) -> None:
+ mock_cache.get.return_value = "tok"
+ result = _driver().run(
+ "publisher_performance",
+ [
+ ScriptedResponse(json={"marketers": [{"id": "m1"}, {"id": "m2"}]}),
+ ScriptedResponse(json={"results": [{"metadata": {"id": "m1-last"}, "metrics": {}}]}),
+ ScriptedResponse(json={"results": [{"metadata": {"id": "m2-first"}, "metrics": {}}]}),
+ ],
+ resume_state=OutbrainResumeConfig(
+ next_index=0, next_offset=REPORT_ROW_LIMIT, window_from="2020-01-01", window_to="2020-01-31"
+ ),
+ )
+
+ assert result.raised is None
+ assert [(request.param("from"), request.param("to")) for request in result.requests[1:]] == [
+ ("2020-01-01", "2020-01-31")
+ ] * 2
+ assert [request.param("offset") for request in result.requests[1:]] == [str(REPORT_ROW_LIMIT), "0"]
+ assert [row["_marketer_id"] for row in result.rows] == ["m1", "m2"]
+
class TestOutbrainSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))