Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse-sources): add four outbrain endpoint tables (#124782)

Tom Owers89ea610058c2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 189ca9fbe50e..4f6e48b08acc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9008,12 +9008,12 @@ Diffed against: <https://jsapi.apiary.io/apis/amplifyv01/api-description-documen
- [x] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)
- [x] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)
- [x] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)
-- [ ] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)
+- [x] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)
- [ ] `/reports/marketers/{id}/geo` — geo breakdown with country/region granularity (medium)
- [ ] `/reports/marketers/{id}/platforms` — device/platform breakdown (medium)
-- [ ] `/reports/marketers/{id}/interests` — interest-targeting breakdown (medium)
-- [ ] `/marketers/{id}/segments` — audience segments lookup for segment-targeted campaigns (medium)
-- [ ] `/campaigns/{campaignId}/locations` — campaign geo targeting rows, needed to interpret geo report deltas (medium)
+- [x] `/reports/marketers/{id}/interests` — interest-targeting breakdown (medium)
+- [x] `/marketers/{id}/segments` — audience segments lookup for segment-targeted campaigns (medium)
+- [x] `/campaigns/{campaignId}/locations` — campaign geo targeting rows, needed to interpret geo report deltas (medium)
- [ ] `/marketers/{id}/engagementMetrics` — marketer-level engagement metrics not present in the periodic report (low)
- [ ] `/campaigns/{campaignId}/promotedLinksSequences` — sequence grouping for promoted links (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
index 0d5953f378ae..b67fc4bf8cae 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py
@@ -112,6 +112,41 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"lastModified": "Time at which the conversion event was last modified.",
},
},
+ "segments": {
+ "description": "An audience segment (pixel-based or campaign clickers) defined by a marketer, including archived segments. Resolves the segment ids used in campaign targeting.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/segments",
+ "columns": {
+ "id": "Unique identifier for the segment.",
+ "_marketer_id": "ID of the marketer that owns the segment (injected during sync).",
+ "marketerId": "ID of the marketer that owns the segment.",
+ "name": "The segment's name.",
+ "description": "The segment's description.",
+ "segmentType": "How segment membership is built (PIXEL or CLICKERS).",
+ "pixelTypeRules": "URL rules that add users to a PIXEL segment.",
+ "clickersCampaigns": "Campaigns whose clickers make up a CLICKERS segment.",
+ "enabled": "Whether the segment is enabled.",
+ "archived": "Whether the segment is archived.",
+ "membershipDuration": "Number of days a user stays in the segment.",
+ "trackingStatus": "Whether the segment is currently receiving data (ACTIVE or INACTIVE).",
+ "size": "Number of users in the segment.",
+ "creationTime": "Time at which the segment was created.",
+ "lastModified": "Time at which the segment was last modified.",
+ },
+ },
+ "campaign_locations": {
+ "description": "The geo locations a campaign targets or excludes, one row per location per campaign.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/campaigns",
+ "columns": {
+ "id": "Unique identifier for the geo location.",
+ "_campaign_id": "ID of the campaign the targeting belongs to (injected during sync).",
+ "_excluded": "True when the campaign excludes the location, false when it targets it (injected during sync).",
+ "name": "The location's name.",
+ "canonicalName": "The location's full name, including its parent locations.",
+ "geoType": "Location level (e.g. Country, Region, DMA, City, PostalCode).",
+ "code": "Location code, e.g. the ISO country code for countries.",
+ "parent": "The parent location, for locations below country level.",
+ },
+ },
"marketer_performance_daily": {
"description": "Daily performance metrics (impressions, clicks, spend, conversions) per marketer.",
"docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",
@@ -122,6 +157,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
**_REPORT_METRICS,
},
},
+ "campaign_performance_daily": {
+ "description": "Daily performance metrics (impressions, clicks, spend, conversions) per campaign, including paused and archived campaigns.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/periodic-by-campaign",
+ "columns": {
+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",
+ "_campaign_id": "ID of the campaign the metrics belong to (injected during sync).",
+ "_date": "The day the metrics cover, used as the incremental cursor (injected during sync).",
+ "metadata": "Metadata for the report row, including the date range it covers.",
+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the campaign on that day.",
+ },
+ },
"campaign_performance": {
"description": "Per-campaign performance totals for a trailing window (a snapshot, full-refreshed each sync).",
"docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",
@@ -158,4 +204,13 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"metrics": "Performance metrics (impressions, clicks, spend, conversions) for the section.",
},
},
+ "interest_performance": {
+ "description": "Per-interest performance totals for a trailing window (a snapshot, full-refreshed each sync). A user can have many interests, so the same action counts toward each of them and rows do not sum to the account total.",
+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/interests",
+ "columns": {
+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",
+ "metadata": "The interest the metrics belong to, including its id, name (e.g. 'News > Weather') and targetingStatus.",
+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the interest.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
index c727e12b0257..e6abbb3fef42 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py
@@ -27,6 +27,9 @@ PAGE_SIZE = 100
# Periodic reports return one row per day; 500 covers the full default backfill.
# Breakdown reports page through with offset at the same size.
REPORT_ROW_LIMIT = 500
+# The campaign periodic report applies its limit per campaign, so a page holds
+# this many days for every campaign; kept small to bound the response size.
+CAMPAIGN_PERIODIC_ROW_LIMIT = 31
# Tokens are valid 30 days but /login is capped at 2 requests/hour, so the
# token is cached across runs and only re-minted when it expires.
TOKEN_CACHE_TTL_SECONDS = 29 * 24 * 60 * 60
@@ -146,11 +149,17 @@ def get_rows(
def fetch_paginated(path: str) -> Iterator[list[dict[str, Any]]]:
offset = 0
while True:
- params = {"limit": PAGE_SIZE, "offset": offset} if config.paginated else {}
+ params: dict[str, Any] = {"limit": PAGE_SIZE, "offset": offset} if config.paginated else {}
+ params.update(config.params)
url = f"{OUTBRAIN_BASE_URL}{path}"
if params:
url = f"{url}?{urlencode(params)}"
- rows = rows_of(fetch(url), config.data_key)
+ data = fetch(url)
+ rows = rows_of(data, config.data_key)
+ if config.excluded_data_key:
+ rows = [{**row, "_excluded": False} for row in rows] + [
+ {**row, "_excluded": True} for row in rows_of(data, config.excluded_data_key)
+ ]
if rows:
yield rows
if not config.paginated or len(rows) < PAGE_SIZE:
@@ -220,6 +229,8 @@ def get_rows(
saved_from, saved_to = _to_date(resume_config.window_from), _to_date(resume_config.window_to)
if saved_from is not None and saved_to is not None:
window_start, today = saved_from, saved_to
+ is_periodic = config.kind in ("periodic_report", "campaign_periodic_report")
+ row_limit = CAMPAIGN_PERIODIC_ROW_LIMIT if config.kind == "campaign_periodic_report" else REPORT_ROW_LIMIT
marketer_ids = list_marketer_ids()
for index in range(start_index, len(marketer_ids)):
marketer_id = marketer_ids[index]
@@ -227,33 +238,47 @@ def get_rows(
offset = start_offset if index == start_index else 0
while True:
params: dict[str, Any] = {"from": window_start.isoformat(), "to": today.isoformat()}
- if config.kind == "periodic_report":
+ if is_periodic:
params["breakdown"] = "daily"
- params["limit"] = REPORT_ROW_LIMIT
+ params["limit"] = row_limit
if config.paginated:
params["offset"] = offset
if config.include_inactive_campaigns:
params["includeArchivedCampaigns"] = "true"
params["enabledCampaignsOnly"] = "false"
- rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}"), config.data_key)
+ data = fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}")
+
+ # The campaign periodic report nests each campaign's daily rows under it.
+ row_groups: list[tuple[Optional[str], list[dict[str, Any]]]]
+ if config.kind == "campaign_periodic_report":
+ row_groups = [
+ (str(campaign["campaignId"]), rows_of(campaign, "results"))
+ for campaign in rows_of(data, config.data_key)
+ if campaign.get("campaignId")
+ ]
+ else:
+ row_groups = [(None, rows_of(data, config.data_key))]
out: list[dict[str, Any]] = []
- for row in rows:
- enriched = {**row, "_marketer_id": marketer_id}
- if config.kind == "periodic_report":
- metadata_value = row.get("metadata")
- metadata = metadata_value if isinstance(metadata_value, dict) else {}
- row_date = metadata.get("fromDate") or metadata.get("id")
- if not row_date:
- continue
- enriched["_date"] = row_date
- out.append(enriched)
-
- has_more = config.paginated and len(rows) >= REPORT_ROW_LIMIT
+ for campaign_id, rows in row_groups:
+ for row in rows:
+ enriched = {**row, "_marketer_id": marketer_id}
+ if campaign_id is not None:
+ enriched["_campaign_id"] = campaign_id
+ if is_periodic:
+ metadata_value = row.get("metadata")
+ metadata = metadata_value if isinstance(metadata_value, dict) else {}
+ row_date = metadata.get("fromDate") or metadata.get("id")
+ if not row_date:
+ continue
+ enriched["_date"] = row_date
+ out.append(enriched)
+
+ has_more = config.paginated and any(len(rows) >= row_limit for _, rows in row_groups)
next_state = (
OutbrainResumeConfig(
next_index=index,
- next_offset=offset + len(rows),
+ next_offset=offset + row_limit,
window_from=window_start.isoformat(),
window_to=today.isoformat(),
)
@@ -269,7 +294,7 @@ def get_rows(
if not has_more:
break
- offset += len(rows)
+ offset += row_limit
def _to_date(value: Any) -> Optional[date]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
index 927442e086c6..e2257f79950d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py
@@ -11,7 +11,9 @@ REPORT_LOOKBACK_DAYS = 30
# Default history pulled on the first sync of the periodic report stream.
REPORT_DEFAULT_BACKFILL_DAYS = 365
-OutbrainEndpointKind = Literal["marketers", "per_marketer", "per_campaign", "periodic_report", "snapshot_report"]
+OutbrainEndpointKind = Literal[
+ "marketers", "per_marketer", "per_campaign", "periodic_report", "campaign_periodic_report", "snapshot_report"
+]
_DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [
{
@@ -32,6 +34,11 @@ class OutbrainEndpointConfig:
path: str
# Key holding the row list in the response body.
data_key: str
+ # Key holding a second list of excluded targeting rows in the same body;
+ # both lists land in one table, tagged with `_excluded`.
+ excluded_data_key: str | None = None
+ # Static query params sent on every request.
+ params: dict[str, str] = field(default_factory=dict)
primary_keys: list[str] | None = field(default_factory=lambda: ["id"])
incremental_fields: list[IncrementalField] = field(default_factory=list)
# Entity and breakdown report endpoints paginate with limit/offset.
@@ -76,6 +83,23 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
path="/marketers/{marketer_id}/conversionEvents",
data_key="conversionEvents",
),
+ # Archived segments can still be referenced by campaign targeting.
+ "segments": OutbrainEndpointConfig(
+ name="segments",
+ kind="per_marketer",
+ path="/marketers/{marketer_id}/segments",
+ data_key="segments",
+ params={"includeArchived": "true"},
+ ),
+ # Location ids are global geo ids, so the campaign is part of the key.
+ "campaign_locations": OutbrainEndpointConfig(
+ name="campaign_locations",
+ kind="per_campaign",
+ path="/campaigns/{campaign_id}/locations",
+ data_key="geoLocations",
+ excluded_data_key="excludedGeoLocations",
+ primary_keys=["_campaign_id", "id"],
+ ),
# Daily periodic performance per marketer; rows carry the day in
# metadata, injected as `_date` for the incremental cursor.
"marketer_performance_daily": OutbrainEndpointConfig(
@@ -86,6 +110,18 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
primary_keys=["_marketer_id", "_date"],
incremental_fields=list(_DATE_INCREMENTAL_FIELDS),
),
+ # Daily periodic performance per campaign; rows are nested under each
+ # campaign and carry the day in metadata, like the marketer report.
+ "campaign_performance_daily": OutbrainEndpointConfig(
+ name="campaign_performance_daily",
+ kind="campaign_periodic_report",
+ path="/reports/marketers/{marketer_id}/campaigns/periodic",
+ data_key="campaignResults",
+ primary_keys=["_marketer_id", "_campaign_id", "_date"],
+ incremental_fields=list(_DATE_INCREMENTAL_FIELDS),
+ paginated=True,
+ include_inactive_campaigns=True,
+ ),
# Per-campaign totals for a trailing window; rows carry no date, so this
# is a snapshot (full refresh).
"campaign_performance": OutbrainEndpointConfig(
@@ -96,7 +132,7 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
primary_keys=None,
),
# Breakdown reports have no date dimension either, so they are trailing
- # window snapshots too. One row per publisher/section/promoted link.
+ # window snapshots too. One row per publisher/section/promoted link/interest.
"promoted_link_performance": OutbrainEndpointConfig(
name="promoted_link_performance",
kind="snapshot_report",
@@ -124,6 +160,15 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {
paginated=True,
include_inactive_campaigns=True,
),
+ "interest_performance": OutbrainEndpointConfig(
+ name="interest_performance",
+ kind="snapshot_report",
+ path="/reports/marketers/{marketer_id}/interests",
+ data_key="results",
+ primary_keys=None,
+ paginated=True,
+ include_inactive_campaigns=True,
+ ),
}
ENDPOINTS = tuple(OUTBRAIN_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
index 7524a743e5f1..b7dc9ec8d56e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py
@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_
OutbrainSourceConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.outbrain.outbrain import (
+ CAMPAIGN_PERIODIC_ROW_LIMIT,
PAGE_SIZE,
REPORT_ROW_LIMIT,
OutbrainResumeConfig,
@@ -82,6 +83,44 @@ class TestEntityStreams:
assert [(row["id"], row["_campaign_id"]) for row in result.rows] == [("pl1", "c1")]
assert result.requests[2].path == "/amplify/v0.1/campaigns/c1/promotedLinks"
+ def test_segments_include_archived(self, mock_cache: mock.MagicMock) -> None:
+ mock_cache.get.return_value = "tok"
+ result = _driver().run(
+ "segments",
+ [
+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),
+ ScriptedResponse(json={"segments": [{"id": "s1"}], "count": 1}),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["id"], row["_marketer_id"]) for row in result.rows] == [("s1", "m1")]
+ assert result.requests[1].path == "/amplify/v0.1/marketers/m1/segments"
+ assert result.requests[1].param("includeArchived") == "true"
+
+ def test_campaign_locations_tag_targeted_and_excluded_rows(self, mock_cache: mock.MagicMock) -> None:
+ mock_cache.get.return_value = "tok"
+ result = _driver().run(
+ "campaign_locations",
+ [
+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),
+ ScriptedResponse(json={"campaigns": [{"id": "c1"}]}),
+ ScriptedResponse(
+ json={
+ "geoLocations": [{"id": "us", "geoType": "Country"}],
+ "excludedGeoLocations": [{"id": "al", "geoType": "Region", "parent": {"id": "us"}}],
+ }
+ ),
+ ],
+ )
+
+ assert result.raised is None
+ assert [(row["id"], row["_campaign_id"], row["_excluded"]) for row in result.rows] == [
+ ("us", "c1", False),
+ ("al", "c1", True),
+ ]
+ assert result.requests[2].path == "/amplify/v0.1/campaigns/c1/locations"
+
def test_resumes_from_saved_parent_index(self, mock_cache: mock.MagicMock) -> None:
mock_cache.get.return_value = "tok"
result = _driver().run(
@@ -179,12 +218,64 @@ class TestReportStreams:
assert "_date" not in result.rows[0]
assert result.requests[1].param("breakdown") is None
+ def test_campaign_periodic_report_flattens_campaigns_and_pages_until_no_campaign_is_full(
+ self, mock_cache: mock.MagicMock
+ ) -> None:
+ mock_cache.get.return_value = "tok"
+ full_campaign = [
+ {"metadata": {"fromDate": f"2024-01-{day:02d}"}, "metrics": {"clicks": day}}
+ for day in range(1, CAMPAIGN_PERIODIC_ROW_LIMIT + 1)
+ ]
+ result = _driver().run(
+ "campaign_performance_daily",
+ [
+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),
+ ScriptedResponse(
+ json={
+ "campaignResults": [
+ {"campaignId": "c1", "results": full_campaign},
+ {"campaignId": "c2", "results": [{"metadata": {"fromDate": "2024-01-05"}, "metrics": {}}]},
+ {"results": [{"metadata": {"fromDate": "2024-01-05"}, "metrics": {}}]},
+ ],
+ "totalCampaigns": 3,
+ }
+ ),
+ ScriptedResponse(
+ json={
+ "campaignResults": [
+ {"campaignId": "c1", "results": [{"metadata": {"fromDate": "2024-02-01"}, "metrics": {}}]}
+ ]
+ }
+ ),
+ ],
+ incremental_field="_date",
+ db_incremental_field_last_value="2024-06-01",
+ )
+
+ assert result.raised is None
+ assert [len(batch) for batch in result.items] == [CAMPAIGN_PERIODIC_ROW_LIMIT + 1, 1]
+ assert result.rows[CAMPAIGN_PERIODIC_ROW_LIMIT] == {
+ "metadata": {"fromDate": "2024-01-05"},
+ "metrics": {},
+ "_marketer_id": "m1",
+ "_campaign_id": "c2",
+ "_date": "2024-01-05",
+ }
+ assert result.rows[-1]["_campaign_id"] == "c1"
+ assert result.paths[1:] == ["/amplify/v0.1/reports/marketers/m1/campaigns/periodic"] * 2
+ assert [request.param("offset") for request in result.requests[1:]] == ["0", str(CAMPAIGN_PERIODIC_ROW_LIMIT)]
+ assert all(request.param("limit") == str(CAMPAIGN_PERIODIC_ROW_LIMIT) for request in result.requests[1:])
+ assert all(request.param("breakdown") == "daily" for request in result.requests[1:])
+ assert all(request.param("includeArchivedCampaigns") == "true" for request in result.requests[1:])
+ assert result.requests[1].param("from") == "2024-05-02"
+
@pytest.mark.parametrize(
"endpoint, report_path",
[
("promoted_link_performance", "promotedContent"),
("publisher_performance", "publishers"),
("section_performance", "sections"),
+ ("interest_performance", "interests"),
],
)
def test_breakdown_report_pages_with_offset(