:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse-sources): add four outbrain endpoint tables (#124782)
Tom Owers89ea610058c2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 189ca9fbe50e..4f6e48b08acc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9008,12 +9008,12 @@ Diffed against: <https://jsapi.apiary.io/apis/amplifyv01/api-description-documen- [x] `/reports/marketers/{id}/publishers` — publisher breakdown, the core Outbrain optimization dimension (high)- [x] `/reports/marketers/{id}/sections` — section/placement breakdown - the finest-grained supply dimension advertisers block or bid on (high)- [x] `/marketers/{id}/conversionEvents` — lookup table resolving the conversion event ids that appear in every report's conversion details (high)-- [ ] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)+- [x] `/reports/marketers/{id}/campaigns/periodic` — daily per-campaign time series; campaign_performance today is only a date-range rollup (high)- [ ] `/reports/marketers/{id}/geo` — geo breakdown with country/region granularity (medium)- [ ] `/reports/marketers/{id}/platforms` — device/platform breakdown (medium)-- [ ] `/reports/marketers/{id}/interests` — interest-targeting breakdown (medium)-- [ ] `/marketers/{id}/segments` — audience segments lookup for segment-targeted campaigns (medium)-- [ ] `/campaigns/{campaignId}/locations` — campaign geo targeting rows, needed to interpret geo report deltas (medium)+- [x] `/reports/marketers/{id}/interests` — interest-targeting breakdown (medium)+- [x] `/marketers/{id}/segments` — audience segments lookup for segment-targeted campaigns (medium)+- [x] `/campaigns/{campaignId}/locations` — campaign geo targeting rows, needed to interpret geo report deltas (medium)- [ ] `/marketers/{id}/engagementMetrics` — marketer-level engagement metrics not present in the periodic report (low)- [ ] `/campaigns/{campaignId}/promotedLinksSequences` — sequence grouping for promoted links (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.pyindex 0d5953f378ae..b67fc4bf8cae 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/canonical_descriptions.py@@ -112,6 +112,41 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"lastModified": "Time at which the conversion event was last modified.",},},+ "segments": {+ "description": "An audience segment (pixel-based or campaign clickers) defined by a marketer, including archived segments. Resolves the segment ids used in campaign targeting.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/segments",+ "columns": {+ "id": "Unique identifier for the segment.",+ "_marketer_id": "ID of the marketer that owns the segment (injected during sync).",+ "marketerId": "ID of the marketer that owns the segment.",+ "name": "The segment's name.",+ "description": "The segment's description.",+ "segmentType": "How segment membership is built (PIXEL or CLICKERS).",+ "pixelTypeRules": "URL rules that add users to a PIXEL segment.",+ "clickersCampaigns": "Campaigns whose clickers make up a CLICKERS segment.",+ "enabled": "Whether the segment is enabled.",+ "archived": "Whether the segment is archived.",+ "membershipDuration": "Number of days a user stays in the segment.",+ "trackingStatus": "Whether the segment is currently receiving data (ACTIVE or INACTIVE).",+ "size": "Number of users in the segment.",+ "creationTime": "Time at which the segment was created.",+ "lastModified": "Time at which the segment was last modified.",+ },+ },+ "campaign_locations": {+ "description": "The geo locations a campaign targets or excludes, one row per location per campaign.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/campaigns",+ "columns": {+ "id": "Unique identifier for the geo location.",+ "_campaign_id": "ID of the campaign the targeting belongs to (injected during sync).",+ "_excluded": "True when the campaign excludes the location, false when it targets it (injected during sync).",+ "name": "The location's name.",+ "canonicalName": "The location's full name, including its parent locations.",+ "geoType": "Location level (e.g. Country, Region, DMA, City, PostalCode).",+ "code": "Location code, e.g. the ISO country code for countries.",+ "parent": "The parent location, for locations below country level.",+ },+ },"marketer_performance_daily": {"description": "Daily performance metrics (impressions, clicks, spend, conversions) per marketer.","docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",@@ -122,6 +157,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {**_REPORT_METRICS,},},+ "campaign_performance_daily": {+ "description": "Daily performance metrics (impressions, clicks, spend, conversions) per campaign, including paused and archived campaigns.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/periodic-by-campaign",+ "columns": {+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",+ "_campaign_id": "ID of the campaign the metrics belong to (injected during sync).",+ "_date": "The day the metrics cover, used as the incremental cursor (injected during sync).",+ "metadata": "Metadata for the report row, including the date range it covers.",+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the campaign on that day.",+ },+ },"campaign_performance": {"description": "Per-campaign performance totals for a trailing window (a snapshot, full-refreshed each sync).","docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting",@@ -158,4 +204,13 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"metrics": "Performance metrics (impressions, clicks, spend, conversions) for the section.",},},+ "interest_performance": {+ "description": "Per-interest performance totals for a trailing window (a snapshot, full-refreshed each sync). A user can have many interests, so the same action counts toward each of them and rows do not sum to the account total.",+ "docs_url": "https://amplifyv01.docs.apiary.io/#reference/performance-reporting/interests",+ "columns": {+ "_marketer_id": "ID of the marketer the metrics belong to (injected during sync).",+ "metadata": "The interest the metrics belong to, including its id, name (e.g. 'News > Weather') and targetingStatus.",+ "metrics": "Performance metrics (impressions, clicks, spend, conversions) for the interest.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.pyindex c727e12b0257..e6abbb3fef42 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/outbrain.py@@ -27,6 +27,9 @@ PAGE_SIZE = 100# Periodic reports return one row per day; 500 covers the full default backfill.# Breakdown reports page through with offset at the same size.REPORT_ROW_LIMIT = 500+# The campaign periodic report applies its limit per campaign, so a page holds+# this many days for every campaign; kept small to bound the response size.+CAMPAIGN_PERIODIC_ROW_LIMIT = 31# Tokens are valid 30 days but /login is capped at 2 requests/hour, so the# token is cached across runs and only re-minted when it expires.TOKEN_CACHE_TTL_SECONDS = 29 * 24 * 60 * 60@@ -146,11 +149,17 @@ def get_rows(def fetch_paginated(path: str) -> Iterator[list[dict[str, Any]]]:offset = 0while True:- params = {"limit": PAGE_SIZE, "offset": offset} if config.paginated else {}+ params: dict[str, Any] = {"limit": PAGE_SIZE, "offset": offset} if config.paginated else {}+ params.update(config.params)url = f"{OUTBRAIN_BASE_URL}{path}"if params:url = f"{url}?{urlencode(params)}"- rows = rows_of(fetch(url), config.data_key)+ data = fetch(url)+ rows = rows_of(data, config.data_key)+ if config.excluded_data_key:+ rows = [{**row, "_excluded": False} for row in rows] + [+ {**row, "_excluded": True} for row in rows_of(data, config.excluded_data_key)+ ]if rows:yield rowsif not config.paginated or len(rows) < PAGE_SIZE:@@ -220,6 +229,8 @@ def get_rows(saved_from, saved_to = _to_date(resume_config.window_from), _to_date(resume_config.window_to)if saved_from is not None and saved_to is not None:window_start, today = saved_from, saved_to+ is_periodic = config.kind in ("periodic_report", "campaign_periodic_report")+ row_limit = CAMPAIGN_PERIODIC_ROW_LIMIT if config.kind == "campaign_periodic_report" else REPORT_ROW_LIMITmarketer_ids = list_marketer_ids()for index in range(start_index, len(marketer_ids)):marketer_id = marketer_ids[index]@@ -227,33 +238,47 @@ def get_rows(offset = start_offset if index == start_index else 0while True:params: dict[str, Any] = {"from": window_start.isoformat(), "to": today.isoformat()}- if config.kind == "periodic_report":+ if is_periodic:params["breakdown"] = "daily"- params["limit"] = REPORT_ROW_LIMIT+ params["limit"] = row_limitif config.paginated:params["offset"] = offsetif config.include_inactive_campaigns:params["includeArchivedCampaigns"] = "true"params["enabledCampaignsOnly"] = "false"- rows = rows_of(fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}"), config.data_key)+ data = fetch(f"{OUTBRAIN_BASE_URL}{path}?{urlencode(params)}")++ # The campaign periodic report nests each campaign's daily rows under it.+ row_groups: list[tuple[Optional[str], list[dict[str, Any]]]]+ if config.kind == "campaign_periodic_report":+ row_groups = [+ (str(campaign["campaignId"]), rows_of(campaign, "results"))+ for campaign in rows_of(data, config.data_key)+ if campaign.get("campaignId")+ ]+ else:+ row_groups = [(None, rows_of(data, config.data_key))]out: list[dict[str, Any]] = []- for row in rows:- enriched = {**row, "_marketer_id": marketer_id}- if config.kind == "periodic_report":- metadata_value = row.get("metadata")- metadata = metadata_value if isinstance(metadata_value, dict) else {}- row_date = metadata.get("fromDate") or metadata.get("id")- if not row_date:- continue- enriched["_date"] = row_date- out.append(enriched)-- has_more = config.paginated and len(rows) >= REPORT_ROW_LIMIT+ for campaign_id, rows in row_groups:+ for row in rows:+ enriched = {**row, "_marketer_id": marketer_id}+ if campaign_id is not None:+ enriched["_campaign_id"] = campaign_id+ if is_periodic:+ metadata_value = row.get("metadata")+ metadata = metadata_value if isinstance(metadata_value, dict) else {}+ row_date = metadata.get("fromDate") or metadata.get("id")+ if not row_date:+ continue+ enriched["_date"] = row_date+ out.append(enriched)++ has_more = config.paginated and any(len(rows) >= row_limit for _, rows in row_groups)next_state = (OutbrainResumeConfig(next_index=index,- next_offset=offset + len(rows),+ next_offset=offset + row_limit,window_from=window_start.isoformat(),window_to=today.isoformat(),)@@ -269,7 +294,7 @@ def get_rows(if not has_more:break- offset += len(rows)+ offset += row_limitdef _to_date(value: Any) -> Optional[date]:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.pyindex 927442e086c6..e2257f79950d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/settings.py@@ -11,7 +11,9 @@ REPORT_LOOKBACK_DAYS = 30# Default history pulled on the first sync of the periodic report stream.REPORT_DEFAULT_BACKFILL_DAYS = 365-OutbrainEndpointKind = Literal["marketers", "per_marketer", "per_campaign", "periodic_report", "snapshot_report"]+OutbrainEndpointKind = Literal[+ "marketers", "per_marketer", "per_campaign", "periodic_report", "campaign_periodic_report", "snapshot_report"+]_DATE_INCREMENTAL_FIELDS: list[IncrementalField] = [{@@ -32,6 +34,11 @@ class OutbrainEndpointConfig:path: str# Key holding the row list in the response body.data_key: str+ # Key holding a second list of excluded targeting rows in the same body;+ # both lists land in one table, tagged with `_excluded`.+ excluded_data_key: str | None = None+ # Static query params sent on every request.+ params: dict[str, str] = field(default_factory=dict)primary_keys: list[str] | None = field(default_factory=lambda: ["id"])incremental_fields: list[IncrementalField] = field(default_factory=list)# Entity and breakdown report endpoints paginate with limit/offset.@@ -76,6 +83,23 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {path="/marketers/{marketer_id}/conversionEvents",data_key="conversionEvents",),+ # Archived segments can still be referenced by campaign targeting.+ "segments": OutbrainEndpointConfig(+ name="segments",+ kind="per_marketer",+ path="/marketers/{marketer_id}/segments",+ data_key="segments",+ params={"includeArchived": "true"},+ ),+ # Location ids are global geo ids, so the campaign is part of the key.+ "campaign_locations": OutbrainEndpointConfig(+ name="campaign_locations",+ kind="per_campaign",+ path="/campaigns/{campaign_id}/locations",+ data_key="geoLocations",+ excluded_data_key="excludedGeoLocations",+ primary_keys=["_campaign_id", "id"],+ ),# Daily periodic performance per marketer; rows carry the day in# metadata, injected as `_date` for the incremental cursor."marketer_performance_daily": OutbrainEndpointConfig(@@ -86,6 +110,18 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {primary_keys=["_marketer_id", "_date"],incremental_fields=list(_DATE_INCREMENTAL_FIELDS),),+ # Daily periodic performance per campaign; rows are nested under each+ # campaign and carry the day in metadata, like the marketer report.+ "campaign_performance_daily": OutbrainEndpointConfig(+ name="campaign_performance_daily",+ kind="campaign_periodic_report",+ path="/reports/marketers/{marketer_id}/campaigns/periodic",+ data_key="campaignResults",+ primary_keys=["_marketer_id", "_campaign_id", "_date"],+ incremental_fields=list(_DATE_INCREMENTAL_FIELDS),+ paginated=True,+ include_inactive_campaigns=True,+ ),# Per-campaign totals for a trailing window; rows carry no date, so this# is a snapshot (full refresh)."campaign_performance": OutbrainEndpointConfig(@@ -96,7 +132,7 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {primary_keys=None,),# Breakdown reports have no date dimension either, so they are trailing- # window snapshots too. One row per publisher/section/promoted link.+ # window snapshots too. One row per publisher/section/promoted link/interest."promoted_link_performance": OutbrainEndpointConfig(name="promoted_link_performance",kind="snapshot_report",@@ -124,6 +160,15 @@ OUTBRAIN_ENDPOINTS: dict[str, OutbrainEndpointConfig] = {paginated=True,include_inactive_campaigns=True,),+ "interest_performance": OutbrainEndpointConfig(+ name="interest_performance",+ kind="snapshot_report",+ path="/reports/marketers/{marketer_id}/interests",+ data_key="results",+ primary_keys=None,+ paginated=True,+ include_inactive_campaigns=True,+ ),}ENDPOINTS = tuple(OUTBRAIN_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.pyindex 7524a743e5f1..b7dc9ec8d56e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/outbrain/tests/test_outbrain.py@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_OutbrainSourceConfig,)from products.warehouse_sources.backend.temporal.data_imports.sources.outbrain.outbrain import (+ CAMPAIGN_PERIODIC_ROW_LIMIT,PAGE_SIZE,REPORT_ROW_LIMIT,OutbrainResumeConfig,@@ -82,6 +83,44 @@ class TestEntityStreams:assert [(row["id"], row["_campaign_id"]) for row in result.rows] == [("pl1", "c1")]assert result.requests[2].path == "/amplify/v0.1/campaigns/c1/promotedLinks"+ def test_segments_include_archived(self, mock_cache: mock.MagicMock) -> None:+ mock_cache.get.return_value = "tok"+ result = _driver().run(+ "segments",+ [+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),+ ScriptedResponse(json={"segments": [{"id": "s1"}], "count": 1}),+ ],+ )++ assert result.raised is None+ assert [(row["id"], row["_marketer_id"]) for row in result.rows] == [("s1", "m1")]+ assert result.requests[1].path == "/amplify/v0.1/marketers/m1/segments"+ assert result.requests[1].param("includeArchived") == "true"++ def test_campaign_locations_tag_targeted_and_excluded_rows(self, mock_cache: mock.MagicMock) -> None:+ mock_cache.get.return_value = "tok"+ result = _driver().run(+ "campaign_locations",+ [+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),+ ScriptedResponse(json={"campaigns": [{"id": "c1"}]}),+ ScriptedResponse(+ json={+ "geoLocations": [{"id": "us", "geoType": "Country"}],+ "excludedGeoLocations": [{"id": "al", "geoType": "Region", "parent": {"id": "us"}}],+ }+ ),+ ],+ )++ assert result.raised is None+ assert [(row["id"], row["_campaign_id"], row["_excluded"]) for row in result.rows] == [+ ("us", "c1", False),+ ("al", "c1", True),+ ]+ assert result.requests[2].path == "/amplify/v0.1/campaigns/c1/locations"+def test_resumes_from_saved_parent_index(self, mock_cache: mock.MagicMock) -> None:mock_cache.get.return_value = "tok"result = _driver().run(@@ -179,12 +218,64 @@ class TestReportStreams:assert "_date" not in result.rows[0]assert result.requests[1].param("breakdown") is None+ def test_campaign_periodic_report_flattens_campaigns_and_pages_until_no_campaign_is_full(+ self, mock_cache: mock.MagicMock+ ) -> None:+ mock_cache.get.return_value = "tok"+ full_campaign = [+ {"metadata": {"fromDate": f"2024-01-{day:02d}"}, "metrics": {"clicks": day}}+ for day in range(1, CAMPAIGN_PERIODIC_ROW_LIMIT + 1)+ ]+ result = _driver().run(+ "campaign_performance_daily",+ [+ ScriptedResponse(json={"marketers": [{"id": "m1"}]}),+ ScriptedResponse(+ json={+ "campaignResults": [+ {"campaignId": "c1", "results": full_campaign},+ {"campaignId": "c2", "results": [{"metadata": {"fromDate": "2024-01-05"}, "metrics": {}}]},+ {"results": [{"metadata": {"fromDate": "2024-01-05"}, "metrics": {}}]},+ ],+ "totalCampaigns": 3,+ }+ ),+ ScriptedResponse(+ json={+ "campaignResults": [+ {"campaignId": "c1", "results": [{"metadata": {"fromDate": "2024-02-01"}, "metrics": {}}]}+ ]+ }+ ),+ ],+ incremental_field="_date",+ db_incremental_field_last_value="2024-06-01",+ )++ assert result.raised is None+ assert [len(batch) for batch in result.items] == [CAMPAIGN_PERIODIC_ROW_LIMIT + 1, 1]+ assert result.rows[CAMPAIGN_PERIODIC_ROW_LIMIT] == {+ "metadata": {"fromDate": "2024-01-05"},+ "metrics": {},+ "_marketer_id": "m1",+ "_campaign_id": "c2",+ "_date": "2024-01-05",+ }+ assert result.rows[-1]["_campaign_id"] == "c1"+ assert result.paths[1:] == ["/amplify/v0.1/reports/marketers/m1/campaigns/periodic"] * 2+ assert [request.param("offset") for request in result.requests[1:]] == ["0", str(CAMPAIGN_PERIODIC_ROW_LIMIT)]+ assert all(request.param("limit") == str(CAMPAIGN_PERIODIC_ROW_LIMIT) for request in result.requests[1:])+ assert all(request.param("breakdown") == "daily" for request in result.requests[1:])+ assert all(request.param("includeArchivedCampaigns") == "true" for request in result.requests[1:])+ assert result.requests[1].param("from") == "2024-05-02"+@pytest.mark.parametrize("endpoint, report_path",[("promoted_link_performance", "promotedContent"),("publisher_performance", "publishers"),("section_performance", "sections"),+ ("interest_performance", "interests"),],)def test_breakdown_report_pages_with_offset(