:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 3 missing tables to the AdRoll source (#125006)
Tom Owers045fbd0fdb8c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 28af0ca9cbb2..67308cc4f0fa 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -193,10 +193,10 @@ Diffed against: <https://apidocs.nextroll.com/crud-api/reference.html>- [x] `report/campaign, report/ad, report/advertisable` — delivery metrics - impressions, clicks, spend; the whole point of an ads connector (high). `report/adgroup` does not exist on the CRUD API.- [x] `segments (GET /api/v1/advertisable/get_segments)` — audience segments targeted by adgroups (high). `segment/get` is a single-segment lookup, covered by the list endpoint's `get` representation.- [x] `organization (GET /api/v1/organization/get, get_accounts)` — lookup resolving the org/account EIDs that advertisables hang off (medium). `get_advertisables` already backs the `advertisables` table.-- [ ] `pixel (GET /api/v1/advertisable/get_pixel, /api/v1/pixel/get)` — conversion tracking pixel per advertisable (medium)-- [ ] `rules (GET /api/v1/pixel/get_rules, /api/v1/rule/get)` — conversion and retargeting rule definitions that decode segment membership (medium)+- [x] `pixel (GET /api/v1/advertisable/get_pixel, /api/v1/pixel/get)` — conversion tracking pixel per advertisable (medium)+- [x] `rules (GET /api/v1/pixel/get_rules, /api/v1/rule/get)` — conversion and retargeting rule definitions that decode segment membership (medium)- [ ] `invoice (GET /api/v1/invoice/get)` — billed spend reconciliation against reported campaign spend (medium)-- [ ] `organization users (GET /api/v1/organization/get_users)` — lookup for who owns/changed campaigns (low)+- [x] `organization users (GET /api/v1/organization/get_users)` — lookup for who owns/changed campaigns (low)- [ ] `product_feeds/get_products` — catalog products behind dynamic creative (low)- [ ] `marketplace_deals (GET /api/v1/marketplace_deals/get)` — programmatic deal terms for spend analysis (low)- [ ] `contextual_categories` — lookup decoding contextual targeting categories on adgroups (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.pyindex 91a148a137db..5efcdb141115 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py@@ -3,7 +3,10 @@ from urllib.parse import urlencodefrom posthog.dataclasses import frozen-from products.warehouse_sources.backend.temporal.data_imports.sources.adroll.settings import ADROLL_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.adroll.settings import (+ ADROLL_ENDPOINTS,+ AdRollEndpointConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,@@ -13,7 +16,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (SinglePagePaginator,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ EndpointResource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -26,6 +32,8 @@ MAX_RETRY_ATTEMPTS = 3# injects the parent advertisable EID into child rows as `_advertisable_eid` — the same key the# rows carried before the rest_source migration._ADVERTISABLE_PARENT = "advertisable"+# Second fan-out level for pixel-scoped endpoints; child rows carry the pixel EID as `_pixel_eid`.+_PIXEL_PARENT = "pixel"@frozen@@ -53,6 +61,27 @@ def _client_config(client_id: str, personal_access_token: str) -> ClientConfig:}+def _scoped_resource(name: str, config: AdRollEndpointConfig, parent: str, client_id: str) -> EndpointResource:+ return {+ "name": name,+ "endpoint": {+ # AdRoll scopes the per-parent lists with a query param, not a path segment; bind the+ # resolve param inside the query string.+ "path": f"{config.path}?{config.resolve_param}={{{config.resolve_param}}}",+ "params": {+ config.resolve_param: {+ "type": "resolve",+ "resource": parent,+ "field": "eid",+ },+ "apikey": client_id,+ **config.extra_params,+ },+ "data_selector": config.data_selector,+ },+ }++def adroll_source(client_id: str,personal_access_token: str,@@ -90,36 +119,30 @@ def adroll_source(if state is not None:resumable_source_manager.save_state(AdRollResumeConfig(fanout_state=state))+ parent_resources: list[str | EndpointResource] = [+ {+ "name": _ADVERTISABLE_PARENT,+ "endpoint": {+ "path": ADROLL_ENDPOINTS["advertisables"].path,+ "params": {"apikey": client_id},+ "data_selector": ADROLL_ENDPOINTS["advertisables"].data_selector,+ },+ },+ ]+ child_parent = _ADVERTISABLE_PARENT+ if config.pixel_scoped:+ # With two dependent levels the framework resumes neither, so a retried run re-fetches+ # every advertisable's pixel and rules.+ parent_resources.append(+ _scoped_resource(_PIXEL_PARENT, ADROLL_ENDPOINTS["pixels"], _ADVERTISABLE_PARENT, client_id)+ )+ child_parent = _PIXEL_PARENT+rest_config = {"client": _client_config(client_id, personal_access_token),"resources": [- {- "name": _ADVERTISABLE_PARENT,- "endpoint": {- "path": ADROLL_ENDPOINTS["advertisables"].path,- "params": {"apikey": client_id},- "data_selector": ADROLL_ENDPOINTS["advertisables"].data_selector,- },- },- {- "name": endpoint,- "endpoint": {- # AdRoll scopes the per-advertisable lists with a query param, not a path- # segment; bind the resolve param inside the query string.- "path": f"{config.path}?{config.resolve_param}={{{config.resolve_param}}}",- "params": {- config.resolve_param: {- "type": "resolve",- "resource": _ADVERTISABLE_PARENT,- "field": "eid",- },- "apikey": client_id,- **config.extra_params,- },- "data_selector": config.data_selector,- },- "include_from_parent": ["eid"],- },+ *parent_resources,+ {**_scoped_resource(endpoint, config, child_parent, client_id), "include_from_parent": ["eid"]},],}resources = {@@ -135,6 +158,8 @@ def adroll_source(}# An advertisable without an EID can't scope a child request — skip it, as before.resources[_ADVERTISABLE_PARENT].add_filter(lambda item: bool(item.get("eid")))+ if config.pixel_scoped:+ resources[_PIXEL_PARENT].add_filter(lambda item: bool(item.get("eid")))resource = resources[endpoint]return SourceResponse(@@ -144,6 +169,7 @@ def adroll_source(partition_count=1,partition_size=1,sort_mode="asc",+ supports_resume=not config.pixel_scoped,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.pyindex e2b43c6a3195..0f2f0d1b1c10 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py@@ -118,6 +118,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"mobile": "Mobile extension data for the segment.",},},+ "users": {+ "description": "A user who can see your AdRoll organization, with their organization role and the advertisables they can access.",+ "docs_url": _DOCS_URL,+ "columns": {+ "eid": "Unique encoded identifier (EID) for the user.",+ "id": "Legacy identifier of the user. Deprecated in favor of eid.",+ "username": "Login name of the user.",+ "email": "Email address of the user.",+ "email_verified": "Whether the user has verified their email address.",+ "name": "Full name of the user.",+ "first_name": "First name of the user.",+ "last_name": "Last name of the user.",+ "organization": "EID of the organization the user belongs to.",+ "organization_role": "Role of the user in the organization, either admin or user.",+ "role": "User type as AdRoll records it, usually user.",+ "advertisables": "Advertisables the user can access.",+ "locale": "Language and locale preference of the user.",+ "default_timezone": "Time zone the user picked for reporting and billing.",+ "created_date": "Date the user was created.",+ },+ },+ "pixels": {+ "description": "The active AdRoll tracking pixel of an advertisable, which collects site visits and conversions.",+ "docs_url": _DOCS_URL,+ "columns": {+ "eid": "Unique encoded identifier (EID) for the pixel.",+ "_advertisable_eid": "EID of the parent advertisable the pixel belongs to (added during sync).",+ "status": "Status of the pixel.",+ "code": "HTML and JavaScript snippet that the advertiser places on their site.",+ "is_consistent": "Whether the pixel is consistent.",+ "placed_code_date": "Last time the pixel code was placed.",+ "dropped_code_date": "Last time the pixel code was dropped.",+ "tag_source": "Source of the pixel tag.",+ },+ },+ "rules": {+ "description": "A rule on an advertisable's active pixel that matches visited URLs and puts visitors into segments or counts conversions.",+ "docs_url": _DOCS_URL,+ "columns": {+ "eid": "Unique encoded identifier (EID) for the rule.",+ "_pixel_eid": "EID of the pixel the rule belongs to (added during sync).",+ "name": "Name of the rule.",+ "display_name": "Name of the rule shown in the AdRoll UI.",+ "pattern": "URL pattern that the rule matches.",+ "type": "Type of the rule, one of c (conversion), s (segment), p (prospect), b (cart).",+ "source": "Source of the users the rule matches (JavaScript pixel, mobile or CRM data).",+ "order": "Priority of the rule among the pixel's rules. A lower number has a higher priority.",+ "segments": "Segments attached to the rule.",+ },+ },"advertisable_reports": {"description": "Lifetime delivery metrics per advertisable, as returned by the CRUD reporting API in its entity format.","docs_url": _DOCS_URL,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.pyindex e1274de39036..164fd7093b7c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py@@ -15,6 +15,8 @@ class AdRollEndpointConfig:# Fan-out parent (campaigns/ads are fetched per advertisable EID).advertisable_scoped: bool = Falseparent_key: Optional[str] = None+ # Fan-out through the advertisable's active pixel: advertisable -> get_pixel -> this endpoint.+ pixel_scoped: bool = False# Query param the parent advertisable EID binds to. The reporting endpoints filter on a list# of advertisables (`advertisables`); the entity endpoints scope to one (`advertisable`).resolve_param: str = "advertisable"@@ -26,6 +28,8 @@ class AdRollEndpointConfig:def __post_init__(self) -> None:if self.advertisable_scoped and self.parent_key is None:raise ValueError(f"advertisable_scoped endpoint '{self.name}' must define parent_key")+ if self.pixel_scoped and not self.advertisable_scoped:+ raise ValueError(f"pixel_scoped endpoint '{self.name}' must also be advertisable_scoped")# AdRoll's default quota is only 100 API requests per day, so every endpoint is either a single@@ -46,6 +50,11 @@ ADROLL_ENDPOINTS: dict[str, AdRollEndpointConfig] = {name="advertisables",path="/api/v1/organization/get_advertisables",),+ # Omitting `per_page` opts out of pagination, so the response carries every user.+ "users": AdRollEndpointConfig(+ name="users",+ path="/api/v1/organization/get_users",+ ),"campaigns": AdRollEndpointConfig(name="campaigns",path="/api/v1/campaign/get_all",@@ -74,6 +83,23 @@ ADROLL_ENDPOINTS: dict[str, AdRollEndpointConfig] = {advertisable_scoped=True,parent_key="_advertisable_eid",),+ # The advertisable's active pixel; `results` is a single object, not a list.+ "pixels": AdRollEndpointConfig(+ name="pixels",+ path="/api/v1/advertisable/get_pixel",+ advertisable_scoped=True,+ parent_key="_advertisable_eid",+ ),+ # Conversion, segment, prospect and cart rules of the advertisable's active pixel. Costs two+ # requests per advertisable (get_pixel, then get_rules) against the daily quota.+ "rules": AdRollEndpointConfig(+ name="rules",+ path="/api/v1/pixel/get_rules",+ advertisable_scoped=True,+ pixel_scoped=True,+ parent_key="_pixel_eid",+ resolve_param="pixel",+ ),# `data_format=entity` is the only response shape the CRUD API documents: one row per entity# carrying its delivery metrics. Per-day breakdowns need the GraphQL Reporting API."advertisable_reports": AdRollEndpointConfig(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.pyindex 2329d7d0aa6d..b47f32d6195e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py@@ -16,11 +16,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.adroll import AdRollSourceConfigCAMPAIGNS_PATH = ADROLL_ENDPOINTS["campaigns"].path+ADVERTISABLES_PATH = ADROLL_ENDPOINTS["advertisables"].path+PIXEL_PATH = ADROLL_ENDPOINTS["pixels"].path+RULES_PATH = ADROLL_ENDPOINTS["rules"].pathdef _driver() -> SourceDriver:@@ -86,6 +90,79 @@ class TestRows:assert len(result.requests) == 1+class TestPixelFanOut:+ @pytest.mark.parametrize(+ "endpoint, pixel_answers, rules_answers, expected_rows, expected_rule_pixels",+ [+ (+ "pixels",+ [+ ScriptedResponse(json={"results": {"eid": "PIX1", "status": "active"}}),+ ScriptedResponse(json={"results": {"eid": "PIX2", "status": "inactive"}}),+ ],+ [],+ [+ {"eid": "PIX1", "status": "active", "_advertisable_eid": "ADV1"},+ {"eid": "PIX2", "status": "inactive", "_advertisable_eid": "ADV2"},+ ],+ [],+ ),+ (+ "rules",+ [+ ScriptedResponse(json={"results": {"eid": "PIX1"}}),+ ScriptedResponse(json={"results": {"eid": "PIX2"}}),+ ],+ [_page([{"eid": "R1", "type": "c"}, {"eid": "R2", "type": "s"}]), _page([{"eid": "R3", "type": "b"}])],+ [+ {"eid": "R1", "type": "c", "_pixel_eid": "PIX1"},+ {"eid": "R2", "type": "s", "_pixel_eid": "PIX1"},+ {"eid": "R3", "type": "b", "_pixel_eid": "PIX2"},+ ],+ ["PIX1", "PIX2"],+ ),+ (+ "rules",+ [ScriptedResponse(json={"results": None}), ScriptedResponse(json={"results": {"eid": "PIX2"}})],+ [_page([{"eid": "R3", "type": "b"}])],+ [{"eid": "R3", "type": "b", "_pixel_eid": "PIX2"}],+ ["PIX2"],+ ),+ (+ "rules",+ [ScriptedResponse(json={"results": {}}), ScriptedResponse(json={"results": {"status": "inactive"}})],+ [],+ [],+ [],+ ),+ ],+ )+ def test_fan_out_through_advertisable_pixel(+ self,+ endpoint: str,+ pixel_answers: list[ScriptedResponse],+ rules_answers: list[ScriptedResponse],+ expected_rows: list[dict[str, Any]],+ expected_rule_pixels: list[str],+ ) -> None:+ result = _driver().run(+ endpoint,+ route(+ {+ ADVERTISABLES_PATH: [_page([{"eid": "ADV1"}, {"eid": "ADV2"}])],+ PIXEL_PATH: pixel_answers,+ RULES_PATH: rules_answers,+ }+ ),+ )++ assert result.raised is None+ assert result.rows == expected_rows+ assert [r.param("advertisable") for r in result.requests if r.path == PIXEL_PATH] == ["ADV1", "ADV2"]+ assert [r.param("pixel") for r in result.requests if r.path == RULES_PATH] == expected_rule_pixels+ assert all(r.param("apikey") == "cid" for r in result.requests)++class TestFanOutResume:def test_resume_skips_completed_advertisables(self) -> None:result = _driver().run(@@ -124,3 +201,4 @@ class TestAdRollSourceResponse:assert result.response.sort_mode == "asc"assert result.response.partition_mode is Noneassert result.response.partition_keys is None+ assert result.response.supports_resume is (not config.pixel_scoped)