Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 3 missing tables to the AdRoll source (#125006)

Tom Owers045fbd0fdb8c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 28af0ca9cbb2..67308cc4f0fa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -193,10 +193,10 @@ Diffed against: <https://apidocs.nextroll.com/crud-api/reference.html>
- [x] `report/campaign, report/ad, report/advertisable` — delivery metrics - impressions, clicks, spend; the whole point of an ads connector (high). `report/adgroup` does not exist on the CRUD API.
- [x] `segments (GET /api/v1/advertisable/get_segments)` — audience segments targeted by adgroups (high). `segment/get` is a single-segment lookup, covered by the list endpoint's `get` representation.
- [x] `organization (GET /api/v1/organization/get, get_accounts)` — lookup resolving the org/account EIDs that advertisables hang off (medium). `get_advertisables` already backs the `advertisables` table.
-- [ ] `pixel (GET /api/v1/advertisable/get_pixel, /api/v1/pixel/get)` — conversion tracking pixel per advertisable (medium)
-- [ ] `rules (GET /api/v1/pixel/get_rules, /api/v1/rule/get)` — conversion and retargeting rule definitions that decode segment membership (medium)
+- [x] `pixel (GET /api/v1/advertisable/get_pixel, /api/v1/pixel/get)` — conversion tracking pixel per advertisable (medium)
+- [x] `rules (GET /api/v1/pixel/get_rules, /api/v1/rule/get)` — conversion and retargeting rule definitions that decode segment membership (medium)
- [ ] `invoice (GET /api/v1/invoice/get)` — billed spend reconciliation against reported campaign spend (medium)
-- [ ] `organization users (GET /api/v1/organization/get_users)` — lookup for who owns/changed campaigns (low)
+- [x] `organization users (GET /api/v1/organization/get_users)` — lookup for who owns/changed campaigns (low)
- [ ] `product_feeds/get_products` — catalog products behind dynamic creative (low)
- [ ] `marketplace_deals (GET /api/v1/marketplace_deals/get)` — programmatic deal terms for spend analysis (low)
- [ ] `contextual_categories` — lookup decoding contextual targeting categories on adgroups (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py
index 91a148a137db..5efcdb141115 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/adroll.py
@@ -3,7 +3,10 @@ from urllib.parse import urlencode
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.adroll.settings import ADROLL_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.adroll.settings import (
+ ADROLL_ENDPOINTS,
+ AdRollEndpointConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
@@ -13,7 +16,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
SinglePagePaginator,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ EndpointResource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -26,6 +32,8 @@ MAX_RETRY_ATTEMPTS = 3
# injects the parent advertisable EID into child rows as `_advertisable_eid` — the same key the
# rows carried before the rest_source migration.
_ADVERTISABLE_PARENT = "advertisable"
+# Second fan-out level for pixel-scoped endpoints; child rows carry the pixel EID as `_pixel_eid`.
+_PIXEL_PARENT = "pixel"
@frozen
@@ -53,6 +61,27 @@ def _client_config(client_id: str, personal_access_token: str) -> ClientConfig:
}
+def _scoped_resource(name: str, config: AdRollEndpointConfig, parent: str, client_id: str) -> EndpointResource:
+ return {
+ "name": name,
+ "endpoint": {
+ # AdRoll scopes the per-parent lists with a query param, not a path segment; bind the
+ # resolve param inside the query string.
+ "path": f"{config.path}?{config.resolve_param}={{{config.resolve_param}}}",
+ "params": {
+ config.resolve_param: {
+ "type": "resolve",
+ "resource": parent,
+ "field": "eid",
+ },
+ "apikey": client_id,
+ **config.extra_params,
+ },
+ "data_selector": config.data_selector,
+ },
+ }
+
+
def adroll_source(
client_id: str,
personal_access_token: str,
@@ -90,36 +119,30 @@ def adroll_source(
if state is not None:
resumable_source_manager.save_state(AdRollResumeConfig(fanout_state=state))
+ parent_resources: list[str | EndpointResource] = [
+ {
+ "name": _ADVERTISABLE_PARENT,
+ "endpoint": {
+ "path": ADROLL_ENDPOINTS["advertisables"].path,
+ "params": {"apikey": client_id},
+ "data_selector": ADROLL_ENDPOINTS["advertisables"].data_selector,
+ },
+ },
+ ]
+ child_parent = _ADVERTISABLE_PARENT
+ if config.pixel_scoped:
+ # With two dependent levels the framework resumes neither, so a retried run re-fetches
+ # every advertisable's pixel and rules.
+ parent_resources.append(
+ _scoped_resource(_PIXEL_PARENT, ADROLL_ENDPOINTS["pixels"], _ADVERTISABLE_PARENT, client_id)
+ )
+ child_parent = _PIXEL_PARENT
+
rest_config = {
"client": _client_config(client_id, personal_access_token),
"resources": [
- {
- "name": _ADVERTISABLE_PARENT,
- "endpoint": {
- "path": ADROLL_ENDPOINTS["advertisables"].path,
- "params": {"apikey": client_id},
- "data_selector": ADROLL_ENDPOINTS["advertisables"].data_selector,
- },
- },
- {
- "name": endpoint,
- "endpoint": {
- # AdRoll scopes the per-advertisable lists with a query param, not a path
- # segment; bind the resolve param inside the query string.
- "path": f"{config.path}?{config.resolve_param}={{{config.resolve_param}}}",
- "params": {
- config.resolve_param: {
- "type": "resolve",
- "resource": _ADVERTISABLE_PARENT,
- "field": "eid",
- },
- "apikey": client_id,
- **config.extra_params,
- },
- "data_selector": config.data_selector,
- },
- "include_from_parent": ["eid"],
- },
+ *parent_resources,
+ {**_scoped_resource(endpoint, config, child_parent, client_id), "include_from_parent": ["eid"]},
],
}
resources = {
@@ -135,6 +158,8 @@ def adroll_source(
}
# An advertisable without an EID can't scope a child request — skip it, as before.
resources[_ADVERTISABLE_PARENT].add_filter(lambda item: bool(item.get("eid")))
+ if config.pixel_scoped:
+ resources[_PIXEL_PARENT].add_filter(lambda item: bool(item.get("eid")))
resource = resources[endpoint]
return SourceResponse(
@@ -144,6 +169,7 @@ def adroll_source(
partition_count=1,
partition_size=1,
sort_mode="asc",
+ supports_resume=not config.pixel_scoped,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py
index e2b43c6a3195..0f2f0d1b1c10 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/canonical_descriptions.py
@@ -118,6 +118,56 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"mobile": "Mobile extension data for the segment.",
},
},
+ "users": {
+ "description": "A user who can see your AdRoll organization, with their organization role and the advertisables they can access.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "eid": "Unique encoded identifier (EID) for the user.",
+ "id": "Legacy identifier of the user. Deprecated in favor of eid.",
+ "username": "Login name of the user.",
+ "email": "Email address of the user.",
+ "email_verified": "Whether the user has verified their email address.",
+ "name": "Full name of the user.",
+ "first_name": "First name of the user.",
+ "last_name": "Last name of the user.",
+ "organization": "EID of the organization the user belongs to.",
+ "organization_role": "Role of the user in the organization, either admin or user.",
+ "role": "User type as AdRoll records it, usually user.",
+ "advertisables": "Advertisables the user can access.",
+ "locale": "Language and locale preference of the user.",
+ "default_timezone": "Time zone the user picked for reporting and billing.",
+ "created_date": "Date the user was created.",
+ },
+ },
+ "pixels": {
+ "description": "The active AdRoll tracking pixel of an advertisable, which collects site visits and conversions.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "eid": "Unique encoded identifier (EID) for the pixel.",
+ "_advertisable_eid": "EID of the parent advertisable the pixel belongs to (added during sync).",
+ "status": "Status of the pixel.",
+ "code": "HTML and JavaScript snippet that the advertiser places on their site.",
+ "is_consistent": "Whether the pixel is consistent.",
+ "placed_code_date": "Last time the pixel code was placed.",
+ "dropped_code_date": "Last time the pixel code was dropped.",
+ "tag_source": "Source of the pixel tag.",
+ },
+ },
+ "rules": {
+ "description": "A rule on an advertisable's active pixel that matches visited URLs and puts visitors into segments or counts conversions.",
+ "docs_url": _DOCS_URL,
+ "columns": {
+ "eid": "Unique encoded identifier (EID) for the rule.",
+ "_pixel_eid": "EID of the pixel the rule belongs to (added during sync).",
+ "name": "Name of the rule.",
+ "display_name": "Name of the rule shown in the AdRoll UI.",
+ "pattern": "URL pattern that the rule matches.",
+ "type": "Type of the rule, one of c (conversion), s (segment), p (prospect), b (cart).",
+ "source": "Source of the users the rule matches (JavaScript pixel, mobile or CRM data).",
+ "order": "Priority of the rule among the pixel's rules. A lower number has a higher priority.",
+ "segments": "Segments attached to the rule.",
+ },
+ },
"advertisable_reports": {
"description": "Lifetime delivery metrics per advertisable, as returned by the CRUD reporting API in its entity format.",
"docs_url": _DOCS_URL,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py
index e1274de39036..164fd7093b7c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/settings.py
@@ -15,6 +15,8 @@ class AdRollEndpointConfig:
# Fan-out parent (campaigns/ads are fetched per advertisable EID).
advertisable_scoped: bool = False
parent_key: Optional[str] = None
+ # Fan-out through the advertisable's active pixel: advertisable -> get_pixel -> this endpoint.
+ pixel_scoped: bool = False
# Query param the parent advertisable EID binds to. The reporting endpoints filter on a list
# of advertisables (`advertisables`); the entity endpoints scope to one (`advertisable`).
resolve_param: str = "advertisable"
@@ -26,6 +28,8 @@ class AdRollEndpointConfig:
def __post_init__(self) -> None:
if self.advertisable_scoped and self.parent_key is None:
raise ValueError(f"advertisable_scoped endpoint '{self.name}' must define parent_key")
+ if self.pixel_scoped and not self.advertisable_scoped:
+ raise ValueError(f"pixel_scoped endpoint '{self.name}' must also be advertisable_scoped")
# AdRoll's default quota is only 100 API requests per day, so every endpoint is either a single
@@ -46,6 +50,11 @@ ADROLL_ENDPOINTS: dict[str, AdRollEndpointConfig] = {
name="advertisables",
path="/api/v1/organization/get_advertisables",
),
+ # Omitting `per_page` opts out of pagination, so the response carries every user.
+ "users": AdRollEndpointConfig(
+ name="users",
+ path="/api/v1/organization/get_users",
+ ),
"campaigns": AdRollEndpointConfig(
name="campaigns",
path="/api/v1/campaign/get_all",
@@ -74,6 +83,23 @@ ADROLL_ENDPOINTS: dict[str, AdRollEndpointConfig] = {
advertisable_scoped=True,
parent_key="_advertisable_eid",
),
+ # The advertisable's active pixel; `results` is a single object, not a list.
+ "pixels": AdRollEndpointConfig(
+ name="pixels",
+ path="/api/v1/advertisable/get_pixel",
+ advertisable_scoped=True,
+ parent_key="_advertisable_eid",
+ ),
+ # Conversion, segment, prospect and cart rules of the advertisable's active pixel. Costs two
+ # requests per advertisable (get_pixel, then get_rules) against the daily quota.
+ "rules": AdRollEndpointConfig(
+ name="rules",
+ path="/api/v1/pixel/get_rules",
+ advertisable_scoped=True,
+ pixel_scoped=True,
+ parent_key="_pixel_eid",
+ resolve_param="pixel",
+ ),
# `data_format=entity` is the only response shape the CRUD API documents: one row per entity
# carrying its delivery metrics. Per-day breakdowns need the GraphQL Reporting API.
"advertisable_reports": AdRollEndpointConfig(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py
index 2329d7d0aa6d..b47f32d6195e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/adroll/tests/test_adroll.py
@@ -16,11 +16,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.adroll import AdRollSourceConfig
CAMPAIGNS_PATH = ADROLL_ENDPOINTS["campaigns"].path
+ADVERTISABLES_PATH = ADROLL_ENDPOINTS["advertisables"].path
+PIXEL_PATH = ADROLL_ENDPOINTS["pixels"].path
+RULES_PATH = ADROLL_ENDPOINTS["rules"].path
def _driver() -> SourceDriver:
@@ -86,6 +90,79 @@ class TestRows:
assert len(result.requests) == 1
+class TestPixelFanOut:
+ @pytest.mark.parametrize(
+ "endpoint, pixel_answers, rules_answers, expected_rows, expected_rule_pixels",
+ [
+ (
+ "pixels",
+ [
+ ScriptedResponse(json={"results": {"eid": "PIX1", "status": "active"}}),
+ ScriptedResponse(json={"results": {"eid": "PIX2", "status": "inactive"}}),
+ ],
+ [],
+ [
+ {"eid": "PIX1", "status": "active", "_advertisable_eid": "ADV1"},
+ {"eid": "PIX2", "status": "inactive", "_advertisable_eid": "ADV2"},
+ ],
+ [],
+ ),
+ (
+ "rules",
+ [
+ ScriptedResponse(json={"results": {"eid": "PIX1"}}),
+ ScriptedResponse(json={"results": {"eid": "PIX2"}}),
+ ],
+ [_page([{"eid": "R1", "type": "c"}, {"eid": "R2", "type": "s"}]), _page([{"eid": "R3", "type": "b"}])],
+ [
+ {"eid": "R1", "type": "c", "_pixel_eid": "PIX1"},
+ {"eid": "R2", "type": "s", "_pixel_eid": "PIX1"},
+ {"eid": "R3", "type": "b", "_pixel_eid": "PIX2"},
+ ],
+ ["PIX1", "PIX2"],
+ ),
+ (
+ "rules",
+ [ScriptedResponse(json={"results": None}), ScriptedResponse(json={"results": {"eid": "PIX2"}})],
+ [_page([{"eid": "R3", "type": "b"}])],
+ [{"eid": "R3", "type": "b", "_pixel_eid": "PIX2"}],
+ ["PIX2"],
+ ),
+ (
+ "rules",
+ [ScriptedResponse(json={"results": {}}), ScriptedResponse(json={"results": {"status": "inactive"}})],
+ [],
+ [],
+ [],
+ ),
+ ],
+ )
+ def test_fan_out_through_advertisable_pixel(
+ self,
+ endpoint: str,
+ pixel_answers: list[ScriptedResponse],
+ rules_answers: list[ScriptedResponse],
+ expected_rows: list[dict[str, Any]],
+ expected_rule_pixels: list[str],
+ ) -> None:
+ result = _driver().run(
+ endpoint,
+ route(
+ {
+ ADVERTISABLES_PATH: [_page([{"eid": "ADV1"}, {"eid": "ADV2"}])],
+ PIXEL_PATH: pixel_answers,
+ RULES_PATH: rules_answers,
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == expected_rows
+ assert [r.param("advertisable") for r in result.requests if r.path == PIXEL_PATH] == ["ADV1", "ADV2"]
+ assert [r.param("pixel") for r in result.requests if r.path == RULES_PATH] == expected_rule_pixels
+ assert all(r.param("apikey") == "cid" for r in result.requests)
+
+
class TestFanOutResume:
def test_resume_skips_completed_advertisables(self) -> None:
result = _driver().run(
@@ -124,3 +201,4 @@ class TestAdRollSourceResponse:
assert result.response.sort_mode == "asc"
assert result.response.partition_mode is None
assert result.response.partition_keys is None
+ assert result.response.supports_resume is (not config.pixel_scoped)