:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse): add x ads lead forms table (#120208)
Tom Owers3c31a02dfc5c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex c87e8bd6d012..e07daa40099b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -12721,6 +12721,16 @@ Diffed against: <https://docs.x.com/openapi.json>Note: Compared every GET operation in the current official X API specification with the seven source paths. Post and list engagement endpoints need requests for each parent; user lookup accepts batches of IDs. Analytics and direct messages require user authorization, while the current source accepts an application bearer token. Excluded write operations, uploads, webhook administration, compliance jobs, and continuous streams; plan access was not tested.+## TwitterAds — gaps++Today (8): `campaign_stats`, `campaigns`, `funding_instruments`, `lead_forms`, `line_item_stats`, `line_items`, `media_creatives`, `promoted_tweets`++Diffed against: <https://docs.x.com/x-ads-api/lead-generation/reference>++- [x] `Lead forms (GET /12/accounts/{account_id}/lead_forms)` — list native lead forms used by Lead Gen campaigns, with their fields and privacy policy (medium)++Note: Added from the X API changelog entry for the Lead Generation API, not from a full sweep of the X Ads API. Leads themselves are only available through asynchronous export jobs, so they are not covered here.+## TyntecSMS — adequateToday (4): `Contacts`, `MessageStatus`, `PhoneNumbers`, `PhoneRegistrations`diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/canonical_descriptions.pyindex 9a3bd5a4eed1..10daeb898d57 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/canonical_descriptions.py@@ -45,6 +45,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"docs_url": "https://docs.x.com/x-ads-api/campaign-management/reference#media-creatives","columns": {"id": "Media creative identifier.", "line_item_id": "Line item that promotes the creative."},},+ "lead_forms": {+ "description": "A native lead form that Lead Gen ads attach to a card to collect leads.",+ "docs_url": "https://docs.x.com/x-ads-api/lead-generation/reference#lead-forms",+ "columns": {+ "id": "Lead form identifier (UUID).",+ "name": "Form name.",+ "purpose": "What the collected leads are used for.",+ "fields": "Fields the lead fills in, each with a key such as EMAIL and whether it is optional.",+ "privacy_policy": "Privacy policy URL and company name of the advertiser collecting the leads.",+ "deleted": "Whether the lead form has been archived.",+ },+ },**{table: {"description": f"Daily unsegmented X advertising metrics for a {entity}, separated by placement.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/settings.pyindex 4c649277e304..a05154311167 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/settings.py@@ -1,5 +1,8 @@API_VERSION = "12"-ENTITY_TABLES = ("campaigns", "line_items", "promoted_tweets", "funding_instruments", "media_creatives")+ENTITY_TABLES = ("campaigns", "line_items", "promoted_tweets", "funding_instruments", "media_creatives", "lead_forms")+# Lead forms cap `count` at 200 and list newest first with no `sort_by`.+PAGE_SIZES = {"lead_forms": 200}+UNSORTABLE_TABLES = ("media_creatives", "lead_forms")STATS_TABLES = {"campaign_stats": "CAMPAIGN", "line_item_stats": "LINE_ITEM"}PLACEMENTS = ("ALL_ON_TWITTER", "SPOTLIGHT", "TREND")LOOKBACK_SECONDS = 3 * 24 * 60 * 60diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/tests/test_twitter_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/tests/test_twitter_ads.pyindex a4946cf024de..73835a88b000 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/tests/test_twitter_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/tests/test_twitter_ads.py@@ -87,6 +87,26 @@ def test_signing_pagination_and_resume(client: TwitterAdsClient, manager: MagicMsend.assert_not_called()+ "table,count,sort_by",+ [+ ("campaigns", "1000", ["created_at-asc"]),+ ("media_creatives", "1000", None),+ ("lead_forms", "200", None),+ ],+)+def test_entity_page_request_respects_per_table_limits(+ client: TwitterAdsClient, manager: MagicMock, table: str, count: str, sort_by: list[str] | None+) -> None:+ with patch.object(client.session, "send", return_value=response({"data": [], "next_cursor": None})) as send:+ list(sync_items(twitter_ads_source(client, "account", table, manager, None)))+ url = urlparse(request_url(send.call_args.args[0]))+ params = parse_qs(url.query)+ assert url.path == f"/12/accounts/account/{table}"+ assert params["count"] == [count]+ assert params.get("sort_by") == sort_by++@pytest.mark.parametrize("table", ["campaign_stats", "line_item_stats"])@pytest.mark.parametrize("incremental_since", [None, date(2025, 10, 30)])def test_stats_limits_daily_rows_currency_and_dst(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.pyindex 1140f44a636d..cb4e079d7761 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/twitter_ads/twitter_ads.py@@ -21,10 +21,12 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.twitter_adMAX_STATS_BACKFILL_DAYS,MISSING_APP,MISSING_INTEGRATION,+ PAGE_SIZES,PARTITION_KEYS,PLACEMENTS,PRIMARY_KEYS,STATS_TABLES,+ UNSORTABLE_TABLES,)@@ -118,9 +120,10 @@ class TwitterAdsClient:return response.json()def pages(self, path: str, cursor: str | None = None) -> Iterator[dict[str, Any]]:+ table = path.rsplit("/", 1)[-1]while True:- params: dict[str, str | int] = {"count": 1000, "with_deleted": "true"}- if path.rsplit("/", 1)[-1] != "media_creatives":+ params: dict[str, str | int] = {"count": PAGE_SIZES.get(table, 1000), "with_deleted": "true"}+ if table not in UNSORTABLE_TABLES:params["sort_by"] = "created_at-asc"if cursor:params["cursor"] = cursor