:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): let google ads users skip removed keywords (#118963)
posthog[bot]a44612d396d3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.pyindex c1922f961944..0da47de898da 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py@@ -14,5 +14,6 @@ class GoogleAdsIsMccAccountConfig(config.Config):class GoogleAdsSourceConfig(config.Config):customer_id: strgoogle_ads_integration_id: int = config.value(converter=config.str_to_int)- is_mcc_account: GoogleAdsIsMccAccountConfig | None = Nonestart_date: str | None = None+ skip_removed_keywords: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)+ is_mcc_account: GoogleAdsIsMccAccountConfig | None = Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.pyindex 973c972b7c95..0c26ace20e70 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py@@ -449,6 +449,7 @@ class GoogleAdsTable(Table[GoogleAdsColumn]):description: str | None,partition_keys: list[str] | None = None,extra_where: str | None = None,+ removed_filter: str | None = None,**kwargs,):self.requires_filter = requires_filter@@ -457,6 +458,7 @@ class GoogleAdsTable(Table[GoogleAdsColumn]):self.description = descriptionself.partition_keys = [pkey.replace(".", "_") for pkey in partition_keys] if partition_keys else Noneself.extra_where = extra_where+ self.removed_filter = removed_filtersuper().__init__(*args, **kwargs)@@ -493,6 +495,7 @@ def get_schemas(config: GoogleAdsSourceConfigUnion, team_id: int, api_version: srequires_filter = resource_contents.get("filter_field_names", None) is not Noneprimary_key = typing.cast(list[str], resource_contents.get("primary_key", []))extra_where = typing.cast(str | None, resource_contents.get("extra_where", None))+ removed_filter = typing.cast(str | None, resource_contents.get("removed_filter", None))partition_keys = typing.cast(list[str] | None, resource_contents.get("partition_keys", None))should_sync_default = resource_contents.get("should_sync_default", True)@@ -527,6 +530,7 @@ def get_schemas(config: GoogleAdsSourceConfigUnion, team_id: int, api_version: srequires_filter=requires_filter,primary_key=primary_key,extra_where=extra_where,+ removed_filter=removed_filter,partition_keys=partition_keys,columns=columns,parents=None,@@ -642,6 +646,7 @@ def google_ads_source(db_incremental_field_last_value_before_lookback: typing.Any = None,history_start: typing.Any = None,requested_start: str | None = None,+ skip_removed: bool = False,) -> SourceResponse:"""A data warehouse Google Ads source.@@ -720,6 +725,8 @@ def google_ads_source(conditions.append(f"{incremental_field} < {upper_literal}")if table.extra_where:conditions.append(table.extra_where)+ if skip_removed and table.removed_filter:+ conditions.append(table.removed_filter)if conditions:query += " WHERE " + " AND ".join(conditions)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.pyindex 27764826936d..fddc0411b8b6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py@@ -527,6 +527,7 @@ RESOURCE_SCHEMAS: dict[str, dict[str, Any]] = {},"keyword": {"resource_name": "keyword_view",+ "removed_filter": "ad_group_criterion.status != 'REMOVED' AND ad_group.status != 'REMOVED' AND campaign.status != 'REMOVED'","primary_key": ["ad_group_criterion.criterion_id", "ad_group.id", "campaign.id", "customer.id"],"field_names": ["ad_group_criterion.criterion_id",@@ -653,6 +654,7 @@ RESOURCE_SCHEMAS: dict[str, dict[str, Any]] = {},"ad_group_criterion": {"resource_name": "ad_group_criterion",+ "removed_filter": "ad_group_criterion.status != 'REMOVED' AND ad_group.status != 'REMOVED' AND campaign.status != 'REMOVED'","primary_key": ["ad_group_criterion.criterion_id", "ad_group.id", "campaign.id", "customer.id"],"should_sync_default": False,"description": "Every targeting criterion at ad group level, including negative keywords and other exclusions. Unlike the keyword table (which is backed by keyword_view and only returns positive, servable keywords), this reaches negative and non-keyword criteria. Filter on ad_group_criterion.negative to isolate exclusions.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.pyindex ed7e0cf48e62..02b22e48839f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py@@ -25,6 +25,9 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfigType,SourceFieldOauthAccountSelectConfig,SourceFieldOauthConfig,+ SourceFieldSelectConfig,+ SourceFieldSelectConfigConverter,+ SourceFieldSelectConfigOption,SourceFieldSwitchGroupConfig,SuggestedTable,)@@ -303,6 +306,7 @@ class GoogleAdsSource(db_incremental_field_last_value_before_lookback=inputs.db_incremental_field_last_value_before_lookback,history_start=inputs.history_start,requested_start=config.start_date if isinstance(config, GoogleAdsSourceConfig) else None,+ skip_removed=isinstance(config, GoogleAdsSourceConfig) and config.skip_removed_keywords,)@property@@ -349,6 +353,23 @@ class GoogleAdsSource(placeholder="2020-01-01",secret=False,),+ SourceFieldSelectConfig(+ name="skip_removed_keywords",+ label="Skip removed keywords?",+ caption=(+ "Leave out keywords that are removed, or that sit in a removed ad group or "+ "campaign, from the keyword and ad group criterion tables. Every sync reads "+ "these tables in full, so removed keywords count towards your billed row "+ "usage on each sync."+ ),+ required=True,+ defaultValue="false",+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,+ options=[+ SourceFieldSelectConfigOption(label="Yes", value="true"),+ SourceFieldSelectConfigOption(label="No", value="false"),+ ],+ ),SourceFieldSwitchGroupConfig(name="is_mcc_account",label="Using MCC account?",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.pyindex af0f4cbdb05c..fa86ee1210b2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py@@ -1849,6 +1849,34 @@ class TestGoogleAdsQueryConstruction:"AND metrics.impressions > 0 ORDER BY segments.date ASC LIMIT 1")+ @pytest.mark.parametrize(+ "skip_removed,expected_query",+ [+ (False, "SELECT ad_group_criterion.criterion_id FROM ad_group_criterion"),+ (+ True,+ "SELECT ad_group_criterion.criterion_id FROM ad_group_criterion "+ "WHERE ad_group_criterion.status != 'REMOVED'",+ ),+ ],+ )+ def test_skip_removed_filters_only_when_enabled(self, skip_removed: bool, expected_query: str) -> None:+ table = GoogleAdsTable(+ name="ad_group_criterion",+ alias="ad_group_criterion",+ columns=[_string_column("ad_group_criterion.criterion_id")],+ parents=None,+ requires_filter=False,+ primary_key=["ad_group_criterion.criterion_id"],+ should_sync_default=False,+ description=None,+ removed_filter="ad_group_criterion.status != 'REMOVED'",+ )++ _response, queries = self._run_source(table, skip_removed=skip_removed)++ assert queries == [expected_query]+def test_full_refresh_report_table_scans_the_full_range_without_windows(self):# A full-refresh pipeline persists no cursor, so a budgeted windowed drain restarts from# the same backfill date every run and the refresh replaces the whole table with that same