Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): let google ads users skip removed keywords (#118963)

posthog[bot]a44612d396d3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py
index c1922f961944..0da47de898da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/googleads.py
@@ -14,5 +14,6 @@ class GoogleAdsIsMccAccountConfig(config.Config):
class GoogleAdsSourceConfig(config.Config):
customer_id: str
google_ads_integration_id: int = config.value(converter=config.str_to_int)
- is_mcc_account: GoogleAdsIsMccAccountConfig | None = None
start_date: str | None = None
+ skip_removed_keywords: bool = config.value(default=config.str_to_bool("false"), converter=config.str_to_bool)
+ is_mcc_account: GoogleAdsIsMccAccountConfig | None = None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
index 973c972b7c95..0c26ace20e70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/google_ads.py
@@ -449,6 +449,7 @@ class GoogleAdsTable(Table[GoogleAdsColumn]):
description: str | None,
partition_keys: list[str] | None = None,
extra_where: str | None = None,
+ removed_filter: str | None = None,
**kwargs,
):
self.requires_filter = requires_filter
@@ -457,6 +458,7 @@ class GoogleAdsTable(Table[GoogleAdsColumn]):
self.description = description
self.partition_keys = [pkey.replace(".", "_") for pkey in partition_keys] if partition_keys else None
self.extra_where = extra_where
+ self.removed_filter = removed_filter
super().__init__(*args, **kwargs)
@@ -493,6 +495,7 @@ def get_schemas(config: GoogleAdsSourceConfigUnion, team_id: int, api_version: s
requires_filter = resource_contents.get("filter_field_names", None) is not None
primary_key = typing.cast(list[str], resource_contents.get("primary_key", []))
extra_where = typing.cast(str | None, resource_contents.get("extra_where", None))
+ removed_filter = typing.cast(str | None, resource_contents.get("removed_filter", None))
partition_keys = typing.cast(list[str] | None, resource_contents.get("partition_keys", None))
should_sync_default = resource_contents.get("should_sync_default", True)
@@ -527,6 +530,7 @@ def get_schemas(config: GoogleAdsSourceConfigUnion, team_id: int, api_version: s
requires_filter=requires_filter,
primary_key=primary_key,
extra_where=extra_where,
+ removed_filter=removed_filter,
partition_keys=partition_keys,
columns=columns,
parents=None,
@@ -642,6 +646,7 @@ def google_ads_source(
db_incremental_field_last_value_before_lookback: typing.Any = None,
history_start: typing.Any = None,
requested_start: str | None = None,
+ skip_removed: bool = False,
) -> SourceResponse:
"""A data warehouse Google Ads source.
@@ -720,6 +725,8 @@ def google_ads_source(
conditions.append(f"{incremental_field} < {upper_literal}")
if table.extra_where:
conditions.append(table.extra_where)
+ if skip_removed and table.removed_filter:
+ conditions.append(table.removed_filter)
if conditions:
query += " WHERE " + " AND ".join(conditions)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py
index 27764826936d..fddc0411b8b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/schemas.py
@@ -527,6 +527,7 @@ RESOURCE_SCHEMAS: dict[str, dict[str, Any]] = {
},
"keyword": {
"resource_name": "keyword_view",
+ "removed_filter": "ad_group_criterion.status != 'REMOVED' AND ad_group.status != 'REMOVED' AND campaign.status != 'REMOVED'",
"primary_key": ["ad_group_criterion.criterion_id", "ad_group.id", "campaign.id", "customer.id"],
"field_names": [
"ad_group_criterion.criterion_id",
@@ -653,6 +654,7 @@ RESOURCE_SCHEMAS: dict[str, dict[str, Any]] = {
},
"ad_group_criterion": {
"resource_name": "ad_group_criterion",
+ "removed_filter": "ad_group_criterion.status != 'REMOVED' AND ad_group.status != 'REMOVED' AND campaign.status != 'REMOVED'",
"primary_key": ["ad_group_criterion.criterion_id", "ad_group.id", "campaign.id", "customer.id"],
"should_sync_default": False,
"description": "Every targeting criterion at ad group level, including negative keywords and other exclusions. Unlike the keyword table (which is backed by keyword_view and only returns positive, servable keywords), this reaches negative and non-keyword criteria. Filter on ad_group_criterion.negative to isolate exclusions.",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
index ed7e0cf48e62..02b22e48839f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/source.py
@@ -25,6 +25,9 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldInputConfigType,
SourceFieldOauthAccountSelectConfig,
SourceFieldOauthConfig,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigConverter,
+ SourceFieldSelectConfigOption,
SourceFieldSwitchGroupConfig,
SuggestedTable,
)
@@ -303,6 +306,7 @@ class GoogleAdsSource(
db_incremental_field_last_value_before_lookback=inputs.db_incremental_field_last_value_before_lookback,
history_start=inputs.history_start,
requested_start=config.start_date if isinstance(config, GoogleAdsSourceConfig) else None,
+ skip_removed=isinstance(config, GoogleAdsSourceConfig) and config.skip_removed_keywords,
)
@property
@@ -349,6 +353,23 @@ class GoogleAdsSource(
placeholder="2020-01-01",
secret=False,
),
+ SourceFieldSelectConfig(
+ name="skip_removed_keywords",
+ label="Skip removed keywords?",
+ caption=(
+ "Leave out keywords that are removed, or that sit in a removed ad group or "
+ "campaign, from the keyword and ad group criterion tables. Every sync reads "
+ "these tables in full, so removed keywords count towards your billed row "
+ "usage on each sync."
+ ),
+ required=True,
+ defaultValue="false",
+ converter=SourceFieldSelectConfigConverter.STR_TO_BOOL,
+ options=[
+ SourceFieldSelectConfigOption(label="Yes", value="true"),
+ SourceFieldSelectConfigOption(label="No", value="false"),
+ ],
+ ),
SourceFieldSwitchGroupConfig(
name="is_mcc_account",
label="Using MCC account?",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
index af0f4cbdb05c..fa86ee1210b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/tests/test_google_ads_source.py
@@ -1849,6 +1849,34 @@ class TestGoogleAdsQueryConstruction:
"AND metrics.impressions > 0 ORDER BY segments.date ASC LIMIT 1"
)
+ @pytest.mark.parametrize(
+ "skip_removed,expected_query",
+ [
+ (False, "SELECT ad_group_criterion.criterion_id FROM ad_group_criterion"),
+ (
+ True,
+ "SELECT ad_group_criterion.criterion_id FROM ad_group_criterion "
+ "WHERE ad_group_criterion.status != 'REMOVED'",
+ ),
+ ],
+ )
+ def test_skip_removed_filters_only_when_enabled(self, skip_removed: bool, expected_query: str) -> None:
+ table = GoogleAdsTable(
+ name="ad_group_criterion",
+ alias="ad_group_criterion",
+ columns=[_string_column("ad_group_criterion.criterion_id")],
+ parents=None,
+ requires_filter=False,
+ primary_key=["ad_group_criterion.criterion_id"],
+ should_sync_default=False,
+ description=None,
+ removed_filter="ad_group_criterion.status != 'REMOVED'",
+ )
+
+ _response, queries = self._run_source(table, skip_removed=skip_removed)
+
+ assert queries == [expected_query]
+
def test_full_refresh_report_table_scans_the_full_range_without_windows(self):
# A full-refresh pipeline persists no cursor, so a budgeted windowed drain restarts from
# the same backfill date every run and the refresh replaces the whole table with that same