:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(stripe): skip the poll for webhook tables with no webhook (#123644)
posthog[bot]9c07e768e1c1
diff --git a/posthog/temporal/health_checks/registry.py b/posthog/temporal/health_checks/registry.pyindex 2dfd983439a3..e261b9b56e8a 100644--- a/posthog/temporal/health_checks/registry.py+++ b/posthog/temporal/health_checks/registry.py@@ -21,6 +21,7 @@ HEALTH_CHECK_MODULES = ["products.cdp.backend.temporal.health_checks.ingestion_warnings","products.data_warehouse.backend.temporal.health_checks.materialized_view_failure","products.data_warehouse.backend.temporal.health_checks.webhook_subscription_stale",+ "products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered","products.web_analytics.backend.temporal.health_checks.scroll_depth","products.web_analytics.backend.temporal.health_checks.authorized_urls","products.web_analytics.backend.temporal.health_checks.reverse_proxy",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.pyindex 785a86621b92..e2f2e5c0b138 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py@@ -171,6 +171,32 @@ class TestWebhookSourceManager:):assert await manager.webhook_enabled(webhook_only) is expected+ @parameterized.expand(+ [+ ("no_hog_function", False, True, True, False, True),+ ("has_hog_function", True, True, True, False, False),+ ("initial_sync_not_complete", False, True, False, False, False),+ ("reset_pipeline", False, True, True, True, False),+ ("not_webhook", False, False, True, False, False),+ ]+ )+ async def test_webhook_unregistered_conditions(+ self, _name, has_webhook_function, is_webhook, initial_sync_complete, reset_pipeline, expected+ ):+ manager = _make_manager(reset_pipeline=reset_pipeline)++ mock_schema = MagicMock()+ mock_schema.is_webhook = is_webhook+ mock_schema.initial_sync_complete = initial_sync_complete++ results = iter([mock_schema, has_webhook_function])++ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3.database_sync_to_async_pool",+ side_effect=lambda fn: AsyncMock(return_value=next(results)),+ ):+ assert await manager.webhook_unregistered() is expected+async def test_list_parquet_files_filters_correctly(self):manager = _make_manager()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.pyindex 13160963a017..7c8400854b8a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py@@ -199,6 +199,7 @@ class SourceDriver:_REAL_WEBHOOK_CHECKS = {"webhook_enabled": WebhookSourceManager.webhook_enabled,"schema_is_webhook": WebhookSourceManager.schema_is_webhook,+ "webhook_unregistered": WebhookSourceManager.webhook_unregistered,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.pyindex 41e54b66a516..f616c5a9fc34 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py@@ -40,7 +40,6 @@ class WebhookSourceManager:return s3_path.replace("s3://", "")async def webhook_enabled(self, webhook_only: bool = False) -> bool:- from products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemaschema = await database_sync_to_async_pool(db_read_with_retry)(@@ -62,7 +61,28 @@ class WebhookSourceManager:)return False- has_webhook_function = await database_sync_to_async_pool(db_read_with_retry)(+ return await self._has_enabled_webhook_function()++ async def webhook_unregistered(self) -> bool:+ """Whether a webhook schema is past its initial sync but its source has no enabled webhook.++ Such a schema falls back to a poll. A schema created with webhook sync has no stored cursor,+ so that poll re-reads the whole table on every run. The caller decides whether to skip it.+ """+ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema++ schema = await database_sync_to_async_pool(db_read_with_retry)(+ lambda: ExternalDataSchema.objects.get(id=self._inputs.schema_id, team_id=self._inputs.team_id)+ )+ if not schema.is_webhook or not schema.initial_sync_complete or self._inputs.reset_pipeline:+ return False++ return not await self._has_enabled_webhook_function()++ async def _has_enabled_webhook_function(self) -> bool:+ from products.cdp.backend.models.hog_functions.hog_function import HogFunction++ return await database_sync_to_async_pool(db_read_with_retry)(lambda: HogFunction.objects.filter(inputs__source_id__value=self._inputs.source_id,team_id=self._inputs.team_id,@@ -72,8 +92,6 @@ class WebhookSourceManager:).exists())- return has_webhook_function-async def schema_is_webhook(self) -> bool:from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemadiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.pyindex da29d89b8b7f..332aa0810168 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py@@ -984,6 +984,7 @@ def get_rows(logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[StripeResumeConfig],api_version: str,+ skip_full_listing: bool = False,should_use_incremental_field: bool = False,warehouse_parent: Optional["ParentTableRef"] = None,include_zero_balance_customers: bool = False,@@ -1026,6 +1027,14 @@ def get_rows(or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)or isinstance(resource, StripeNestedResource)):+ if skip_full_listing:+ logger.warning(+ f"Stripe: skipped the {endpoint} sync. This table uses webhook sync, but the source has no "+ "active webhook. Set up the webhook on the source's Webhook tab, or change the table to a "+ "different sync method."+ )+ return+logger.debug(f"Stripe: iterating all objects from resource")# Set only on the nested warehouse path; the final flush below is shared with flat# resources, which have no parent sweep to checkpoint.@@ -1609,6 +1618,11 @@ def stripe_source(# re-enable — a poll could never rebuild the table.webhook_only = endpoint in WEBHOOK_ONLY_ENDPOINTSwebhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)(webhook_only=webhook_only)+ # A webhook table with no webhook falls back to a poll. Only a poll that lists the whole table+ # is skipped: a table that kept its `created` cursor still polls cheaply with `created[gt]`.+ webhook_unregistered = (+ not webhook_enabled and not webhook_only and async_to_sync(webhook_source_manager.webhook_unregistered)()+ )warehouse_parent = _resolve_warehouse_parent(endpoint, team_id, source_id, use_warehouse_parent)warehouse_table_id: Optional[str] = Noneif not webhook_enabled:@@ -1652,6 +1666,7 @@ def stripe_source(logger=logger,should_use_incremental_field=should_use_incremental_field,resumable_source_manager=resumable_source_manager,+ skip_full_listing=webhook_unregistered,api_version=api_version,warehouse_parent=warehouse_parent,include_zero_balance_customers=include_zero_balance_customers,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.pyindex 2c97b8b6b1d0..edf6f7a4f1f5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py@@ -238,6 +238,44 @@ class TestStripeGetRowsIncrementalCursor:assert [row["id"] for row in rows] == [f"ch_{i}" for i in range(5)]assert len(tables) > 1+ @pytest.mark.parametrize(+ "skip_full_listing,last_value,expected_ids",+ [+ (True, None, []),+ (True, "1700000050", ["ch_2"]),+ (False, None, ["ch_2", "ch_1"]),+ ],+ )+ def test_skip_full_listing_keeps_the_incremental_poll(self, skip_full_listing, last_value, expected_ids):+ objects = [+ {"id": "ch_2", "created": 1700000100},+ {"id": "ch_1", "created": 1700000040},+ ]+ method = MagicMock(side_effect=lambda params: _FakeStripeList(objects))+ resumable_source_manager = mock.MagicMock()+ resumable_source_manager.can_resume.return_value = False++ with mock.patch.object(+ stripe_module, "_build_resources", return_value={"charge": StripeResource(method=method)}+ ):+ tables = list(+ get_rows(+ api_key="sk_test_123",+ endpoint="charge",+ account_id=None,+ db_incremental_field_last_value=last_value,+ db_incremental_field_earliest_value=None,+ logger=mock.MagicMock(),+ resumable_source_manager=resumable_source_manager,+ api_version=STRIPE_API_VERSION_ACACIA,+ should_use_incremental_field=True,+ skip_full_listing=skip_full_listing,+ )+ )++ assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids+ assert method.called is bool(expected_ids)+NOW = 1_800_000_000@@ -1446,6 +1484,7 @@ class TestWebhookOnlyResponseWiring:def _make_manager(self, enabled: bool) -> MagicMock:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=enabled)+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)return managerdef _source(self, endpoint: str, manager: MagicMock) -> Any:@@ -1475,6 +1514,18 @@ class TestWebhookOnlyResponseWiring:assert response.webhook_only is Falsemanager.webhook_enabled.assert_awaited_once_with(webhook_only=False)+ @parameterized.expand(+ [("unregistered_webhook_skips_full_poll", True, []), ("no_webhook_schema_polls", False, ["cus_1"])]+ )+ def test_unregistered_webhook_poll(self, _name: str, unregistered: bool, expected_ids: list[str]) -> None:+ manager = self._make_manager(enabled=False)+ manager.webhook_unregistered = mock.AsyncMock(return_value=unregistered)+ resource = StripeResource(method=lambda params: cast(ListObject[Any], _FakeStripeList([{"id": "cus_1"}])))+ with mock.patch.object(stripe_module, "_build_resources", return_value={CUSTOMER_RESOURCE_NAME: resource}):+ response = self._source(CUSTOMER_RESOURCE_NAME, manager)+ tables = list(response.items())+ assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids+def test_discount_partitions_on_start_not_created(self) -> None:# Discount objects carry `start`/`end`, not `created`. If the incremental-field entry is# dropped the partition key falls back to "created" and the Delta partitioner KeyErrors on@@ -1497,6 +1548,7 @@ class TestCustomerPaymentMethodHistory:def _make_manager(self, enabled: bool) -> MagicMock:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=enabled)+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)return managerdef _source(self, endpoint: str, manager: MagicMock) -> Any:@@ -1913,6 +1965,7 @@ class TestWebhookRefetch:)webhooks = MagicMock(spec=WebhookSourceManager)webhooks.webhook_enabled = mock.AsyncMock(return_value=True)+ webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):@@ -1965,6 +2018,7 @@ class TestPartitioningAndColumnHints:def _source(self, endpoint: str) -> Any:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=False)+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)return stripe_module.stripe_source(api_key="sk_test_123",account_id=None,@@ -2775,6 +2829,7 @@ class _PipelineAroundTheSweep:)webhooks = MagicMock(spec=WebhookSourceManager)webhooks.webhook_enabled = mock.AsyncMock(return_value=False)+ webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)with (patch.object(stripe_module, "StripeClient"),patch.object(stripe_module, "STRIPE_CHUNK_SIZE", 1),diff --git a/products/warehouse_sources/backend/temporal/health_checks/__init__.py b/products/warehouse_sources/backend/temporal/health_checks/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.pynew file mode 100644index 000000000000..e1837eafb419--- /dev/null+++ b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py@@ -0,0 +1,106 @@+from posthog.job_owners import JobOwners+from posthog.models.health_issue import HealthIssue+from posthog.temporal.health_checks.detectors import HealthExecutionPolicy+from posthog.temporal.health_checks.framework import AlertContent, HealthCheck, Remediation+from posthog.temporal.health_checks.models import HealthCheckResult++from products.cdp.backend.facade.models import HogFunction+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource++_MAX_LISTED = 20+++class WebhookNotRegisteredCheck(HealthCheck):+ """Flag a source whose tables use webhook sync but that has no active webhook.++ These tables receive no webhook data. What the sync does instead depends on the source: a+ webhook-only table gets no data, Stripe skips a poll that would list the whole table, and other+ sources poll the vendor API, often reading the whole table on every run.+ """++ name = "webhook_not_registered"+ kind = "webhook_not_registered"+ owner = JobOwners.TEAM_WAREHOUSE_SOURCES+ policy = HealthExecutionPolicy(batch_size=50, max_concurrent=3)+ schedule = "50 7 * * *"+ active_since_days = 30+ access_controlled_resource = "external_data_source"+ remediation = Remediation(+ human="""+ Some tables on this source use webhook sync, but the source has no active webhook, so+ these tables get no webhook updates. Depending on the table, the sync either skips it or+ reads it from the source API instead, which can be slow and can read the whole table on+ every sync. Open the source (Data pipeline / Data warehouse → Sources), go to the Webhook+ tab, and create the webhook. If you can't create a webhook for this account, change the+ affected tables to incremental or full refresh sync instead.+ """,+ agent="""+ Confirm with `external-data-sources-retrieve` that the listed schemas use webhook sync.+ Have the user create the webhook from the source's Webhook tab, or enable it if it is+ off. If the account can't have a webhook, change the affected schemas to another sync+ type with `external-data-schemas-partial-update`. The check clears once the source has+ an enabled webhook or no webhook-synced schemas.+ """,+ )++ @classmethod+ def render_alert(cls, issue: HealthIssue) -> AlertContent:+ name = issue.payload.get("pipeline_name") or issue.payload.get("source_type", "a webhook source")+ return AlertContent(+ title="Webhook isn't set up for webhook-synced tables",+ summary=f"{name} has tables set to webhook sync, but the source has no active webhook",+ link="/health/pipeline-status",+ )++ def detect(self, team_ids: list[int]) -> dict[int, list[HealthCheckResult]]:+ schemas = ExternalDataSchema.objects.filter(+ team_id__in=team_ids,+ deleted=False,+ should_sync=True,+ initial_sync_complete=True,+ sync_type=ExternalDataSchema.SyncType.WEBHOOK,+ ).values_list("source_id", "name")++ schema_names_by_source: dict[str, list[str]] = {}+ for source_id, schema_name in schemas:+ schema_names_by_source.setdefault(str(source_id), []).append(schema_name)+ if not schema_names_by_source:+ return {}++ sources_with_webhook = {+ str(source_id)+ for source_id in HogFunction.objects.filter(+ team_id__in=team_ids,+ type="warehouse_source_webhook",+ enabled=True,+ deleted=False,+ inputs__source_id__value__in=list(schema_names_by_source),+ ).values_list("inputs__source_id__value", flat=True)+ }++ sources = (+ ExternalDataSource.objects.filter(+ pk__in=[pk for pk in schema_names_by_source if pk not in sources_with_webhook],+ )+ .exclude(deleted=True)+ .values_list("pk", "team_id", "prefix", "source_type")+ )++ issues: dict[int, list[HealthCheckResult]] = {}+ for source_pk, team_id, prefix, source_type in sources:+ issues.setdefault(team_id, []).append(+ HealthCheckResult(+ severity=HealthIssue.Severity.WARNING,+ payload={+ "pipeline_type": "external_data_webhook",+ "pipeline_id": str(source_pk),+ "pipeline_name": prefix or source_type,+ "source_type": source_type,+ "affected_schemas": sorted(schema_names_by_source[str(source_pk)])[:_MAX_LISTED],+ },+ hash_keys=["pipeline_type", "pipeline_id"],+ )+ )++ return issuesdiff --git a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.pynew file mode 100644index 000000000000..8c03fcfb8455--- /dev/null+++ b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py@@ -0,0 +1,58 @@+import uuid++from posthog.test.base import BaseTest++from parameterized import parameterized++from products.cdp.backend.models.hog_functions.hog_function import HogFunction+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered import WebhookNotRegisteredCheck+++class TestWebhookNotRegisteredCheck(BaseTest):+ @parameterized.expand(+ [+ ("no_webhook_function", None, True, True),+ ("disabled_webhook_function", False, True, True),+ ("enabled_webhook_function", True, True, False),+ ("initial_sync_not_complete", None, False, False),+ ]+ )+ def test_flags_webhook_schemas_without_an_enabled_webhook(+ self, _name: str, webhook_enabled: bool | None, initial_sync_complete: bool, flagged: bool+ ) -> None:+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()),+ connection_id=str(uuid.uuid4()),+ team=self.team,+ source_type="Stripe",+ job_inputs={"stripe_secret_key": "sk_test_x"},+ )+ if webhook_enabled is not None:+ HogFunction.objects.create(+ team=self.team,+ type="warehouse_source_webhook",+ hog="return event",+ enabled=webhook_enabled,+ inputs_schema=[{"key": "source_id", "type": "string"}],+ inputs={"source_id": {"value": str(source.pk)}},+ )+ ExternalDataSchema.objects.create(+ name="Customer",+ team=self.team,+ source=source,+ sync_type=ExternalDataSchema.SyncType.WEBHOOK,+ should_sync=True,+ initial_sync_complete=initial_sync_complete,+ )++ issues = WebhookNotRegisteredCheck().detect([self.team.id])+ payloads = [result.payload for result in issues.get(self.team.id, [])]++ if flagged:+ assert len(payloads) == 1+ assert payloads[0]["pipeline_id"] == str(source.pk)+ assert payloads[0]["affected_schemas"] == ["Customer"]+ else:+ assert payloads == []