Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(stripe): skip the poll for webhook tables with no webhook (#123644)

posthog[bot]9c07e768e1c1
diff --git a/posthog/temporal/health_checks/registry.py b/posthog/temporal/health_checks/registry.py
index 2dfd983439a3..e261b9b56e8a 100644
--- a/posthog/temporal/health_checks/registry.py
+++ b/posthog/temporal/health_checks/registry.py
@@ -21,6 +21,7 @@ HEALTH_CHECK_MODULES = [
"products.cdp.backend.temporal.health_checks.ingestion_warnings",
"products.data_warehouse.backend.temporal.health_checks.materialized_view_failure",
"products.data_warehouse.backend.temporal.health_checks.webhook_subscription_stale",
+ "products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered",
"products.web_analytics.backend.temporal.health_checks.scroll_depth",
"products.web_analytics.backend.temporal.health_checks.authorized_urls",
"products.web_analytics.backend.temporal.health_checks.reverse_proxy",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
index 785a86621b92..e2f2e5c0b138 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
@@ -171,6 +171,32 @@ class TestWebhookSourceManager:
):
assert await manager.webhook_enabled(webhook_only) is expected
+ @parameterized.expand(
+ [
+ ("no_hog_function", False, True, True, False, True),
+ ("has_hog_function", True, True, True, False, False),
+ ("initial_sync_not_complete", False, True, False, False, False),
+ ("reset_pipeline", False, True, True, True, False),
+ ("not_webhook", False, False, True, False, False),
+ ]
+ )
+ async def test_webhook_unregistered_conditions(
+ self, _name, has_webhook_function, is_webhook, initial_sync_complete, reset_pipeline, expected
+ ):
+ manager = _make_manager(reset_pipeline=reset_pipeline)
+
+ mock_schema = MagicMock()
+ mock_schema.is_webhook = is_webhook
+ mock_schema.initial_sync_complete = initial_sync_complete
+
+ results = iter([mock_schema, has_webhook_function])
+
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3.database_sync_to_async_pool",
+ side_effect=lambda fn: AsyncMock(return_value=next(results)),
+ ):
+ assert await manager.webhook_unregistered() is expected
+
async def test_list_parquet_files_filters_correctly(self):
manager = _make_manager()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
index 13160963a017..7c8400854b8a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
@@ -199,6 +199,7 @@ class SourceDriver:
_REAL_WEBHOOK_CHECKS = {
"webhook_enabled": WebhookSourceManager.webhook_enabled,
"schema_is_webhook": WebhookSourceManager.schema_is_webhook,
+ "webhook_unregistered": WebhookSourceManager.webhook_unregistered,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
index 41e54b66a516..f616c5a9fc34 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
@@ -40,7 +40,6 @@ class WebhookSourceManager:
return s3_path.replace("s3://", "")
async def webhook_enabled(self, webhook_only: bool = False) -> bool:
- from products.cdp.backend.models.hog_functions.hog_function import HogFunction
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
schema = await database_sync_to_async_pool(db_read_with_retry)(
@@ -62,7 +61,28 @@ class WebhookSourceManager:
)
return False
- has_webhook_function = await database_sync_to_async_pool(db_read_with_retry)(
+ return await self._has_enabled_webhook_function()
+
+ async def webhook_unregistered(self) -> bool:
+ """Whether a webhook schema is past its initial sync but its source has no enabled webhook.
+
+ Such a schema falls back to a poll. A schema created with webhook sync has no stored cursor,
+ so that poll re-reads the whole table on every run. The caller decides whether to skip it.
+ """
+ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+
+ schema = await database_sync_to_async_pool(db_read_with_retry)(
+ lambda: ExternalDataSchema.objects.get(id=self._inputs.schema_id, team_id=self._inputs.team_id)
+ )
+ if not schema.is_webhook or not schema.initial_sync_complete or self._inputs.reset_pipeline:
+ return False
+
+ return not await self._has_enabled_webhook_function()
+
+ async def _has_enabled_webhook_function(self) -> bool:
+ from products.cdp.backend.models.hog_functions.hog_function import HogFunction
+
+ return await database_sync_to_async_pool(db_read_with_retry)(
lambda: HogFunction.objects.filter(
inputs__source_id__value=self._inputs.source_id,
team_id=self._inputs.team_id,
@@ -72,8 +92,6 @@ class WebhookSourceManager:
).exists()
)
- return has_webhook_function
-
async def schema_is_webhook(self) -> bool:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index da29d89b8b7f..332aa0810168 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -984,6 +984,7 @@ def get_rows(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[StripeResumeConfig],
api_version: str,
+ skip_full_listing: bool = False,
should_use_incremental_field: bool = False,
warehouse_parent: Optional["ParentTableRef"] = None,
include_zero_balance_customers: bool = False,
@@ -1026,6 +1027,14 @@ def get_rows(
or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)
or isinstance(resource, StripeNestedResource)
):
+ if skip_full_listing:
+ logger.warning(
+ f"Stripe: skipped the {endpoint} sync. This table uses webhook sync, but the source has no "
+ "active webhook. Set up the webhook on the source's Webhook tab, or change the table to a "
+ "different sync method."
+ )
+ return
+
logger.debug(f"Stripe: iterating all objects from resource")
# Set only on the nested warehouse path; the final flush below is shared with flat
# resources, which have no parent sweep to checkpoint.
@@ -1609,6 +1618,11 @@ def stripe_source(
# re-enable — a poll could never rebuild the table.
webhook_only = endpoint in WEBHOOK_ONLY_ENDPOINTS
webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)(webhook_only=webhook_only)
+ # A webhook table with no webhook falls back to a poll. Only a poll that lists the whole table
+ # is skipped: a table that kept its `created` cursor still polls cheaply with `created[gt]`.
+ webhook_unregistered = (
+ not webhook_enabled and not webhook_only and async_to_sync(webhook_source_manager.webhook_unregistered)()
+ )
warehouse_parent = _resolve_warehouse_parent(endpoint, team_id, source_id, use_warehouse_parent)
warehouse_table_id: Optional[str] = None
if not webhook_enabled:
@@ -1652,6 +1666,7 @@ def stripe_source(
logger=logger,
should_use_incremental_field=should_use_incremental_field,
resumable_source_manager=resumable_source_manager,
+ skip_full_listing=webhook_unregistered,
api_version=api_version,
warehouse_parent=warehouse_parent,
include_zero_balance_customers=include_zero_balance_customers,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 2c97b8b6b1d0..edf6f7a4f1f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -238,6 +238,44 @@ class TestStripeGetRowsIncrementalCursor:
assert [row["id"] for row in rows] == [f"ch_{i}" for i in range(5)]
assert len(tables) > 1
+ @pytest.mark.parametrize(
+ "skip_full_listing,last_value,expected_ids",
+ [
+ (True, None, []),
+ (True, "1700000050", ["ch_2"]),
+ (False, None, ["ch_2", "ch_1"]),
+ ],
+ )
+ def test_skip_full_listing_keeps_the_incremental_poll(self, skip_full_listing, last_value, expected_ids):
+ objects = [
+ {"id": "ch_2", "created": 1700000100},
+ {"id": "ch_1", "created": 1700000040},
+ ]
+ method = MagicMock(side_effect=lambda params: _FakeStripeList(objects))
+ resumable_source_manager = mock.MagicMock()
+ resumable_source_manager.can_resume.return_value = False
+
+ with mock.patch.object(
+ stripe_module, "_build_resources", return_value={"charge": StripeResource(method=method)}
+ ):
+ tables = list(
+ get_rows(
+ api_key="sk_test_123",
+ endpoint="charge",
+ account_id=None,
+ db_incremental_field_last_value=last_value,
+ db_incremental_field_earliest_value=None,
+ logger=mock.MagicMock(),
+ resumable_source_manager=resumable_source_manager,
+ api_version=STRIPE_API_VERSION_ACACIA,
+ should_use_incremental_field=True,
+ skip_full_listing=skip_full_listing,
+ )
+ )
+
+ assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids
+ assert method.called is bool(expected_ids)
+
NOW = 1_800_000_000
@@ -1446,6 +1484,7 @@ class TestWebhookOnlyResponseWiring:
def _make_manager(self, enabled: bool) -> MagicMock:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=enabled)
+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return manager
def _source(self, endpoint: str, manager: MagicMock) -> Any:
@@ -1475,6 +1514,18 @@ class TestWebhookOnlyResponseWiring:
assert response.webhook_only is False
manager.webhook_enabled.assert_awaited_once_with(webhook_only=False)
+ @parameterized.expand(
+ [("unregistered_webhook_skips_full_poll", True, []), ("no_webhook_schema_polls", False, ["cus_1"])]
+ )
+ def test_unregistered_webhook_poll(self, _name: str, unregistered: bool, expected_ids: list[str]) -> None:
+ manager = self._make_manager(enabled=False)
+ manager.webhook_unregistered = mock.AsyncMock(return_value=unregistered)
+ resource = StripeResource(method=lambda params: cast(ListObject[Any], _FakeStripeList([{"id": "cus_1"}])))
+ with mock.patch.object(stripe_module, "_build_resources", return_value={CUSTOMER_RESOURCE_NAME: resource}):
+ response = self._source(CUSTOMER_RESOURCE_NAME, manager)
+ tables = list(response.items())
+ assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids
+
def test_discount_partitions_on_start_not_created(self) -> None:
# Discount objects carry `start`/`end`, not `created`. If the incremental-field entry is
# dropped the partition key falls back to "created" and the Delta partitioner KeyErrors on
@@ -1497,6 +1548,7 @@ class TestCustomerPaymentMethodHistory:
def _make_manager(self, enabled: bool) -> MagicMock:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=enabled)
+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return manager
def _source(self, endpoint: str, manager: MagicMock) -> Any:
@@ -1913,6 +1965,7 @@ class TestWebhookRefetch:
)
webhooks = MagicMock(spec=WebhookSourceManager)
webhooks.webhook_enabled = mock.AsyncMock(return_value=True)
+ webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)
events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])
with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):
@@ -1965,6 +2018,7 @@ class TestPartitioningAndColumnHints:
def _source(self, endpoint: str) -> Any:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=False)
+ manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return stripe_module.stripe_source(
api_key="sk_test_123",
account_id=None,
@@ -2775,6 +2829,7 @@ class _PipelineAroundTheSweep:
)
webhooks = MagicMock(spec=WebhookSourceManager)
webhooks.webhook_enabled = mock.AsyncMock(return_value=False)
+ webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)
with (
patch.object(stripe_module, "StripeClient"),
patch.object(stripe_module, "STRIPE_CHUNK_SIZE", 1),
diff --git a/products/warehouse_sources/backend/temporal/health_checks/__init__.py b/products/warehouse_sources/backend/temporal/health_checks/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py
new file mode 100644
index 000000000000..e1837eafb419
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py
@@ -0,0 +1,106 @@
+from posthog.job_owners import JobOwners
+from posthog.models.health_issue import HealthIssue
+from posthog.temporal.health_checks.detectors import HealthExecutionPolicy
+from posthog.temporal.health_checks.framework import AlertContent, HealthCheck, Remediation
+from posthog.temporal.health_checks.models import HealthCheckResult
+
+from products.cdp.backend.facade.models import HogFunction
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+
+_MAX_LISTED = 20
+
+
+class WebhookNotRegisteredCheck(HealthCheck):
+ """Flag a source whose tables use webhook sync but that has no active webhook.
+
+ These tables receive no webhook data. What the sync does instead depends on the source: a
+ webhook-only table gets no data, Stripe skips a poll that would list the whole table, and other
+ sources poll the vendor API, often reading the whole table on every run.
+ """
+
+ name = "webhook_not_registered"
+ kind = "webhook_not_registered"
+ owner = JobOwners.TEAM_WAREHOUSE_SOURCES
+ policy = HealthExecutionPolicy(batch_size=50, max_concurrent=3)
+ schedule = "50 7 * * *"
+ active_since_days = 30
+ access_controlled_resource = "external_data_source"
+ remediation = Remediation(
+ human="""
+ Some tables on this source use webhook sync, but the source has no active webhook, so
+ these tables get no webhook updates. Depending on the table, the sync either skips it or
+ reads it from the source API instead, which can be slow and can read the whole table on
+ every sync. Open the source (Data pipeline / Data warehouse → Sources), go to the Webhook
+ tab, and create the webhook. If you can't create a webhook for this account, change the
+ affected tables to incremental or full refresh sync instead.
+ """,
+ agent="""
+ Confirm with `external-data-sources-retrieve` that the listed schemas use webhook sync.
+ Have the user create the webhook from the source's Webhook tab, or enable it if it is
+ off. If the account can't have a webhook, change the affected schemas to another sync
+ type with `external-data-schemas-partial-update`. The check clears once the source has
+ an enabled webhook or no webhook-synced schemas.
+ """,
+ )
+
+ @classmethod
+ def render_alert(cls, issue: HealthIssue) -> AlertContent:
+ name = issue.payload.get("pipeline_name") or issue.payload.get("source_type", "a webhook source")
+ return AlertContent(
+ title="Webhook isn't set up for webhook-synced tables",
+ summary=f"{name} has tables set to webhook sync, but the source has no active webhook",
+ link="/health/pipeline-status",
+ )
+
+ def detect(self, team_ids: list[int]) -> dict[int, list[HealthCheckResult]]:
+ schemas = ExternalDataSchema.objects.filter(
+ team_id__in=team_ids,
+ deleted=False,
+ should_sync=True,
+ initial_sync_complete=True,
+ sync_type=ExternalDataSchema.SyncType.WEBHOOK,
+ ).values_list("source_id", "name")
+
+ schema_names_by_source: dict[str, list[str]] = {}
+ for source_id, schema_name in schemas:
+ schema_names_by_source.setdefault(str(source_id), []).append(schema_name)
+ if not schema_names_by_source:
+ return {}
+
+ sources_with_webhook = {
+ str(source_id)
+ for source_id in HogFunction.objects.filter(
+ team_id__in=team_ids,
+ type="warehouse_source_webhook",
+ enabled=True,
+ deleted=False,
+ inputs__source_id__value__in=list(schema_names_by_source),
+ ).values_list("inputs__source_id__value", flat=True)
+ }
+
+ sources = (
+ ExternalDataSource.objects.filter(
+ pk__in=[pk for pk in schema_names_by_source if pk not in sources_with_webhook],
+ )
+ .exclude(deleted=True)
+ .values_list("pk", "team_id", "prefix", "source_type")
+ )
+
+ issues: dict[int, list[HealthCheckResult]] = {}
+ for source_pk, team_id, prefix, source_type in sources:
+ issues.setdefault(team_id, []).append(
+ HealthCheckResult(
+ severity=HealthIssue.Severity.WARNING,
+ payload={
+ "pipeline_type": "external_data_webhook",
+ "pipeline_id": str(source_pk),
+ "pipeline_name": prefix or source_type,
+ "source_type": source_type,
+ "affected_schemas": sorted(schema_names_by_source[str(source_pk)])[:_MAX_LISTED],
+ },
+ hash_keys=["pipeline_type", "pipeline_id"],
+ )
+ )
+
+ return issues
diff --git a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py
new file mode 100644
index 000000000000..8c03fcfb8455
--- /dev/null
+++ b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py
@@ -0,0 +1,58 @@
+import uuid
+
+from posthog.test.base import BaseTest
+
+from parameterized import parameterized
+
+from products.cdp.backend.models.hog_functions.hog_function import HogFunction
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered import WebhookNotRegisteredCheck
+
+
+class TestWebhookNotRegisteredCheck(BaseTest):
+ @parameterized.expand(
+ [
+ ("no_webhook_function", None, True, True),
+ ("disabled_webhook_function", False, True, True),
+ ("enabled_webhook_function", True, True, False),
+ ("initial_sync_not_complete", None, False, False),
+ ]
+ )
+ def test_flags_webhook_schemas_without_an_enabled_webhook(
+ self, _name: str, webhook_enabled: bool | None, initial_sync_complete: bool, flagged: bool
+ ) -> None:
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()),
+ connection_id=str(uuid.uuid4()),
+ team=self.team,
+ source_type="Stripe",
+ job_inputs={"stripe_secret_key": "sk_test_x"},
+ )
+ if webhook_enabled is not None:
+ HogFunction.objects.create(
+ team=self.team,
+ type="warehouse_source_webhook",
+ hog="return event",
+ enabled=webhook_enabled,
+ inputs_schema=[{"key": "source_id", "type": "string"}],
+ inputs={"source_id": {"value": str(source.pk)}},
+ )
+ ExternalDataSchema.objects.create(
+ name="Customer",
+ team=self.team,
+ source=source,
+ sync_type=ExternalDataSchema.SyncType.WEBHOOK,
+ should_sync=True,
+ initial_sync_complete=initial_sync_complete,
+ )
+
+ issues = WebhookNotRegisteredCheck().detect([self.team.id])
+ payloads = [result.payload for result in issues.get(self.team.id, [])]
+
+ if flagged:
+ assert len(payloads) == 1
+ assert payloads[0]["pipeline_id"] == str(source.pk)
+ assert payloads[0]["affected_schemas"] == ["Customer"]
+ else:
+ assert payloads == []