Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): revert poll skip for stripe tables with no webhook (#128998)

Tom Owersb0658f19f955
diff --git a/posthog/temporal/health_checks/registry.py b/posthog/temporal/health_checks/registry.py
index e261b9b56e8a..2dfd983439a3 100644
--- a/posthog/temporal/health_checks/registry.py
+++ b/posthog/temporal/health_checks/registry.py
@@ -21,7 +21,6 @@ HEALTH_CHECK_MODULES = [
"products.cdp.backend.temporal.health_checks.ingestion_warnings",
"products.data_warehouse.backend.temporal.health_checks.materialized_view_failure",
"products.data_warehouse.backend.temporal.health_checks.webhook_subscription_stale",
- "products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered",
"products.web_analytics.backend.temporal.health_checks.scroll_depth",
"products.web_analytics.backend.temporal.health_checks.authorized_urls",
"products.web_analytics.backend.temporal.health_checks.reverse_proxy",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
index e2f2e5c0b138..785a86621b92 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py
@@ -171,32 +171,6 @@ class TestWebhookSourceManager:
):
assert await manager.webhook_enabled(webhook_only) is expected
- @parameterized.expand(
- [
- ("no_hog_function", False, True, True, False, True),
- ("has_hog_function", True, True, True, False, False),
- ("initial_sync_not_complete", False, True, False, False, False),
- ("reset_pipeline", False, True, True, True, False),
- ("not_webhook", False, False, True, False, False),
- ]
- )
- async def test_webhook_unregistered_conditions(
- self, _name, has_webhook_function, is_webhook, initial_sync_complete, reset_pipeline, expected
- ):
- manager = _make_manager(reset_pipeline=reset_pipeline)
-
- mock_schema = MagicMock()
- mock_schema.is_webhook = is_webhook
- mock_schema.initial_sync_complete = initial_sync_complete
-
- results = iter([mock_schema, has_webhook_function])
-
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3.database_sync_to_async_pool",
- side_effect=lambda fn: AsyncMock(return_value=next(results)),
- ):
- assert await manager.webhook_unregistered() is expected
-
async def test_list_parquet_files_filters_correctly(self):
manager = _make_manager()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
index 7c8400854b8a..13160963a017 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py
@@ -199,7 +199,6 @@ class SourceDriver:
_REAL_WEBHOOK_CHECKS = {
"webhook_enabled": WebhookSourceManager.webhook_enabled,
"schema_is_webhook": WebhookSourceManager.schema_is_webhook,
- "webhook_unregistered": WebhookSourceManager.webhook_unregistered,
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
index f616c5a9fc34..41e54b66a516 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py
@@ -40,6 +40,7 @@ class WebhookSourceManager:
return s3_path.replace("s3://", "")
async def webhook_enabled(self, webhook_only: bool = False) -> bool:
+ from products.cdp.backend.models.hog_functions.hog_function import HogFunction
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
schema = await database_sync_to_async_pool(db_read_with_retry)(
@@ -61,28 +62,7 @@ class WebhookSourceManager:
)
return False
- return await self._has_enabled_webhook_function()
-
- async def webhook_unregistered(self) -> bool:
- """Whether a webhook schema is past its initial sync but its source has no enabled webhook.
-
- Such a schema falls back to a poll. A schema created with webhook sync has no stored cursor,
- so that poll re-reads the whole table on every run. The caller decides whether to skip it.
- """
- from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-
- schema = await database_sync_to_async_pool(db_read_with_retry)(
- lambda: ExternalDataSchema.objects.get(id=self._inputs.schema_id, team_id=self._inputs.team_id)
- )
- if not schema.is_webhook or not schema.initial_sync_complete or self._inputs.reset_pipeline:
- return False
-
- return not await self._has_enabled_webhook_function()
-
- async def _has_enabled_webhook_function(self) -> bool:
- from products.cdp.backend.models.hog_functions.hog_function import HogFunction
-
- return await database_sync_to_async_pool(db_read_with_retry)(
+ has_webhook_function = await database_sync_to_async_pool(db_read_with_retry)(
lambda: HogFunction.objects.filter(
inputs__source_id__value=self._inputs.source_id,
team_id=self._inputs.team_id,
@@ -92,6 +72,8 @@ class WebhookSourceManager:
).exists()
)
+ return has_webhook_function
+
async def schema_is_webhook(self) -> bool:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
index dc4f2ee97048..9d8fbd3d0520 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py
@@ -1038,7 +1038,6 @@ def get_rows(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[StripeResumeConfig],
api_version: str,
- skip_full_listing: bool = False,
should_use_incremental_field: bool = False,
warehouse_parent: Optional["ParentTableRef"] = None,
include_zero_balance_customers: bool = False,
@@ -1095,14 +1094,6 @@ def get_rows(
or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)
or isinstance(resource, StripeNestedResource)
):
- if skip_full_listing:
- logger.warning(
- f"Stripe: skipped the {endpoint} sync. This table uses webhook sync, but the source has no "
- "active webhook. Set up the webhook on the source's Webhook tab, or change the table to a "
- "different sync method."
- )
- return
-
logger.debug(f"Stripe: iterating all objects from resource")
# Set only on the nested warehouse path; the final flush below is shared with flat
# resources, which have no parent sweep to checkpoint.
@@ -1686,11 +1677,6 @@ def stripe_source(
# re-enable — a poll could never rebuild the table.
webhook_only = endpoint in WEBHOOK_ONLY_ENDPOINTS
webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)(webhook_only=webhook_only)
- # A webhook table with no webhook falls back to a poll. Only a poll that lists the whole table
- # is skipped: a table that kept its `created` cursor still polls cheaply with `created[gt]`.
- webhook_unregistered = (
- not webhook_enabled and not webhook_only and async_to_sync(webhook_source_manager.webhook_unregistered)()
- )
warehouse_parent = _resolve_warehouse_parent(endpoint, team_id, source_id, use_warehouse_parent)
warehouse_table_id: Optional[str] = None
if not webhook_enabled:
@@ -1734,7 +1720,6 @@ def stripe_source(
logger=logger,
should_use_incremental_field=should_use_incremental_field,
resumable_source_manager=resumable_source_manager,
- skip_full_listing=webhook_unregistered,
api_version=api_version,
warehouse_parent=warehouse_parent,
include_zero_balance_customers=include_zero_balance_customers,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
index 3170affaa319..a315325d722d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
@@ -242,44 +242,6 @@ class TestStripeGetRowsIncrementalCursor:
assert [row["id"] for row in rows] == [f"ch_{i}" for i in range(5)]
assert len(tables) > 1
- @pytest.mark.parametrize(
- "skip_full_listing,last_value,expected_ids",
- [
- (True, None, []),
- (True, "1700000050", ["ch_2"]),
- (False, None, ["ch_2", "ch_1"]),
- ],
- )
- def test_skip_full_listing_keeps_the_incremental_poll(self, skip_full_listing, last_value, expected_ids):
- objects = [
- {"id": "ch_2", "created": 1700000100},
- {"id": "ch_1", "created": 1700000040},
- ]
- method = MagicMock(side_effect=lambda params: _FakeStripeList(objects))
- resumable_source_manager = mock.MagicMock()
- resumable_source_manager.can_resume.return_value = False
-
- with mock.patch.object(
- stripe_module, "_build_resources", return_value={"charge": StripeResource(method=method)}
- ):
- tables = list(
- get_rows(
- api_key="sk_test_123",
- endpoint="charge",
- account_id=None,
- db_incremental_field_last_value=last_value,
- db_incremental_field_earliest_value=None,
- logger=mock.MagicMock(),
- resumable_source_manager=resumable_source_manager,
- api_version=STRIPE_API_VERSION_ACACIA,
- should_use_incremental_field=True,
- skip_full_listing=skip_full_listing,
- )
- )
-
- assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids
- assert method.called is bool(expected_ids)
-
NOW = 1_800_000_000
@@ -1583,7 +1545,6 @@ class TestWebhookOnlyResponseWiring:
def _make_manager(self, enabled: bool) -> MagicMock:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=enabled)
- manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return manager
def _source(self, endpoint: str, manager: MagicMock) -> Any:
@@ -1613,18 +1574,6 @@ class TestWebhookOnlyResponseWiring:
assert response.webhook_only is False
manager.webhook_enabled.assert_awaited_once_with(webhook_only=False)
- @parameterized.expand(
- [("unregistered_webhook_skips_full_poll", True, []), ("no_webhook_schema_polls", False, ["cus_1"])]
- )
- def test_unregistered_webhook_poll(self, _name: str, unregistered: bool, expected_ids: list[str]) -> None:
- manager = self._make_manager(enabled=False)
- manager.webhook_unregistered = mock.AsyncMock(return_value=unregistered)
- resource = StripeResource(method=lambda params: cast(ListObject[Any], _FakeStripeList([{"id": "cus_1"}])))
- with mock.patch.object(stripe_module, "_build_resources", return_value={CUSTOMER_RESOURCE_NAME: resource}):
- response = self._source(CUSTOMER_RESOURCE_NAME, manager)
- tables = list(response.items())
- assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids
-
def test_discount_partitions_on_start_not_created(self) -> None:
# Discount objects carry `start`/`end`, not `created`. If the incremental-field entry is
# dropped the partition key falls back to "created" and the Delta partitioner KeyErrors on
@@ -1647,7 +1596,6 @@ class TestCustomerPaymentMethodHistory:
def _make_manager(self, enabled: bool) -> MagicMock:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=enabled)
- manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return manager
def _source(self, endpoint: str, manager: MagicMock) -> Any:
@@ -2064,7 +2012,6 @@ class TestWebhookRefetch:
)
webhooks = MagicMock(spec=WebhookSourceManager)
webhooks.webhook_enabled = mock.AsyncMock(return_value=True)
- webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)
events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])
with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):
@@ -2117,7 +2064,6 @@ class TestPartitioningAndColumnHints:
def _source(self, endpoint: str) -> Any:
manager = MagicMock(spec=WebhookSourceManager)
manager.webhook_enabled = mock.AsyncMock(return_value=False)
- manager.webhook_unregistered = mock.AsyncMock(return_value=False)
return stripe_module.stripe_source(
api_key="sk_test_123",
account_id=None,
@@ -2928,7 +2874,6 @@ class _PipelineAroundTheSweep:
)
webhooks = MagicMock(spec=WebhookSourceManager)
webhooks.webhook_enabled = mock.AsyncMock(return_value=False)
- webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)
with (
patch.object(stripe_module, "StripeClient"),
patch.object(stripe_module, "STRIPE_CHUNK_SIZE", 1),
diff --git a/products/warehouse_sources/backend/temporal/health_checks/__init__.py b/products/warehouse_sources/backend/temporal/health_checks/__init__.py
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py
deleted file mode 100644
index e1837eafb419..000000000000
--- a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py
+++ /dev/null
@@ -1,106 +0,0 @@
-from posthog.job_owners import JobOwners
-from posthog.models.health_issue import HealthIssue
-from posthog.temporal.health_checks.detectors import HealthExecutionPolicy
-from posthog.temporal.health_checks.framework import AlertContent, HealthCheck, Remediation
-from posthog.temporal.health_checks.models import HealthCheckResult
-
-from products.cdp.backend.facade.models import HogFunction
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-
-_MAX_LISTED = 20
-
-
-class WebhookNotRegisteredCheck(HealthCheck):
- """Flag a source whose tables use webhook sync but that has no active webhook.
-
- These tables receive no webhook data. What the sync does instead depends on the source: a
- webhook-only table gets no data, Stripe skips a poll that would list the whole table, and other
- sources poll the vendor API, often reading the whole table on every run.
- """
-
- name = "webhook_not_registered"
- kind = "webhook_not_registered"
- owner = JobOwners.TEAM_WAREHOUSE_SOURCES
- policy = HealthExecutionPolicy(batch_size=50, max_concurrent=3)
- schedule = "50 7 * * *"
- active_since_days = 30
- access_controlled_resource = "external_data_source"
- remediation = Remediation(
- human="""
- Some tables on this source use webhook sync, but the source has no active webhook, so
- these tables get no webhook updates. Depending on the table, the sync either skips it or
- reads it from the source API instead, which can be slow and can read the whole table on
- every sync. Open the source (Data pipeline / Data warehouse → Sources), go to the Webhook
- tab, and create the webhook. If you can't create a webhook for this account, change the
- affected tables to incremental or full refresh sync instead.
- """,
- agent="""
- Confirm with `external-data-sources-retrieve` that the listed schemas use webhook sync.
- Have the user create the webhook from the source's Webhook tab, or enable it if it is
- off. If the account can't have a webhook, change the affected schemas to another sync
- type with `external-data-schemas-partial-update`. The check clears once the source has
- an enabled webhook or no webhook-synced schemas.
- """,
- )
-
- @classmethod
- def render_alert(cls, issue: HealthIssue) -> AlertContent:
- name = issue.payload.get("pipeline_name") or issue.payload.get("source_type", "a webhook source")
- return AlertContent(
- title="Webhook isn't set up for webhook-synced tables",
- summary=f"{name} has tables set to webhook sync, but the source has no active webhook",
- link="/health/pipeline-status",
- )
-
- def detect(self, team_ids: list[int]) -> dict[int, list[HealthCheckResult]]:
- schemas = ExternalDataSchema.objects.filter(
- team_id__in=team_ids,
- deleted=False,
- should_sync=True,
- initial_sync_complete=True,
- sync_type=ExternalDataSchema.SyncType.WEBHOOK,
- ).values_list("source_id", "name")
-
- schema_names_by_source: dict[str, list[str]] = {}
- for source_id, schema_name in schemas:
- schema_names_by_source.setdefault(str(source_id), []).append(schema_name)
- if not schema_names_by_source:
- return {}
-
- sources_with_webhook = {
- str(source_id)
- for source_id in HogFunction.objects.filter(
- team_id__in=team_ids,
- type="warehouse_source_webhook",
- enabled=True,
- deleted=False,
- inputs__source_id__value__in=list(schema_names_by_source),
- ).values_list("inputs__source_id__value", flat=True)
- }
-
- sources = (
- ExternalDataSource.objects.filter(
- pk__in=[pk for pk in schema_names_by_source if pk not in sources_with_webhook],
- )
- .exclude(deleted=True)
- .values_list("pk", "team_id", "prefix", "source_type")
- )
-
- issues: dict[int, list[HealthCheckResult]] = {}
- for source_pk, team_id, prefix, source_type in sources:
- issues.setdefault(team_id, []).append(
- HealthCheckResult(
- severity=HealthIssue.Severity.WARNING,
- payload={
- "pipeline_type": "external_data_webhook",
- "pipeline_id": str(source_pk),
- "pipeline_name": prefix or source_type,
- "source_type": source_type,
- "affected_schemas": sorted(schema_names_by_source[str(source_pk)])[:_MAX_LISTED],
- },
- hash_keys=["pipeline_type", "pipeline_id"],
- )
- )
-
- return issues
diff --git a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py
deleted file mode 100644
index 8c03fcfb8455..000000000000
--- a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py
+++ /dev/null
@@ -1,58 +0,0 @@
-import uuid
-
-from posthog.test.base import BaseTest
-
-from parameterized import parameterized
-
-from products.cdp.backend.models.hog_functions.hog_function import HogFunction
-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-from products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered import WebhookNotRegisteredCheck
-
-
-class TestWebhookNotRegisteredCheck(BaseTest):
- @parameterized.expand(
- [
- ("no_webhook_function", None, True, True),
- ("disabled_webhook_function", False, True, True),
- ("enabled_webhook_function", True, True, False),
- ("initial_sync_not_complete", None, False, False),
- ]
- )
- def test_flags_webhook_schemas_without_an_enabled_webhook(
- self, _name: str, webhook_enabled: bool | None, initial_sync_complete: bool, flagged: bool
- ) -> None:
- source = ExternalDataSource.objects.create(
- source_id=str(uuid.uuid4()),
- connection_id=str(uuid.uuid4()),
- team=self.team,
- source_type="Stripe",
- job_inputs={"stripe_secret_key": "sk_test_x"},
- )
- if webhook_enabled is not None:
- HogFunction.objects.create(
- team=self.team,
- type="warehouse_source_webhook",
- hog="return event",
- enabled=webhook_enabled,
- inputs_schema=[{"key": "source_id", "type": "string"}],
- inputs={"source_id": {"value": str(source.pk)}},
- )
- ExternalDataSchema.objects.create(
- name="Customer",
- team=self.team,
- source=source,
- sync_type=ExternalDataSchema.SyncType.WEBHOOK,
- should_sync=True,
- initial_sync_complete=initial_sync_complete,
- )
-
- issues = WebhookNotRegisteredCheck().detect([self.team.id])
- payloads = [result.payload for result in issues.get(self.team.id, [])]
-
- if flagged:
- assert len(payloads) == 1
- assert payloads[0]["pipeline_id"] == str(source.pk)
- assert payloads[0]["affected_schemas"] == ["Customer"]
- else:
- assert payloads == []