:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): revert poll skip for stripe tables with no webhook (#128998)
Tom Owersb0658f19f955
diff --git a/posthog/temporal/health_checks/registry.py b/posthog/temporal/health_checks/registry.pyindex e261b9b56e8a..2dfd983439a3 100644--- a/posthog/temporal/health_checks/registry.py+++ b/posthog/temporal/health_checks/registry.py@@ -21,7 +21,6 @@ HEALTH_CHECK_MODULES = ["products.cdp.backend.temporal.health_checks.ingestion_warnings","products.data_warehouse.backend.temporal.health_checks.materialized_view_failure","products.data_warehouse.backend.temporal.health_checks.webhook_subscription_stale",- "products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered","products.web_analytics.backend.temporal.health_checks.scroll_depth","products.web_analytics.backend.temporal.health_checks.authorized_urls","products.web_analytics.backend.temporal.health_checks.reverse_proxy",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.pyindex e2f2e5c0b138..785a86621b92 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test/test_webhook_s3.py@@ -171,32 +171,6 @@ class TestWebhookSourceManager:):assert await manager.webhook_enabled(webhook_only) is expected- @parameterized.expand(- [- ("no_hog_function", False, True, True, False, True),- ("has_hog_function", True, True, True, False, False),- ("initial_sync_not_complete", False, True, False, False, False),- ("reset_pipeline", False, True, True, True, False),- ("not_webhook", False, False, True, False, False),- ]- )- async def test_webhook_unregistered_conditions(- self, _name, has_webhook_function, is_webhook, initial_sync_complete, reset_pipeline, expected- ):- manager = _make_manager(reset_pipeline=reset_pipeline)-- mock_schema = MagicMock()- mock_schema.is_webhook = is_webhook- mock_schema.initial_sync_complete = initial_sync_complete-- results = iter([mock_schema, has_webhook_function])-- with patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3.database_sync_to_async_pool",- side_effect=lambda fn: AsyncMock(return_value=next(results)),- ):- assert await manager.webhook_unregistered() is expected-async def test_list_parquet_files_filters_correctly(self):manager = _make_manager()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.pyindex 7c8400854b8a..13160963a017 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/testing/driver.py@@ -199,7 +199,6 @@ class SourceDriver:_REAL_WEBHOOK_CHECKS = {"webhook_enabled": WebhookSourceManager.webhook_enabled,"schema_is_webhook": WebhookSourceManager.schema_is_webhook,- "webhook_unregistered": WebhookSourceManager.webhook_unregistered,}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.pyindex f616c5a9fc34..41e54b66a516 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/webhook_s3.py@@ -40,6 +40,7 @@ class WebhookSourceManager:return s3_path.replace("s3://", "")async def webhook_enabled(self, webhook_only: bool = False) -> bool:+ from products.cdp.backend.models.hog_functions.hog_function import HogFunctionfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemaschema = await database_sync_to_async_pool(db_read_with_retry)(@@ -61,28 +62,7 @@ class WebhookSourceManager:)return False- return await self._has_enabled_webhook_function()-- async def webhook_unregistered(self) -> bool:- """Whether a webhook schema is past its initial sync but its source has no enabled webhook.-- Such a schema falls back to a poll. A schema created with webhook sync has no stored cursor,- so that poll re-reads the whole table on every run. The caller decides whether to skip it.- """- from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema-- schema = await database_sync_to_async_pool(db_read_with_retry)(- lambda: ExternalDataSchema.objects.get(id=self._inputs.schema_id, team_id=self._inputs.team_id)- )- if not schema.is_webhook or not schema.initial_sync_complete or self._inputs.reset_pipeline:- return False-- return not await self._has_enabled_webhook_function()-- async def _has_enabled_webhook_function(self) -> bool:- from products.cdp.backend.models.hog_functions.hog_function import HogFunction-- return await database_sync_to_async_pool(db_read_with_retry)(+ has_webhook_function = await database_sync_to_async_pool(db_read_with_retry)(lambda: HogFunction.objects.filter(inputs__source_id__value=self._inputs.source_id,team_id=self._inputs.team_id,@@ -92,6 +72,8 @@ class WebhookSourceManager:).exists())+ return has_webhook_function+async def schema_is_webhook(self) -> bool:from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemadiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.pyindex dc4f2ee97048..9d8fbd3d0520 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/stripe.py@@ -1038,7 +1038,6 @@ def get_rows(logger: FilteringBoundLogger,resumable_source_manager: ResumableSourceManager[StripeResumeConfig],api_version: str,- skip_full_listing: bool = False,should_use_incremental_field: bool = False,warehouse_parent: Optional["ParentTableRef"] = None,include_zero_balance_customers: bool = False,@@ -1095,14 +1094,6 @@ def get_rows(or (db_incremental_field_last_value is None and db_incremental_field_earliest_value is None)or isinstance(resource, StripeNestedResource)):- if skip_full_listing:- logger.warning(- f"Stripe: skipped the {endpoint} sync. This table uses webhook sync, but the source has no "- "active webhook. Set up the webhook on the source's Webhook tab, or change the table to a "- "different sync method."- )- return-logger.debug(f"Stripe: iterating all objects from resource")# Set only on the nested warehouse path; the final flush below is shared with flat# resources, which have no parent sweep to checkpoint.@@ -1686,11 +1677,6 @@ def stripe_source(# re-enable — a poll could never rebuild the table.webhook_only = endpoint in WEBHOOK_ONLY_ENDPOINTSwebhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)(webhook_only=webhook_only)- # A webhook table with no webhook falls back to a poll. Only a poll that lists the whole table- # is skipped: a table that kept its `created` cursor still polls cheaply with `created[gt]`.- webhook_unregistered = (- not webhook_enabled and not webhook_only and async_to_sync(webhook_source_manager.webhook_unregistered)()- )warehouse_parent = _resolve_warehouse_parent(endpoint, team_id, source_id, use_warehouse_parent)warehouse_table_id: Optional[str] = Noneif not webhook_enabled:@@ -1734,7 +1720,6 @@ def stripe_source(logger=logger,should_use_incremental_field=should_use_incremental_field,resumable_source_manager=resumable_source_manager,- skip_full_listing=webhook_unregistered,api_version=api_version,warehouse_parent=warehouse_parent,include_zero_balance_customers=include_zero_balance_customers,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.pyindex 3170affaa319..a315325d722d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py@@ -242,44 +242,6 @@ class TestStripeGetRowsIncrementalCursor:assert [row["id"] for row in rows] == [f"ch_{i}" for i in range(5)]assert len(tables) > 1- @pytest.mark.parametrize(- "skip_full_listing,last_value,expected_ids",- [- (True, None, []),- (True, "1700000050", ["ch_2"]),- (False, None, ["ch_2", "ch_1"]),- ],- )- def test_skip_full_listing_keeps_the_incremental_poll(self, skip_full_listing, last_value, expected_ids):- objects = [- {"id": "ch_2", "created": 1700000100},- {"id": "ch_1", "created": 1700000040},- ]- method = MagicMock(side_effect=lambda params: _FakeStripeList(objects))- resumable_source_manager = mock.MagicMock()- resumable_source_manager.can_resume.return_value = False-- with mock.patch.object(- stripe_module, "_build_resources", return_value={"charge": StripeResource(method=method)}- ):- tables = list(- get_rows(- api_key="sk_test_123",- endpoint="charge",- account_id=None,- db_incremental_field_last_value=last_value,- db_incremental_field_earliest_value=None,- logger=mock.MagicMock(),- resumable_source_manager=resumable_source_manager,- api_version=STRIPE_API_VERSION_ACACIA,- should_use_incremental_field=True,- skip_full_listing=skip_full_listing,- )- )-- assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids- assert method.called is bool(expected_ids)-NOW = 1_800_000_000@@ -1583,7 +1545,6 @@ class TestWebhookOnlyResponseWiring:def _make_manager(self, enabled: bool) -> MagicMock:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=enabled)- manager.webhook_unregistered = mock.AsyncMock(return_value=False)return managerdef _source(self, endpoint: str, manager: MagicMock) -> Any:@@ -1613,18 +1574,6 @@ class TestWebhookOnlyResponseWiring:assert response.webhook_only is Falsemanager.webhook_enabled.assert_awaited_once_with(webhook_only=False)- @parameterized.expand(- [("unregistered_webhook_skips_full_poll", True, []), ("no_webhook_schema_polls", False, ["cus_1"])]- )- def test_unregistered_webhook_poll(self, _name: str, unregistered: bool, expected_ids: list[str]) -> None:- manager = self._make_manager(enabled=False)- manager.webhook_unregistered = mock.AsyncMock(return_value=unregistered)- resource = StripeResource(method=lambda params: cast(ListObject[Any], _FakeStripeList([{"id": "cus_1"}])))- with mock.patch.object(stripe_module, "_build_resources", return_value={CUSTOMER_RESOURCE_NAME: resource}):- response = self._source(CUSTOMER_RESOURCE_NAME, manager)- tables = list(response.items())- assert [row["id"] for table in tables for row in table.to_pylist()] == expected_ids-def test_discount_partitions_on_start_not_created(self) -> None:# Discount objects carry `start`/`end`, not `created`. If the incremental-field entry is# dropped the partition key falls back to "created" and the Delta partitioner KeyErrors on@@ -1647,7 +1596,6 @@ class TestCustomerPaymentMethodHistory:def _make_manager(self, enabled: bool) -> MagicMock:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=enabled)- manager.webhook_unregistered = mock.AsyncMock(return_value=False)return managerdef _source(self, endpoint: str, manager: MagicMock) -> Any:@@ -2064,7 +2012,6 @@ class TestWebhookRefetch:)webhooks = MagicMock(spec=WebhookSourceManager)webhooks.webhook_enabled = mock.AsyncMock(return_value=True)- webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)events = table_from_py_list([_event_row("evt_1", "invoice.paid", 1700000100, _INCOMPLETE_INVOICE)])with scripted_network(lambda request: ScriptedResponse(json=_stripe_list(_ALL_LINES, request.path))):@@ -2117,7 +2064,6 @@ class TestPartitioningAndColumnHints:def _source(self, endpoint: str) -> Any:manager = MagicMock(spec=WebhookSourceManager)manager.webhook_enabled = mock.AsyncMock(return_value=False)- manager.webhook_unregistered = mock.AsyncMock(return_value=False)return stripe_module.stripe_source(api_key="sk_test_123",account_id=None,@@ -2928,7 +2874,6 @@ class _PipelineAroundTheSweep:)webhooks = MagicMock(spec=WebhookSourceManager)webhooks.webhook_enabled = mock.AsyncMock(return_value=False)- webhooks.webhook_unregistered = mock.AsyncMock(return_value=False)with (patch.object(stripe_module, "StripeClient"),patch.object(stripe_module, "STRIPE_CHUNK_SIZE", 1),diff --git a/products/warehouse_sources/backend/temporal/health_checks/__init__.py b/products/warehouse_sources/backend/temporal/health_checks/__init__.pydeleted file mode 100644index e69de29bb2d1..000000000000diff --git a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py b/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.pydeleted file mode 100644index e1837eafb419..000000000000--- a/products/warehouse_sources/backend/temporal/health_checks/webhook_not_registered.py+++ /dev/null@@ -1,106 +0,0 @@-from posthog.job_owners import JobOwners-from posthog.models.health_issue import HealthIssue-from posthog.temporal.health_checks.detectors import HealthExecutionPolicy-from posthog.temporal.health_checks.framework import AlertContent, HealthCheck, Remediation-from posthog.temporal.health_checks.models import HealthCheckResult--from products.cdp.backend.facade.models import HogFunction-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource--_MAX_LISTED = 20---class WebhookNotRegisteredCheck(HealthCheck):- """Flag a source whose tables use webhook sync but that has no active webhook.-- These tables receive no webhook data. What the sync does instead depends on the source: a- webhook-only table gets no data, Stripe skips a poll that would list the whole table, and other- sources poll the vendor API, often reading the whole table on every run.- """-- name = "webhook_not_registered"- kind = "webhook_not_registered"- owner = JobOwners.TEAM_WAREHOUSE_SOURCES- policy = HealthExecutionPolicy(batch_size=50, max_concurrent=3)- schedule = "50 7 * * *"- active_since_days = 30- access_controlled_resource = "external_data_source"- remediation = Remediation(- human="""- Some tables on this source use webhook sync, but the source has no active webhook, so- these tables get no webhook updates. Depending on the table, the sync either skips it or- reads it from the source API instead, which can be slow and can read the whole table on- every sync. Open the source (Data pipeline / Data warehouse → Sources), go to the Webhook- tab, and create the webhook. If you can't create a webhook for this account, change the- affected tables to incremental or full refresh sync instead.- """,- agent="""- Confirm with `external-data-sources-retrieve` that the listed schemas use webhook sync.- Have the user create the webhook from the source's Webhook tab, or enable it if it is- off. If the account can't have a webhook, change the affected schemas to another sync- type with `external-data-schemas-partial-update`. The check clears once the source has- an enabled webhook or no webhook-synced schemas.- """,- )-- @classmethod- def render_alert(cls, issue: HealthIssue) -> AlertContent:- name = issue.payload.get("pipeline_name") or issue.payload.get("source_type", "a webhook source")- return AlertContent(- title="Webhook isn't set up for webhook-synced tables",- summary=f"{name} has tables set to webhook sync, but the source has no active webhook",- link="/health/pipeline-status",- )-- def detect(self, team_ids: list[int]) -> dict[int, list[HealthCheckResult]]:- schemas = ExternalDataSchema.objects.filter(- team_id__in=team_ids,- deleted=False,- should_sync=True,- initial_sync_complete=True,- sync_type=ExternalDataSchema.SyncType.WEBHOOK,- ).values_list("source_id", "name")-- schema_names_by_source: dict[str, list[str]] = {}- for source_id, schema_name in schemas:- schema_names_by_source.setdefault(str(source_id), []).append(schema_name)- if not schema_names_by_source:- return {}-- sources_with_webhook = {- str(source_id)- for source_id in HogFunction.objects.filter(- team_id__in=team_ids,- type="warehouse_source_webhook",- enabled=True,- deleted=False,- inputs__source_id__value__in=list(schema_names_by_source),- ).values_list("inputs__source_id__value", flat=True)- }-- sources = (- ExternalDataSource.objects.filter(- pk__in=[pk for pk in schema_names_by_source if pk not in sources_with_webhook],- )- .exclude(deleted=True)- .values_list("pk", "team_id", "prefix", "source_type")- )-- issues: dict[int, list[HealthCheckResult]] = {}- for source_pk, team_id, prefix, source_type in sources:- issues.setdefault(team_id, []).append(- HealthCheckResult(- severity=HealthIssue.Severity.WARNING,- payload={- "pipeline_type": "external_data_webhook",- "pipeline_id": str(source_pk),- "pipeline_name": prefix or source_type,- "source_type": source_type,- "affected_schemas": sorted(schema_names_by_source[str(source_pk)])[:_MAX_LISTED],- },- hash_keys=["pipeline_type", "pipeline_id"],- )- )-- return issuesdiff --git a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py b/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.pydeleted file mode 100644index 8c03fcfb8455..000000000000--- a/products/warehouse_sources/backend/tests/test_webhook_not_registered_check.py+++ /dev/null@@ -1,58 +0,0 @@-import uuid--from posthog.test.base import BaseTest--from parameterized import parameterized--from products.cdp.backend.models.hog_functions.hog_function import HogFunction-from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema-from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource-from products.warehouse_sources.backend.temporal.health_checks.webhook_not_registered import WebhookNotRegisteredCheck---class TestWebhookNotRegisteredCheck(BaseTest):- @parameterized.expand(- [- ("no_webhook_function", None, True, True),- ("disabled_webhook_function", False, True, True),- ("enabled_webhook_function", True, True, False),- ("initial_sync_not_complete", None, False, False),- ]- )- def test_flags_webhook_schemas_without_an_enabled_webhook(- self, _name: str, webhook_enabled: bool | None, initial_sync_complete: bool, flagged: bool- ) -> None:- source = ExternalDataSource.objects.create(- source_id=str(uuid.uuid4()),- connection_id=str(uuid.uuid4()),- team=self.team,- source_type="Stripe",- job_inputs={"stripe_secret_key": "sk_test_x"},- )- if webhook_enabled is not None:- HogFunction.objects.create(- team=self.team,- type="warehouse_source_webhook",- hog="return event",- enabled=webhook_enabled,- inputs_schema=[{"key": "source_id", "type": "string"}],- inputs={"source_id": {"value": str(source.pk)}},- )- ExternalDataSchema.objects.create(- name="Customer",- team=self.team,- source=source,- sync_type=ExternalDataSchema.SyncType.WEBHOOK,- should_sync=True,- initial_sync_complete=initial_sync_complete,- )-- issues = WebhookNotRegisteredCheck().detect([self.team.id])- payloads = [result.payload for result in issues.get(self.team.id, [])]-- if flagged:- assert len(payloads) == 1- assert payloads[0]["pipeline_id"] == str(source.pk)- assert payloads[0]["affected_schemas"] == ["Customer"]- else:- assert payloads == []