:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): give control back during empty pages in 71 sources (#127475)
Tom Owers35af22be7703
diff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txtindex 2cdec71af8e6..172d5fe6c04b 100644--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt@@ -4,7 +4,6 @@ adjust/adjust.py::get_rowsadobe_analytics/adobe_analytics.py::_metadata_batchesadobe_analytics/adobe_analytics.py::_report_batchesadobe_commerce/adobe_commerce.py::get_rows-adyen/adyen.py::_iter_cursor_pagesadyen/adyen.py::_iter_fanout_pagesadyen/adyen.py::_iter_offset_pagesadyen/adyen.py::_iter_page_number_pages@@ -31,7 +30,6 @@ appfollow/appfollow.py::_get_app_fanoutappfollow/appfollow.py::_get_ratingsappfollow/appfollow.py::_get_reviewsapple_search_ads/apple_search_ads.py::_iter_rows-applovin/applovin.py::get_rowsappsignal/appsignal.py::_get_incident_rowsappsignal/appsignal.py::_get_log_line_rowsappsignal/appsignal.py::_get_metric_timeseries_rows@@ -92,8 +90,6 @@ codecov/codecov.py::_get_top_level_rowscody/cody.py::_get_windowed_rowscoin_api/coin_api.py::_get_timeseries_rowscoingecko/coingecko.py::_chart_pages-coinmarketcap/coinmarketcap.py::get_batch_rows-coinmarketcap/coinmarketcap.py::get_snapshot_rowscommercetools/commercetools.py::get_rowscommon/rest_source/rest_client.py::RESTClient.paginatecommon/sql/keyset.py::iter_keyset_pages@@ -114,7 +110,6 @@ cronitor/cronitor.py::_get_paginated_rowscronitor/cronitor.py::_get_site_error_rowscrossref/crossref.py::get_rowscursor/cursor.py::_get_spend_rows-cursor/cursor.py::_paginate_windowcustomerly/customerly.py::_get_article_rowsdagster_cloud/dagster_cloud.py::_iter_fanout_rowsdagster_cloud/dagster_cloud.py::_iter_insights_rows@@ -132,9 +127,6 @@ deepsource/deepsource.py::_root_connection_rowsdescope/descope.py::_user_history_rowsdixa/dixa.py::_fan_out_rowsdixa/dixa.py::get_rows-dockerhub/dockerhub.py::_audit_log_pages-dockerhub/dockerhub.py::_cursor_pages-dockerhub/dockerhub.py::_tag_pagesdocusign/docusign.py::get_rowsdoppler/doppler.py::_get_fan_out_rowsdoppler/doppler.py::get_rows@@ -163,7 +155,6 @@ firebase/firebase.py::iter_realtime_databasefireworks_ai/fireworks_ai.py::_iter_usage_windowsflagsmith/flagsmith.py::_paginate_resourcefloat_app/float_app.py::_report_window_pages-formbricks/formbricks.py::get_rowsfred/fred.py::get_rowsfreshsales/freshsales.py::_get_paginated_fanout_rowsfreshsales/freshsales.py::get_rows@@ -243,7 +234,6 @@ klaviyo/klaviyo.py::_get_fan_out_rowsklaviyo/klaviyo.py::get_rowskong_konnect/kong_konnect.py::get_rowskubecost/kubecost.py::get_rows-lacework/lacework.py::get_rowslangfuse/langfuse.py::get_rowslangsmith/langsmith.py::_emit_batcheslinear/linear.py::_make_paginated_request@@ -261,13 +251,9 @@ marketo/marketo.py::_asset_offset_rowsmarketo/marketo.py::_bulk_rowsmarketo/marketo.py::_rest_token_rowsmatomo/matomo.py::get_rows-mention/mention.py::_alert_rows-mention/mention.py::_alert_tag_rows-mention/mention.py::_mention_rowsmeta_ads/meta_ads.py::_iter_simple_paginationmeta_ads/meta_ads.py::_iter_time_range_paginationmetaplane/metaplane.py::_get_evaluation_rows-meteostat/meteostat.py::_get_rowsmistral_ai/mistral_ai.py::get_rowsmixpanel/mixpanel.py::_iter_engagemixpanel/mixpanel.py::_iter_export@@ -282,7 +268,6 @@ new_relic/new_relic.py::_get_event_rowsnew_york_times/new_york_times.py::_get_article_search_rowsnotion/notion.py::_page_fan_outnotion/notion.py::_search_stream-npm_registry/npm_registry.py::_iter_downloadsnpm_registry/npm_registry.py::get_rowsnuget/nuget.py::_get_catalog_event_rowsnuget/nuget.py::_get_package_rows@@ -298,7 +283,6 @@ openrouter/openrouter.py::_get_offset_rowsortto/ortto.py::get_rowsoutbrain/outbrain.py::get_rowspabbly_subscriptions_billing/pabbly_subscriptions_billing.py::_iter_fan_out-pabbly_subscriptions_billing/pabbly_subscriptions_billing.py::_iter_top_levelpackagist/packagist.py::get_rowspardot/pardot.py::get_rowspayfit/payfit.py::_get_payslip_rows@@ -311,7 +295,6 @@ pinterest_ads/pinterest_ads.py::_iter_fanned_out_analyticsplatform_sh/platform_sh.py::get_rowspolar/polar.py::get_rowspostgres/postgres.py::postgres_source.get_rows.offset_chunking-power_bi_admin/power_bi_admin.py::_activity_event_pagespower_bi_admin/power_bi_admin.py::_odata_pagesprintify/printify.py::_iter_endpoint_pagesprofound/profound.py::_report_pages@@ -320,9 +303,6 @@ pulumi_cloud/pulumi_cloud.py::_get_deployment_rowspulumi_cloud/pulumi_cloud.py::_get_resource_rowspulumi_cloud/pulumi_cloud.py::_get_stack_rowspulumi_cloud/pulumi_cloud.py::_get_stack_update_rows-pylon/pylon.py::_get_fan_out_rows-pylon/pylon.py::_get_simple_rows-pylon/pylon.py::_get_windowed_rowsqualtrics/qualtrics.py::_iter_collectionqualtrics/qualtrics.py::get_rowsqualys_vmdr/qualys_vmdr.py::get_rows@@ -333,9 +313,6 @@ render/render.py::_get_top_level_rowsreplicate/replicate.py::get_rowsrokt_ads/rokt_ads.py::rokt_ads_sourcerollbar/rollbar.py::get_rows-runpod/runpod.py::_iter_billing_rows-sage_hr/sage_hr.py::_get_windowed_rows-sage_hr/sage_hr.py::get_rowsscaleway/scaleway.py::get_rowssegment/segment.py::get_rowssentry/sentry.py::_iter_issue_tag_values_rows@@ -346,9 +323,6 @@ signoz/signoz.py::get_rowssimilarweb/similarweb.py::_paginated_rowssingular/singular.py::_report_rowsslack/slack.py::_channel_messages_generator-smaily/smaily.py::_campaign_statistics_rows-smaily/smaily.py::_list_rows-smaily/smaily.py::_segment_subscriber_rowssnapchat_ads/snapchat_ads.py::_iter_rowssnowplow/snowplow.py::_get_failed_event_metric_rowssnowplow/snowplow.py::_get_windowed_job_rows@@ -376,9 +350,7 @@ sumo_logic/sumo_logic.py::_get_collector_fan_out_rowssumo_logic/sumo_logic.py::_get_log_rowssumo_logic/sumo_logic.py::_get_offset_paginated_rowssumo_logic/sumo_logic.py::_get_token_paginated_rows-swarmia/swarmia.py::get_rowstaboola/taboola.py::get_rows-tailscale/tailscale.py::_audit_log_rowstawk_to/tawk_to.py::_paged_property_rowstawk_to/tawk_to.py::get_rowsteamcity/teamcity.py::_fan_out_rows@@ -406,9 +378,7 @@ vercel/vercel.py::get_rowsweights_and_biases/weights_and_biases.py::_get_fan_out_rowsweights_and_biases/weights_and_biases.py::_get_project_rowsweights_and_biases/weights_and_biases.py::get_rows-wikipedia_pageviews/wikipedia_pageviews.py::_get_rowswordpress/wordpress.py::get_rows-writesonic/writesonic.py::_iter_configwritesonic/writesonic.py::_iter_dailyyoutube_analytics/youtube_analytics.py::get_rowszendesk_sunshine/zendesk_sunshine.py::_object_records_incrementaldiff --git a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txtindex c832115fe4f9..ee2aeff8e6e2 100644--- a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt+++ b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt@@ -5,10 +5,8 @@ ashby/ashby.py::_iter_pagesassemblyai/assemblyai.py::assemblyai_source.itemsaviationstack/aviationstack.py::aviationstack_source.itemsbrowse_ai/browse_ai.py::browse_ai_source.rows-cal_com/cal_com.py::_booking_attendees_itemscalendly/calendly.py::calendly_source.get_rowscallrail/callrail.py::get_rows-canvas_lms/canvas_lms.py::_non_fanout_source.itemschargedesk/chargedesk.py::_run_fanout_passchargedesk/chargedesk.py::_run_passcheckly/checkly.py::checkly_source.items@@ -38,21 +36,16 @@ noaa_cdo/noaa_cdo.py::noaa_cdo_source.itemsokendo/okendo.py::okendo_source.itemsokta/okta.py::okta_source.itemspaymongo/paymongo.py::pull_rows-pipeliner/pipeliner.py::pipeliner_source.items-plunk/plunk.py::plunk_source.itemspoplar/poplar.py::poplar_source.rowsprofound/profound.py::_report_pagespromptwatch/promptwatch.py::promptwatch_sourceretell_ai/retell_ai.py::retell_ai_source.items-revenuecat/revenuecat.py::revenuecat_api_source.itemssentinelone/sentinelone.py::sentinelone_source._itemssentry/sentry.py::_skip_endpoint_on_forbidden-sevdesk/sevdesk.py::sevdesk_source.get_rowsshutterstock/shutterstock.py::get_rowstelli/telli.py::telli_source.get_rowstwenty/twenty.py::twenty_source.itemstwitter/twitter.py::_iter_pages-unleash/unleash.py::unleash_source.itemsvitally/vitally.py::vitally_sourcewho_gho/who_gho.py::_catalog_pageswho_gho/who_gho.py::_dimension_values_pagesdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.pyindex 7662122b4418..369e5949c360 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py@@ -1,5 +1,5 @@import dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetime, timedelta, tzinfofrom email.utils import parsedate_to_datetimefrom typing import Any, Optional@@ -382,8 +382,12 @@ def _iter_resource_ids(logger: FilteringBoundLogger,path: str,page_size: int,+ safe_point: Callable[[], None] | None = None,) -> Iterator[str]:- """Page through a Klaviyo collection and yield each row's id, following the cursor links."""+ """Page through a Klaviyo collection and yield each row's id, following the cursor links.++ `safe_point` runs after each page. Pass it only when the caller buffers no rows.+ """# Some collections (e.g. /object-types) reject page[size]; page_size <= 0 means "omit it and# page by cursor links alone", mirroring _build_initial_params for top-level endpoints.params = {"page[size]": page_size} if page_size > 0 else {}@@ -392,6 +396,8 @@ def _iter_resource_ids(data = _fetch_page(session, url, headers, logger)for item in data.get("data", []):yield item["id"]+ if safe_point is not None:+ safe_point()next_url = data.get("links", {}).get("next")if not next_url:@@ -404,6 +410,7 @@ def _iter_fan_out_parents(headers: dict[str, str],logger: FilteringBoundLogger,fan_out: KlaviyoFanOutConfig,+ safe_point: Callable[[], None],) -> Iterator[tuple[dict[str, str], str]]:"""Yield (ancestor columns, parent id) for every parent the child endpoint fans out over.@@ -412,17 +419,19 @@ def _iter_fan_out_parents(formats the parent path with each grandparent id, tagging every parent with that id."""if fan_out.grandparent is None:- for parent_id in _iter_resource_ids(session, headers, logger, fan_out.parent_path, fan_out.parent_page_size):+ for parent_id in _iter_resource_ids(+ session, headers, logger, fan_out.parent_path, fan_out.parent_page_size, safe_point+ ):yield {}, parent_idreturngrandparent = fan_out.grandparentfor grandparent_id in _iter_resource_ids(- session, headers, logger, grandparent.parent_path, grandparent.parent_page_size+ session, headers, logger, grandparent.parent_path, grandparent.parent_page_size, safe_point):path = fan_out.parent_path.format(**{grandparent.parent_id_column: grandparent_id})try:- for parent_id in _iter_resource_ids(session, headers, logger, path, fan_out.parent_page_size):+ for parent_id in _iter_resource_ids(session, headers, logger, path, fan_out.parent_page_size, safe_point):yield {grandparent.parent_id_column: grandparent_id}, parent_idexcept requests.HTTPError as exc:# A grandparent deleted between enumeration and this fetch 404s; its children are gone@@ -476,7 +485,8 @@ def _get_fan_out_rows("""fan_out = config.fan_outassert fan_out is not None- parents = list(_iter_fan_out_parents(session, headers, logger, fan_out))+ # Parents are listed before any row is batched, so each parent page is a safe point.+ parents = list(_iter_fan_out_parents(session, headers, logger, fan_out, resumable_source_manager.safe_point))parent_ids = [parent_id for _, parent_id in parents]# Resolve the saved parent-ID bookmark to the slice of parents still to process. If the@@ -515,6 +525,9 @@ def _get_fan_out_rows(KlaviyoResumeConfig(next_url=next_url, list_id=parent_id))+ if not batcher.should_yield(include_incomplete_chunk=True):+ resumable_source_manager.safe_point()+if not next_url:breakurl = next_urldiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.pyindex 97fad6640413..972e1e7ad49f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py@@ -10,6 +10,7 @@ it is redacted from logs and raised error messages.import dataclassesfrom collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetime+from functools import partialfrom typing import Any, Optionalfrom urllib.parse import urljoin, urlparse@@ -29,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resJSONResponsePaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (ClientConfig,EndpointResource,@@ -321,7 +323,7 @@ def _memories_pages(db_incremental_field_last_value: Any,resume_hook: Optional[Callable[[Optional[dict[str, Any]]], None]] = None,initial_paginator_state: dict[str, Any] | None = None,-) -> Iterator[list[Any]]:+) -> Resource:config = MEM0_ENDPOINTS[MEMORIES_ENDPOINT]rest_config: RESTAPIConfig = {@@ -351,7 +353,13 @@ def _memories_pages(initial_paginator_state=initial_paginator_state,)- return _pages_until_page_rejected(resource, resumed=initial_paginator_state is not None)+ # The rejection guard hands each page on unchanged and holds no rows. As a `Resource` it keeps+ # the framework's per-page safe points.+ return Resource(+ partial(_pages_until_page_rejected, resource, resumed=initial_paginator_state is not None),+ name=MEMORIES_ENDPOINT,+ hints=resource._hints,+ )def _entities_resource(@@ -455,7 +463,7 @@ def _memory_history_resource({"name": MEMORIES_ENDPOINT,"endpoint": {"path": MEM0_ENDPOINTS[MEMORIES_ENDPOINT].path, "method": "POST"},- "data_iterator": lambda: _memories_pages(client, team_id, job_id, filters, None),+ "data_iterator": lambda: iter(_memories_pages(client, team_id, job_id, filters, None)),},{"name": MEMORY_HISTORY_ENDPOINT,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.pyindex aade691b8edb..477b824c0984 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py@@ -34,6 +34,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resPageNumberPaginator,SinglePagePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpointfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -165,10 +166,9 @@ def _make_bounded_session(return session[email protected](frozen=True)class PlunkResumeConfig:- # Exactly one of these is set, matching the endpoint's pagination style. Persisted after each- # page is yielded, so a crash before the write re-yields the last page (merge dedupes on `id`).+ # Exactly one of these is set, matching the endpoint's pagination style.cursor: str = ""page: int = 0@@ -309,8 +309,7 @@ def plunk_source(initial_paginator_state = {"page": resume.page}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist only when a next page remains; the hook fires AFTER a page is yielded so a crash- # re-yields the last page (merge dedupes on the primary key) rather than skipping it.+ # Persist only when a next page remains. The framework stages it with the page it covers.if not state:returnif state.get("cursor"):@@ -348,7 +347,7 @@ def plunk_source(],}- def items() -> Any:+ def items() -> Resource:# Re-check at run time (not just at source-create) in case the URL was edited or now# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse# plaintext HTTP before the key is used. Both raise before any request leaves the process.@@ -358,7 +357,7 @@ def plunk_source(if not _is_https(resolved_base_url):raise PlunkHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)- yield from rest_api_resource(+ return rest_api_resource(rest_config,team_id,job_id,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.pyindex 7d74ffab858c..541e4ad51649 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponsePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (@@ -407,15 +408,21 @@ def revenuecat_api_source(initial_paginator_state=initial_paginator_state,)- def items() -> Iterator[list[dict[str, Any]]]:+ def dict_rows() -> Iterator[list[dict[str, Any]]]:# The framework passes non-dict page entries through untouched (data_map only runs on# dicts), so drop them here — matching the old client's per-row dict guard — before a# malformed row can reach the delta writer.for page in resource:yield [row for row in page if isinstance(row, dict)]+ # The framework staged the cursor after this page before it handed the page on, and the+ # guard holds no rows, so the run can stop here.+ resumable_source_manager.safe_point()++ # As a `Resource` the guard also keeps the framework's safe points before each retry wait.+ pages = Resource(dict_rows, name=schema_name, hints=resource._hints)return SourceResponse(- items=items,+ items=lambda: pages,primary_keys=endpoint.primary_keys,name=schema_name,partition_keys=endpoint.partition_keys,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex 45d5d308b3e9..eba4f8224e68 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -1,67 +1,39 @@-Adjust fail:safe-point-Adyen fail:safe-pointAmazonS3 not-checkable:schemas-Anthropic fail:safe-point-AppLovin fail:safe-pointAppStoreConnect not-checkable:no-requestAppleSearchAds not-checkable:no-requestAppsFlyer fail:safe-point-Appsignal fail:safe-pointAttentive not-checkable:databaseAwsAthena not-checkable:schemas-AwsCostExplorer fail:safe-pointAwsDocumentDB not-checkable:driver:pymongoAzureBlob not-checkable:schemasAzureCostManagement not-checkable:no-requestAzureLogAnalytics not-checkable:schemasBaserow not-checkable:schemas-BigMailer fail:safe-pointBigQuery not-checkable:driver:google.cloudBingAds not-checkable:database-Bitbucket fail:safe-pointBreezometer not-checkable:no-request-Bugsnag fail:safe-point-CalCom fail:safe-point-CanvasLms fail:safe-pointChatwoot not-checkable:database-CircleCI fail:safe-point-CircleciInsights fail:safe-pointCiscoDuo not-checkable:no-requestClickHouse not-checkable:schemasCloudflareD1 not-checkable:schemasCloudflareR2 not-checkable:schemasCoda fail:safe-point-Codecov fail:safe-point-CoinMarketCap fail:safe-pointConvex not-checkable:schemasCosmosDB not-checkable:schemasCoveralls not-checkable:no-request-Cursor fail:safe-pointCustom not-checkable:schemasDatabricks not-checkable:driver:databricks-Datadog fail:safe-pointDepot not-checkable:no-request-Discourse fail:safe-point-Dixa fail:safe-point-Dockerhub fail:safe-pointDocusign not-checkable:no-requestDynamics365 not-checkable:no-requestDynamoDB not-checkable:schemas-EcbDataPortal fail:safe-pointElasticsearch not-checkable:schemas-Fastly fail:safe-pointFeaturebase not-checkable:databaseFileUpload not-checkable:no-request-FinancialModelling fail:safe-pointFirebase not-checkable:schemas-Flagsmith fail:safe-point-Formbricks fail:safe-pointFramer not-checkable:no-request-G2 fail:safe-point-GainsightPx fail:safe-pointGcpCloudBilling not-checkable:no-requestGhost not-checkable:no-request-Gitguardian fail:safe-pointGithub not-checkable:schemasGoogleAds not-checkable:driver:google.adsGoogleAnalytics not-checkable:database@@ -73,94 +45,51 @@ GrafanaIRM not-checkable:no-requestGroq fail:safe-pointHatchet not-checkable:no-requestHelpScout not-checkable:database-Hitpay fail:safe-point-Honeybadger fail:safe-pointHubspot not-checkable:database-Hyperspell fail:safe-pointInstagram not-checkable:database-Instana fail:safe-pointIntercom not-checkable:databaseKafka not-checkable:driver:confluent_kafkaKandji fail:safe-point-KapaAI fail:safe-pointKernel fail:safe-pointKestra not-checkable:no-request-Klaviyo fail:safe-point-Lacework fail:safe-pointLinear not-checkable:databaseLinkedinAds not-checkable:database-LlamaCloud fail:safe-pointLogicmonitor not-checkable:no-request-Lovable fail:safe-point-Luma fail:safe-pointMSSQL not-checkable:driver:pymssql-MailerSend fail:safe-point-Mem0 fail:safe-point-Mention fail:safe-pointMetaAds not-checkable:database-Meteostat fail:safe-point-MoEngage fail:safe-pointModal not-checkable:driver:grpclibMongoDB not-checkable:driver:pymongo-MonteCarlo fail:safe-pointMotherduck not-checkable:driver:duckdbMyHours fail:safe-pointMySQL not-checkable:driver:psycopgNeo4j not-checkable:schemasNeon not-checkable:driver:psycopg-NewRelic fail:safe-point-Notion fail:safe-point-NpmRegistry fail:safe-pointOpenMeteo not-checkable:no-requestOpenWeather not-checkable:no-requestOptimizely fail:safe-point-Ortto fail:safe-point-PabblySubscriptionsBilling fail:safe-pointPardot not-checkable:databasePinterestAds not-checkable:databasePipedrive not-checkable:database-Pipeliner fail:safe-pointPlanetScaleMySQL not-checkable:driver:psycopgPlanetScalePostgres not-checkable:driver:psycopg-Plunk fail:safe-pointPostgres not-checkable:driver:psycopg-PowerBiAdmin fail:safe-pointProofpointTap not-checkable:no-request-Pylon fail:safe-pointRedditAds not-checkable:databaseRedshift not-checkable:driver:psycopg-Replicate fail:safe-point-RevenueCat fail:safe-point-RunPod fail:safe-pointSFTP not-checkable:transport-SageHR fail:safe-pointSalesforce not-checkable:database-Sevdesk fail:safe-point-Shippo fail:safe-pointSlack not-checkable:schemas-Smaily fail:safe-pointSnapchatAds not-checkable:databaseSnowflake not-checkable:driver:snowflake-Square fail:safe-pointStripe not-checkable:database-SumoLogic fail:safe-pointSupabase not-checkable:driver:psycopg-Swarmia fail:safe-point-Tailscale fail:safe-pointTemporalIO not-checkable:driver:temporalio.client-TerraformCloud fail:safe-pointTikTokAds not-checkable:databaseTrino not-checkable:schemasTwitterAds not-checkable:database-Ubidots fail:safe-point-Unleash fail:safe-pointVitally fail:safe-point-WikipediaPageviews fail:safe-pointWrike not-checkable:no-request-Writesonic fail:safe-pointYouTubeAnalytics not-checkable:database-ZapSign fail:safe-pointZapierSupportedStorage fail:safe-point-Zenduty fail:safe-pointshared-path:rest-client-paginate fail:safe-pointshared-path:rest-framework-wrapped-resource fail:safe-pointdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.pyindex acaae9b1d21d..b93fd76e54c1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resources,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (ClientConfig,EndpointResource,@@ -371,15 +372,17 @@ def zapsign_source(if endpoint == DOCUMENTS_RESOURCE:webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)()+ def dict_rows() -> Iterator[list[dict[str, Any]]]:+ for page in resource:+ yield [row for row in page if isinstance(row, dict)]++ # The guard holds no rows between pages. As a `Resource` it keeps the framework's per-page safe points.+ pages = Resource(dict_rows, name=endpoint, hints=resource._hints)+def items() -> Any:if webhook_enabled:return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)-- def pages() -> Iterator[list[dict[str, Any]]]:- for page in resource:- yield [row for row in page if isinstance(row, dict)]-- return pages()+ return pagespartition_keys = PARTITION_KEYS.get(endpoint)return SourceResponse(diff --git a/products/warehouse_sources/sources/adjust/adjust.py b/products/warehouse_sources/sources/adjust/adjust.pyindex e3e52cac5a2e..041b4c90fa0b 100644--- a/products/warehouse_sources/sources/adjust/adjust.py+++ b/products/warehouse_sources/sources/adjust/adjust.py@@ -424,6 +424,8 @@ def get_rows(params = build_report_params(report, window_start, window_end, tokens, metrics)for rows in _iter_report_pages(session, params, report, logger):window_rows.extend(unpivot_event_rows(rows, config.dimensions, event_names))+ if not window_rows:+ resumable_source_manager.safe_point()if window_rows:window_rows.sort(key=lambda row: str(row.get("day") or ""))yield window_rows@@ -432,6 +434,9 @@ def get_rows(for rows in _iter_report_pages(session, params, report, logger):if rows:yield rows+ # The staged cursor points at this window's start or earlier, so a resume re-reads+ # the window and loses no rows.+ resumable_source_manager.safe_point()# Save AFTER yielding the window so a crash re-yields it rather than skipping it (merge# dedupes on the dimension key).diff --git a/products/warehouse_sources/sources/adyen/adyen.py b/products/warehouse_sources/sources/adyen/adyen.pyindex 5c6a3efe7313..9a83d91c5e03 100644--- a/products/warehouse_sources/sources/adyen/adyen.py+++ b/products/warehouse_sources/sources/adyen/adyen.py@@ -304,15 +304,13 @@ def _iter_cursor_pages(resume: Optional[AdyenResumeConfig],) -> Iterator[list[dict[str, Any]]]:resume_window = _to_datetime(resume.window_start) if resume is not None else None- cursor = resume.cursor if resume is not None else None+ page_cursor = resume.cursor if resume is not None else None+ # Start the windows at the saved one. The default start moves with the clock, so the window+ # grid of this attempt may not line up with the saved window start.+ if resume_window is not None:+ start = resume_windowfor window_start, window_end in iter_windows(start, end):- # Skip windows already walked to completion before the interruption.- if resume_window is not None and window_start < resume_window:- continue- page_cursor = cursor if resume_window is not None and window_start == resume_window else None- resume_window = None-while True:params: dict[str, Any] = {"balancePlatform": balance_platform,@@ -326,19 +324,22 @@ def _iter_cursor_pages(payload = _request(session, _build_url(host, config.path, params), logger).json()items = extract_items(payload, config.data_key)+ page_cursor = next_cursor(payload)+ if page_cursor:+ manager.save_state(+ AdyenResumeConfig(window_start=_format_timestamp(window_start), cursor=page_cursor),+ )+ else:+ manager.save_state(AdyenResumeConfig(window_start=_format_timestamp(window_end)))+if items:yield items+ else:+ manager.safe_point()- page_cursor = next_cursor(payload)if not page_cursor:break- # Checkpoint after yielding, so a crash re-yields the last batch (merge dedupes on- # the primary key) instead of skipping it.- manager.save_state(- AdyenResumeConfig(window_start=_format_timestamp(window_start), cursor=page_cursor),- )-def _iter_offset_pages(session: requests.Session,diff --git a/products/warehouse_sources/sources/adyen/tests/test_adyen.py b/products/warehouse_sources/sources/adyen/tests/test_adyen.pyindex 422f96cdfe56..613e8cc62861 100644--- a/products/warehouse_sources/sources/adyen/tests/test_adyen.py+++ b/products/warehouse_sources/sources/adyen/tests/test_adyen.py@@ -309,7 +309,7 @@ class TestCursorPagination:))- def test_state_is_saved_after_each_yielded_page(self) -> None:+ def test_state_is_staged_for_each_page_and_finished_window(self) -> None:session = _FakeSession([_FakeResponse(json_data={"data": [{"id": "t1"}], "_links": {"next": {"href": "https://x?cursor=c2"}}}),@@ -320,8 +320,10 @@ class TestCursorPagination:self._pages(session, manager)- assert [state.cursor for state in manager.saved_states] == ["c2"]- assert manager.saved_states[0].window_start == "2026-01-01T00:00:00Z"+ assert manager.saved_states == [+ AdyenResumeConfig(window_start="2026-01-01T00:00:00Z", cursor="c2"),+ AdyenResumeConfig(window_start="2026-01-02T00:00:00Z"),+ ]def test_resume_starts_from_the_saved_cursor(self) -> None:session = _FakeSession([_FakeResponse(json_data={"data": [{"id": "t9"}]})])diff --git a/products/warehouse_sources/sources/anthropic/anthropic.py b/products/warehouse_sources/sources/anthropic/anthropic.pyindex 16901cb8d09a..ac31b3a2a0fa 100644--- a/products/warehouse_sources/sources/anthropic/anthropic.py+++ b/products/warehouse_sources/sources/anthropic/anthropic.py@@ -962,7 +962,18 @@ def anthropic_source(# Built for the first day only to carry the resource's column hints; the fan-out below# rebuilds a resource per day.resource = build_analytics_resource(analytics_windows[0])- items = partial(_iter_analytics_windows, build_analytics_resource, analytics_windows, save_analytics_checkpoint)+ # The day loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the+ # framework's per-page safe points, so a run of empty days can hand off and commit its day.+ analytics_days = Resource(+ partial(_iter_analytics_windows, build_analytics_resource, analytics_windows, save_analytics_checkpoint),+ name=endpoint,+ hints=resource._hints,+ )++ def analytics_items() -> Iterable[list[dict[str, Any]]]:+ return analytics_days++ items = analytics_itemselif config.fan_out_over_days:# Claude Code analytics: one windowed request per calendar day, driven entirely by the# paginator (there is no parent API resource to resolve days from).diff --git a/products/warehouse_sources/sources/applovin/applovin.py b/products/warehouse_sources/sources/applovin/applovin.pyindex 48335a44dbbf..4e0a627abe6c 100644--- a/products/warehouse_sources/sources/applovin/applovin.py+++ b/products/warehouse_sources/sources/applovin/applovin.py@@ -234,27 +234,31 @@ def get_rows(while True:rows = _fetch_page(session, config, api_key, start, window_end, offset, logger)+ # A short page means the window is exhausted: AppLovin returns an empty set once+ # `offset` passes the total row count.+ window_done = len(rows) < REPORT_PAGE_SIZE+ if window_done:+ next_start = window_end + timedelta(days=1)+ if next_start <= today:+ resumable_source_manager.save_state(+ AppLovinResumeConfig(next_window_start=next_start.isoformat(), next_offset=0)+ )+ else:+ offset += REPORT_PAGE_SIZE+ resumable_source_manager.save_state(+ AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=offset)+ )+if rows:yield rows+ else:+ resumable_source_manager.safe_point()- # A short page means the window is exhausted: AppLovin returns an empty set once- # `offset` passes the total row count.- if len(rows) < REPORT_PAGE_SIZE:+ if window_done:break- offset += REPORT_PAGE_SIZE- # Saved after yielding so a crash re-yields the in-flight page instead of skipping- # it (the merge dedupes on the dimension primary key).- resumable_source_manager.save_state(- AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=offset)- )-start = window_end + timedelta(days=1)offset = 0- if start <= today:- resumable_source_manager.save_state(- AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=0)- )# The whole window is walked, so drop the checkpoint rather than leaving a later attempt to# resume from the final report window.diff --git a/products/warehouse_sources/sources/applovin/tests/test_applovin.py b/products/warehouse_sources/sources/applovin/tests/test_applovin.pyindex 9ce3683395aa..be39b1092c87 100644--- a/products/warehouse_sources/sources/applovin/tests/test_applovin.py+++ b/products/warehouse_sources/sources/applovin/tests/test_applovin.py@@ -120,7 +120,9 @@ class TestWindowWalking:assert isinstance(result.raised, AppLovinAPIError)assert result.saved_states- assert result.final_state == AppLovinResumeConfig(next_window_start=second_window.isoformat())+ # Both windows were handed on, so the commit points at the window that failed.+ third_window = second_window + timedelta(days=REPORT_WINDOW_DAYS)+ assert result.final_state == AppLovinResumeConfig(next_window_start=third_window.isoformat())class TestPagination:diff --git a/products/warehouse_sources/sources/appsignal/appsignal.py b/products/warehouse_sources/sources/appsignal/appsignal.pyindex be57eae8cdac..eb531e2dc68a 100644--- a/products/warehouse_sources/sources/appsignal/appsignal.py+++ b/products/warehouse_sources/sources/appsignal/appsignal.py@@ -335,6 +335,8 @@ def _get_windowed_rows(windows: deque[tuple[int, int]] = deque([(start, now)])while windows:+ # Every row of the windows before this one is yielded, and the staged start is not past them.+ resumable_source_manager.safe_point()window_since, window_before = windows.popleft()count = _window_count(session, api_token, url, config, window_since, window_before, logger)if count == 0:@@ -823,6 +825,7 @@ def _get_metric_timeseries_rows()if not rows:+ resumable_source_manager.safe_point()continuerows.sort(key=lambda row: _to_epoch(row.get("timestamp")) or 0)@@ -963,6 +966,7 @@ def _get_performance_trace_rows(for window in _iter_windows(start, now, TRACES_WINDOW_SECONDS):traces = _window_traces(session, api_token, app_id, window.since, window.before, logger)if not traces:+ resumable_source_manager.safe_point()continueyield tracesresumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))@@ -1033,6 +1037,8 @@ def _get_trace_span_rows(AppsignalResumeConfig(window_start=_to_epoch(trace_time) or window.since))spans, batch_traces = [], 0+ if not spans:+ resumable_source_manager.safe_point()if spans:yield spans@@ -1042,6 +1048,7 @@ def _get_trace_span_rows()returnresumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))+ resumable_source_manager.safe_point()def _fetch_deploy_stats(@@ -1105,6 +1112,7 @@ def _walk_aggregate_windows(if rows:yield rowsresumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))+ resumable_source_manager.safe_point()def _fetch_tracing_range_or_empty_if_aged_out(@@ -1364,7 +1372,6 @@ def _get_slow_event_action_rows(returnbucket = _to_iso(window.since)- rows: list[dict[str, Any]] = []for event in _fetch_slow_events(session, api_token, app_id, window, logger):digests_fetched += 1@@ -1381,15 +1388,19 @@ def _get_slow_event_action_rows("digest": digest,},)- rows.extend(+ rows = [{**action, "digest": digest, "timestamp": bucket}for action in actionsif isinstance(action, dict) and action.get("namespace") and action.get("action_name")- )+ ]+ # The staged start still points at this bucket, so a resume re-reads it in full.+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()- if rows:- yield rowsresumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))+ resumable_source_manager.safe_point()buckets_done += 1diff --git a/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py b/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.pyindex 69046507261b..073548b2e132 100644--- a/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py+++ b/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py@@ -487,6 +487,7 @@ def get_rows(window_services = list_window_services(session, credentials, window)if not window_services:page_token = None+ resumable_source_manager.safe_point()continuewhile True:@@ -508,6 +509,7 @@ def get_rows(window_start=window.start.isoformat(), next_page_token=page_token, slice_key=slice_key))+ resumable_source_manager.safe_point()if not page_token:breakdiff --git a/products/warehouse_sources/sources/bigmailer/bigmailer.py b/products/warehouse_sources/sources/bigmailer/bigmailer.pyindex 91ff79edf050..0ee00fa75854 100644--- a/products/warehouse_sources/sources/bigmailer/bigmailer.py+++ b/products/warehouse_sources/sources/bigmailer/bigmailer.py@@ -1,5 +1,6 @@import dataclassesfrom collections.abc import Iterator+from functools import partialfrom typing import Any, Optionalfrom requests import Response@@ -228,7 +229,8 @@ def bigmailer_source(return SourceResponse(name=endpoint,- items=lambda: _raise_auth_errors(resource),+ # The wrapper hands each page on unchanged, so as a `Resource` it keeps the framework safe points.+ items=lambda: Resource(partial(_raise_auth_errors, resource), name=endpoint, hints=resource._hints),primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/sources/bitbucket/bitbucket.py b/products/warehouse_sources/sources/bitbucket/bitbucket.pyindex 46e44e817d0f..659acd65a409 100644--- a/products/warehouse_sources/sources/bitbucket/bitbucket.py+++ b/products/warehouse_sources/sources/bitbucket/bitbucket.py@@ -299,7 +299,10 @@ def _map_rows(endpoint: str, items: list[dict[str, Any]], repo: dict[str, Any] |def _iter_repositories(- session: requests.Session, workspace: str, logger: FilteringBoundLogger+ session: requests.Session,+ workspace: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[BitbucketResumeConfig],) -> Iterator[dict[str, Any]]:"""Page through the workspace's repositories, oldest-first (created_on is immutable,so the enumeration order is stable across resume attempts)."""@@ -307,6 +310,8 @@ def _iter_repositories(while True:data = _fetch_page(session, url, logger)yield from data.get("values", [])+ # The repo list holds parents, not rows, so the walk can stop here and restart.+ resumable_source_manager.safe_point()next_url = data.get("next")if not next_url:break@@ -351,6 +356,7 @@ def _get_top_level_rows(# the last page rather than skipping it — merge dedupes on the primary key.if next_url:resumable_source_manager.save_state(BitbucketResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()if not next_url:break@@ -367,7 +373,9 @@ def _get_fan_out_rows(cutoff: datetime | None,cursor_field: str | None,) -> Iterator[list[dict[str, Any]]]:- repos = [repo for repo in _iter_repositories(session, workspace, logger) if repo.get("slug")]+ repos = [+ repo for repo in _iter_repositories(session, workspace, logger, resumable_source_manager) if repo.get("slug")+ ]# Resolve the saved repo bookmark to the slice still to process. If the bookmarked# repo no longer exists, start over from the first repo — merge dedupes re-pulled rows.@@ -405,6 +413,7 @@ def _get_fan_out_rows(resumable_source_manager.save_state(BitbucketResumeConfig(next_url=next_url, repo_slug=repo["slug"]))+ resumable_source_manager.safe_point()if not next_url:break@@ -449,6 +458,7 @@ def _iter_fan_out_parents(repo_slug: str,logger: FilteringBoundLogger,since: Any,+ resumable_source_manager: ResumableSourceManager[BitbucketResumeConfig],) -> Iterator[dict[str, Any]]:"""Walk one repository's parent rows for a two-level fan-out, reusing the parentendpoint's own page size and params so the list stays defined in one place."""@@ -485,6 +495,8 @@ def _iter_fan_out_parents()return+ # The caller has handled every parent yielded so far and holds no rows.+ resumable_source_manager.safe_point()if not next_url:returnurl = next_url@@ -506,7 +518,9 @@ def _get_second_level_fan_out_rows(An incremental sync narrows the parent walk to what changed since the watermark. Withoutthat bound every sync would issue one request per parent that ever existed, which noworkspace's rate budget survives."""- repos = [repo for repo in _iter_repositories(session, workspace, logger) if repo.get("slug")]+ repos = [+ repo for repo in _iter_repositories(session, workspace, logger, resumable_source_manager) if repo.get("slug")+ ]resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneremaining = repos@@ -532,7 +546,9 @@ def _get_second_level_fan_out_rows(bookmark_url = resume_url if index == 0 else Nonetry:- for parent in _iter_fan_out_parents(session, fan_out, workspace, repo["slug"], logger, parent_since):+ for parent in _iter_fan_out_parents(+ session, fan_out, workspace, repo["slug"], logger, parent_since, resumable_source_manager+ ):cursor = parent.get(fan_out.order_field)if bookmark_cursor is not None and _parent_precedes_bookmark(fan_out.order, cursor, bookmark_cursor):continue@@ -569,6 +585,7 @@ def _get_second_level_fan_out_rows(if rows:yield rowsresumable_source_manager.save_state(dataclasses.replace(bookmark, next_url=next_url))+ resumable_source_manager.safe_point()if not next_url:breakdiff --git a/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py b/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.pyindex e0400e3b72bc..11a00758206d 100644--- a/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py+++ b/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py@@ -491,7 +491,7 @@ def test_fan_out_parent_walk_stops_at_the_cap_and_warns():logger = MagicMock()with scripted_network([_page(page)]):- parents = list(_iter_fan_out_parents(requests.Session(), fan_out, "ws", "repo-a", logger, None))+ parents = list(_iter_fan_out_parents(requests.Session(), fan_out, "ws", "repo-a", logger, None, MagicMock()))assert [parent["uuid"] for parent in parents] == ["{p1}", "{p2}"]logger.warning.assert_called_once()diff --git a/products/warehouse_sources/sources/bugsnag/bugsnag.py b/products/warehouse_sources/sources/bugsnag/bugsnag.pyindex 332bbf66e249..039cf3cbbb6b 100644--- a/products/warehouse_sources/sources/bugsnag/bugsnag.py+++ b/products/warehouse_sources/sources/bugsnag/bugsnag.py@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)from products.warehouse_sources.sources.sdk.internals import Batcher@@ -234,7 +235,10 @@ def _iter_all_pages(tolerated_statuses: tuple[int, ...] = (),paginate_by_offset: bool = False,) -> Iterator[dict[str, Any]]:- """Yield every item across all pages of a list endpoint, following its next-page cursor."""+ """Yield every item across all pages of a list endpoint, following its next-page cursor.++ Only parent listings use this, and they run before any row is batched, so each page is a safe point.+ """while True:items, next_url = _fetch_list_page(session,@@ -245,6 +249,7 @@ def _iter_all_pages(paginate_by_offset=paginate_by_offset,)yield from items+ reach_safe_point()if not next_url:returnurl = next_url@@ -500,6 +505,12 @@ def _resolve_parents(return parents+def _safe_point_if_drained(batcher: Batcher, manager: ResumableSourceManager[BugsnagResumeConfig]) -> None:+ """Reach a safe point only when the shared batcher holds no rows the staged cursor skips."""+ if not batcher.should_yield(include_incomplete_chunk=True):+ manager.safe_point()++def _iter_top_level(session: requests.Session,headers: dict[str, str],@@ -538,6 +549,7 @@ def _iter_top_level(yield batcher.get_table()# Save AFTER yielding so a crash resumes at this page rather than losing buffered rows.manager.save_state(BugsnagResumeConfig(next_url=checkpoint_url))+ _safe_point_if_drained(batcher, manager)if not next_url:breakurl = next_url@@ -584,6 +596,7 @@ def _iter_fan_out(if batcher.should_yield():yield batcher.get_table()manager.save_state(BugsnagResumeConfig(next_url=None, parent_id=parent.resume_id))+ _safe_point_if_drained(batcher, manager)continuepage_count = 0@@ -610,6 +623,7 @@ def _iter_fan_out(if batcher.should_yield():yield batcher.get_table()manager.save_state(BugsnagResumeConfig(next_url=checkpoint_url, parent_id=parent.resume_id))+ _safe_point_if_drained(batcher, manager)if not next_url:breakif config.max_pages is not None and page_count >= config.max_pages:diff --git a/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py b/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.pyindex 5c4583d2a14c..0c67767cf126 100644--- a/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py+++ b/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py@@ -36,6 +36,9 @@ class _FakeResumableManager:def save_state(self, data: BugsnagResumeConfig) -> None:self.saved.append(data)+ def safe_point(self) -> None:+ pass+def _make_response(status_code: int, body: Any = None, link: str | None = None) -> requests.Response:response = requests.Response()diff --git a/products/warehouse_sources/sources/cal_com/cal_com.py b/products/warehouse_sources/sources/cal_com/cal_com.pyindex ce4112e75523..14d6093d1bd1 100644--- a/products/warehouse_sources/sources/cal_com/cal_com.py+++ b/products/warehouse_sources/sources/cal_com/cal_com.py@@ -25,10 +25,12 @@ from products.warehouse_sources.sources.sdk import (JSONResponseCursorPaginator,OffsetPaginator,RESTAPIConfig,+ RESTClient,ResumableSourceManager,SinglePagePaginator,SourceResponse,build_dependent_resource,+ create_auth,frozen,make_tracked_session,rest_api_resource,@@ -460,40 +462,32 @@ def _booking_attendees_items(parent_params["limit"] = parent_config.page_sizeparent_client_config = _client_config(region, api_key, parent_config)- parent_client_config["paginator"] = (- CalComBookingsWindowPaginator(now=datetime.now(UTC)) if windowed else _make_paginator(parent_config)+ bookings_client = RESTClient(+ base_url=parent_client_config["base_url"],+ headers=parent_client_config.get("headers"),+ auth=create_auth(parent_client_config.get("auth")),+ paginator=CalComBookingsWindowPaginator(now=datetime.now(UTC)) if windowed else _make_paginator(parent_config),+ capture=parent_client_config.get("capture", True),)- parent_rest_config: RESTAPIConfig = {- "client": parent_client_config,- "resources": [- {- "name": BOOKING_ATTENDEES_PARENT,- "endpoint": {- "path": parent_config.path,- "params": parent_params,- "data_selector": "data",- "data_selector_required": True,- },- }- ],- }-initial_paginator_state: Optional[dict[str, Any]] = Noneif resumable_source_manager.can_resume():initial_paginator_state = _resume_paginator_state(resumable_source_manager.load_state(), windowed)- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # The hook runs when this generator asks for the next page, so after its rows are out.- _save_cursor_state(resumable_source_manager, state)+ # The bookings are paged here rather than through a `Resource`: the next-page state must stay+ # unstaged until every attendee request of the current page is done.+ next_page: dict[str, Optional[dict[str, Any]]] = {"state": None}- bookings = rest_api_resource(- parent_rest_config,- team_id,- job_id,- None,- resume_hook=save_checkpoint,+ def remember_next_page(state: Optional[dict[str, Any]], _has_next_page: bool) -> None:+ next_page["state"] = state++ bookings = bookings_client.paginate(+ path=parent_config.path,+ params=parent_params,+ data_selector="data",+ data_selector_required=True,initial_paginator_state=initial_paginator_state,+ page_state_hook=remember_next_page,)# capture=False for the same reason as `_client_config`: attendee rows are contact details.@@ -529,8 +523,11 @@ def _booking_attendees_items("bookingUpdatedAt": booking.get("updatedAt"),})+ _save_cursor_state(resumable_source_manager, next_page["state"])if rows:yield rows+ else:+ resumable_source_manager.safe_point()def cal_com_source(diff --git a/products/warehouse_sources/sources/canvas_lms/canvas_lms.py b/products/warehouse_sources/sources/canvas_lms/canvas_lms.pyindex cbddb19430a2..a58ab267a79f 100644--- a/products/warehouse_sources/sources/canvas_lms/canvas_lms.py+++ b/products/warehouse_sources/sources/canvas_lms/canvas_lms.py@@ -224,6 +224,12 @@ def validate_credentials(return False, f"Could not connect to Canvas (HTTP {response.status_code})"+def _check_host(domain: str, team_id: int) -> None:+ host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)+ if not host_ok:+ raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)++def _non_fanout_source(config: CanvasEndpointConfig,domain: str,@@ -274,10 +280,10 @@ def _non_fanout_source(def items() -> Iterable[Any]:# Re-check at run time (not just at source-create) in case the domain was edited or now# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.- host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)- if not host_ok:- raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)- yield from resource+ _check_host(domain, team_id)+ # Return the framework resource itself, not a generator around it, so the framework+ # reaches its per-page safe points.+ return resourcereturn _make_source_response(config, items)@@ -340,10 +346,8 @@ def _fanout_source()def items() -> Iterable[Any]:- host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)- if not host_ok:- raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)- yield from dependent_resource+ _check_host(domain, team_id)+ return dependent_resourcereturn _make_source_response(config, items)diff --git a/products/warehouse_sources/sources/circleci/circleci.py b/products/warehouse_sources/sources/circleci/circleci.pyindex 6f220d14e7f8..a2838ada5b76 100644--- a/products/warehouse_sources/sources/circleci/circleci.py+++ b/products/warehouse_sources/sources/circleci/circleci.py@@ -405,14 +405,21 @@ def _jobs_for_workflow(def _workflow_rows(- fetch_page: FetchPageFn, pipelines: list[dict[str, Any]], logger: FilteringBoundLogger+ fetch_page: FetchPageFn,+ pipelines: list[dict[str, Any]],+ logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:for pipeline in pipelines:yield from _workflows_for_pipeline(fetch_page, pipeline["id"], logger)+ safe_point()def _job_rows(- fetch_page: FetchPageFn, pipelines: list[dict[str, Any]], logger: FilteringBoundLogger+ fetch_page: FetchPageFn,+ pipelines: list[dict[str, Any]],+ logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:for pipeline in pipelines:for workflows in _workflows_for_pipeline(fetch_page, pipeline["id"], logger):@@ -431,6 +438,8 @@ def _job_rows(}for job in jobs]+ safe_point()+ safe_point()def _project_rows(@@ -448,7 +457,10 @@ def _project_rows(def _component_version_rows(- fetch_page: FetchPageFn, components: list[dict[str, Any]], logger: FilteringBoundLogger+ fetch_page: FetchPageFn,+ components: list[dict[str, Any]],+ logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:for component in components:component_id = component["id"]@@ -469,6 +481,7 @@ def _component_version_rows(}for version in versions]+ safe_point()def _user_rows(@@ -476,6 +489,7 @@ def _user_rows(pipelines: list[dict[str, Any]],logger: FilteringBoundLogger,seen_user_ids: set[str],+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:# v2 has no "list users in org" endpoint, so the actor ids carried on workflows are# resolved one at a time via GET /user/{id}.@@ -496,6 +510,7 @@ def _user_rows(continueraiseyield [user]+ safe_point()def _iter_v3_pages(@@ -673,6 +688,8 @@ def _get_v3_rows(batches = _v3_run_rows(fetch_page, endpoint, projects, logger)yield from _stage_cursor_before_last_batch(batches, resumable_source_manager, next_cursor)+ # The page's batches are all handed on, so the staged projects cursor covers every row.+ resumable_source_manager.safe_point()def get_rows(@@ -713,25 +730,30 @@ def get_rows(else:pages = _iter_pipeline_pages(fetch_page, org_slug, logger, start_token)+ # A fan-out parent's children are all handed on before this runs, and the staged token still+ # reads the current page again, so it loses no rows.+ safe_point = resumable_source_manager.safe_point+for items, next_token in pages:if endpoint in DIRECT_LIST_ENDPOINTS:if items:yield itemselif endpoint == "workflows":- yield from _workflow_rows(fetch_page, items, logger)+ yield from _workflow_rows(fetch_page, items, logger, safe_point)elif endpoint == "jobs":- yield from _job_rows(fetch_page, items, logger)+ yield from _job_rows(fetch_page, items, logger, safe_point)elif endpoint == "projects":yield from _project_rows(fetch_page, items, seen_project_slugs)elif endpoint == "users":- yield from _user_rows(fetch_page, items, logger, seen_user_ids)+ yield from _user_rows(fetch_page, items, logger, seen_user_ids, safe_point)elif endpoint == "component_versions":- yield from _component_version_rows(fetch_page, items, logger)+ yield from _component_version_rows(fetch_page, items, logger, safe_point)# Save state after the page's rows (and any fan-out children) have been yielded, so a# crash re-yields the in-progress page instead of skipping it.if next_token:resumable_source_manager.save_state(CircleCIResumeConfig(next_page_token=next_token))+ safe_point()def circleci_source(diff --git a/products/warehouse_sources/sources/circleci/tests/test_circleci.py b/products/warehouse_sources/sources/circleci/tests/test_circleci.pyindex bde17d5a27b0..2f405c2c0ef7 100644--- a/products/warehouse_sources/sources/circleci/tests/test_circleci.py+++ b/products/warehouse_sources/sources/circleci/tests/test_circleci.py@@ -145,7 +145,8 @@ class TestPipelinesRows:assert result.rows == [{"id": "p1"}]assert result.params("page-token") == [None, "token-2"]assert result.saved_states == [CircleCIResumeConfig(next_page_token="token-2")]- assert result.committed_states == []+ # The safe point after the page commits the cursor, because its rows are already handed on.+ assert result.committed_states == [CircleCIResumeConfig(next_page_token="token-2")]def test_unknown_endpoint_raises(self):with pytest.raises(ValueError, match="Unknown CircleCI endpoint"):diff --git a/products/warehouse_sources/sources/circleci_insights/circleci_insights.py b/products/warehouse_sources/sources/circleci_insights/circleci_insights.pyindex f3fe196ca584..a370e56cab54 100644--- a/products/warehouse_sources/sources/circleci_insights/circleci_insights.py+++ b/products/warehouse_sources/sources/circleci_insights/circleci_insights.py@@ -290,6 +290,7 @@ def _discover_workflow_names(project_slug: str,all_branches: bool,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> list[str]:"""List the workflow names Insights knows for a project, from the workflow metrics listing.Always uses the widest reporting window so fan-out doesn't miss workflows that only ran@@ -307,6 +308,8 @@ def _discover_workflow_names(name = item.get("name")if name and name not in names:names.append(name)+ # The names are parents, not rows, so the staged cursor still covers every row handed on.+ safe_point()return names@@ -384,6 +387,7 @@ def get_rows(# Mark the slug fully synced so a resume moves straight to the next one.resumable_source_manager.save_state(CircleciInsightsResumeConfig(slug=slug, slug_done=True))+ resumable_source_manager.safe_point()def _project_workflow_metrics_rows(@@ -413,6 +417,7 @@ def _project_workflow_metrics_rows(resumable_source_manager.save_state(CircleciInsightsResumeConfig(slug=project_slug, next_page_token=next_token))+ resumable_source_manager.safe_point()def _project_workflow_fan_out_rows(@@ -428,7 +433,9 @@ def _project_workflow_fan_out_rows(db_incremental_field_last_value: Any,) -> Iterator[list[dict[str, Any]]]:config = CIRCLECI_INSIGHTS_ENDPOINTS[endpoint]- workflow_names = _discover_workflow_names(fetch_page, project_slug, all_branches, logger)+ workflow_names = _discover_workflow_names(+ fetch_page, project_slug, all_branches, logger, resumable_source_manager.safe_point+ )params: dict[str, Any] = _endpoint_params(config, window, all_branches)if all_branches and not config.takes_branch_params:@@ -479,6 +486,7 @@ def _project_workflow_fan_out_rows(slug=project_slug, workflow_name=workflow_name, next_page_token=next_token))+ resumable_source_manager.safe_point()def _workflow_summary_row(data: dict[str, Any], project_slug: str, workflow_name: str) -> dict[str, Any]:@@ -519,7 +527,9 @@ def _project_workflow_object_fan_out_rows("""Fan out over a project's workflows for endpoints that return a single object per workflowinstead of a paginated listing, so resume granularity is one workflow."""config = CIRCLECI_INSIGHTS_ENDPOINTS[endpoint]- workflow_names = _discover_workflow_names(fetch_page, project_slug, all_branches, logger)+ workflow_names = _discover_workflow_names(+ fetch_page, project_slug, all_branches, logger, resumable_source_manager.safe_point+ )params = _endpoint_params(config, window, all_branches)resume_workflow = slug_resume.workflow_name if slug_resume is not None else None@@ -547,6 +557,7 @@ def _project_workflow_object_fan_out_rows(resumable_source_manager.save_state(CircleciInsightsResumeConfig(slug=project_slug, workflow_name=workflow_name))+ resumable_source_manager.safe_point()def _project_flaky_tests_rows(diff --git a/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py b/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.pyindex 80ed7672dd1e..d6ac09e121ad 100644--- a/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py+++ b/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py@@ -207,8 +207,11 @@ class TestWorkflowMetricsRows:CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2"),CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True),]- assert result.committed_states == [CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2")]- assert result.final_state == CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2")+ assert result.committed_states == [+ CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2"),+ CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True),+ ]+ assert result.final_state == CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True)def test_resume_skips_completed_project(self):result = _driver("gh/a/one, gh/a/two").run(diff --git a/products/warehouse_sources/sources/codecov/codecov.py b/products/warehouse_sources/sources/codecov/codecov.pyindex 790855798343..62821d380dd4 100644--- a/products/warehouse_sources/sources/codecov/codecov.py+++ b/products/warehouse_sources/sources/codecov/codecov.py@@ -218,7 +218,11 @@ def _iter_pages(def _iter_repo_names(- session: requests.Session, owner_base_url: str, headers: dict[str, str], logger: FilteringBoundLogger+ session: requests.Session,+ owner_base_url: str,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],) -> Iterator[str]:"""Page through the owner's active repos and yield each name. Inactive repos (neverreceived an upload) have no coverage data, so fanning out over them is wasted API cost."""@@ -226,6 +230,8 @@ def _iter_repo_names(for results, _next_url in _iter_pages(session, url, headers, logger):for item in results:yield item["name"]+ # The repo list holds parents, not rows, so the walk can stop here and restart.+ resumable_source_manager.safe_point()def _get_top_level_rows(@@ -256,6 +262,7 @@ def _get_top_level_rows(# merge dedupes on the primary key.if next_url:resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()def _remaining_repos(@@ -292,7 +299,9 @@ def _get_fan_out_rows() -> Iterator[list[dict[str, Any]]]:"""Fan out one paginated walk per repository, injecting the repo name onto every row (itfeeds each endpoint's composite primary key)."""- repo_names = repositories or list(_iter_repo_names(session, owner_base_url, headers, logger))+ repo_names = repositories or list(+ _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)+ )cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else Noneparams: dict[str, Any] = {**config.extra_params}@@ -327,6 +336,7 @@ def _get_fan_out_rows(# re-yields the last page rather than skipping it.if next_url and not stop_after_this_page:resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url, repo=repo))+ resumable_source_manager.safe_point()if stop_after_this_page:breakexcept requests.HTTPError as exc:@@ -340,6 +350,7 @@ def _get_fan_out_rows(# Advance the bookmark to the next repo so a crash between repos resumes correctly.if index + 1 < len(remaining):resumable_source_manager.save_state(CodecovResumeConfig(next_url=None, repo=remaining[index + 1]))+ resumable_source_manager.safe_point()def _iter_child_rows(@@ -392,7 +403,9 @@ def _iter_parent_fan_out_batches() -> Iterator[tuple[list[dict[str, Any]], CodecovResumeConfig | None]]:"""Yield (rows, state) per child response. `state` is set, with no rows, once every row of aparent page has been yielded: it is the position to resume from after that page."""- repo_names = repositories or list(_iter_repo_names(session, owner_base_url, headers, logger))+ repo_names = repositories or list(+ _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)+ )parent_params: dict[str, Any] = {"page_size": PAGE_SIZE}child_params: dict[str, Any] = {**config.extra_params}diff --git a/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py b/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.pyindex 7149a39ee4b3..06414c052c4a 100644--- a/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py+++ b/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py@@ -241,12 +241,11 @@ def get_batch_rows(batch = coin_ids[index : index + config.batch_size]body = _get(session, config.path, {**params, config.id_param: ",".join(str(coin_id) for coin_id in batch)})rows = _rows_from(config.row_shape, body.get("data"))+ resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_coin_id=batch[-1]))if rows:yield rows-- # Saved after the batch is yielded, so a crash re-yields it rather than skipping it;- # the merge de-duplicates on the endpoint's primary key.- resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_coin_id=batch[-1]))+ else:+ resumable_source_manager.safe_point()def _batch_source(@@ -333,11 +332,11 @@ def get_snapshot_rows({**config.extra_params, "date": day.isoformat(), "start": 1, "limit": config.limit},)rows = [row for row in (body.get("data") or []) if isinstance(row, dict)]+ resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_snapshot_date=day.isoformat()))if rows:yield [{**row, config.date_field: day.isoformat()} for row in rows]-- # Saved after the day is yielded, so a crash re-yields that day rather than skipping it.- resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_snapshot_date=day.isoformat()))+ else:+ resumable_source_manager.safe_point()day += datetime.timedelta(days=1)diff --git a/products/warehouse_sources/sources/cursor/cursor.py b/products/warehouse_sources/sources/cursor/cursor.pyindex 47a5ac203884..7974554e69b0 100644--- a/products/warehouse_sources/sources/cursor/cursor.py+++ b/products/warehouse_sources/sources/cursor/cursor.py@@ -570,16 +570,19 @@ def _paginate_window(# no rows — so termination has to read the page's own size, not the row count.page_items = _page_item_count(config, data, items)has_next = config.paginated and _has_next_page(data, page, page_items, config.page_size)+ window_done = not page_items or not has_next++ if not window_done:+ resumable_source_manager.save_state(CursorResumeConfig(window_start=window_start, page=page + 1))+ elif window_end + 1 <= end_ms:+ resumable_source_manager.save_state(CursorResumeConfig(window_start=window_end + 1, page=1))if rows:yield rows- # Save AFTER yielding so a crash re-yields the last batch rather than skipping it.- if has_next:- resumable_source_manager.save_state(CursorResumeConfig(window_start=window_start, page=page + 1))- elif window_end + 1 <= end_ms:- resumable_source_manager.save_state(CursorResumeConfig(window_start=window_end + 1, page=1))+ else:+ resumable_source_manager.safe_point()- if not page_items or not has_next:+ if window_done:breakpage += 1diff --git a/products/warehouse_sources/sources/cursor/tests/test_cursor.py b/products/warehouse_sources/sources/cursor/tests/test_cursor.pyindex e9a0f93d6bb3..da2339c98303 100644--- a/products/warehouse_sources/sources/cursor/tests/test_cursor.py+++ b/products/warehouse_sources/sources/cursor/tests/test_cursor.py@@ -442,7 +442,7 @@ class TestCursorTransport:assert body["startDate"] == window_startassert body["page"] == 4- def test_windowed_saves_state_after_each_batch(self):+ def test_windowed_stages_state_before_each_batch(self):now_ms = 1_700_000_000_000manager = _manager()pages = [@@ -481,12 +481,8 @@ class TestCursorTransport:first_batch = next(iterator)assert first_batch[0]["id"] # synthetic primary key was added- # The generator is suspended at the yield — nothing saved until the consumer has- # taken the batch and pulls the next one, so a crash mid-batch re-fetches it.- assert manager.save_state.call_count == 0-- next(iterator)- # Resuming past the first yield saved a bookmark at page 2 of the same window.+ # The pipeline commits the staged cursor with the batch, so page 2 is staged before the yield.+ assert manager.save_state.call_count == 1saved = manager.save_state.call_args.args[0]assert saved.page == 2assert saved.window_start is not Nonediff --git a/products/warehouse_sources/sources/datadog/datadog.py b/products/warehouse_sources/sources/datadog/datadog.pyindex ce7dd1371b57..8da915d458ae 100644--- a/products/warehouse_sources/sources/datadog/datadog.py+++ b/products/warehouse_sources/sources/datadog/datadog.py@@ -297,11 +297,15 @@ def _walk(fetch_page: Callable[..., Any],host: str,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],save_state: Callable[[str], None] | None = None,allow_missing: bool = False,max_pages: int | None = None,) -> Iterator[list[dict[str, Any]]]:- """Page through one endpoint URL, yielding a normalized batch per response."""+ """Page through one endpoint URL, yielding a normalized batch per response.++ `safe_point` runs after each page, once the caller has taken every row and the cursor is staged.+ """url = start_urlpages = 0while True:@@ -333,6 +337,7 @@ def _walk(# Save state AFTER yielding the batch — a crash re-yields the last batch (merge dedupes# on primary key) instead of skipping it.save_state(next_url)+ safe_point()url = next_url@@ -341,6 +346,7 @@ def _fan_out_rows(fetch_page: Callable[..., Any],host: str,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:"""Walk a parent endpoint and query the child endpoint once per parent id."""fan_out = config.parent@@ -353,7 +359,7 @@ def _fan_out_rows()parents_seen = 0- for parent_batch in _walk(parent_config, parent_url, fetch_page, host, logger):+ for parent_batch in _walk(parent_config, parent_url, fetch_page, host, logger, safe_point):for parent in parent_batch:if parents_seen >= fan_out.max_parents:# Returning here would write a truncated table that looks like a complete sync.@@ -376,7 +382,7 @@ def _fan_out_rows()# A parent deleted between the list call and its child call answers 404; skip it# rather than failing the whole sync.- for child_batch in _walk(config, child_url, fetch_page, host, logger, allow_missing=True):+ for child_batch in _walk(config, child_url, fetch_page, host, logger, safe_point, allow_missing=True):for row in child_batch:row[fan_out.child_id_field] = parent_idyield child_batch@@ -415,7 +421,7 @@ def get_rows(if config.parent is not None:# A fan-out position is a parent cursor plus a child page, which the single-URL resume# state cannot express, so these endpoints restart from the first parent instead.- yield from _fan_out_rows(config, fetch_page, host, logger)+ yield from _fan_out_rows(config, fetch_page, host, logger, resumable_source_manager.safe_point)returnresume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None@@ -440,6 +446,7 @@ def get_rows(fetch_page,host,logger,+ resumable_source_manager.safe_point,save_state=lambda next_url: resumable_source_manager.save_state(DatadogResumeConfig(next_url=next_url)),max_pages=max_pages,)diff --git a/products/warehouse_sources/sources/discourse/discourse.py b/products/warehouse_sources/sources/discourse/discourse.pyindex ba3cb640604b..9219aadba634 100644--- a/products/warehouse_sources/sources/discourse/discourse.py+++ b/products/warehouse_sources/sources/discourse/discourse.py@@ -1,7 +1,7 @@import re-from collections.abc import Callable, Iterable, Iterator+from collections.abc import Callablefrom datetime import datetime-from typing import Any, Optional+from typing import Any, Optional, castfrom urllib.parse import urlparseimport requests@@ -23,6 +23,7 @@ from products.warehouse_sources.sources.sdk import (IncrementalConfig,OffsetPaginator,PageNumberPaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SinglePagePaginator,@@ -230,20 +231,16 @@ def _flatten_directory_item(item: dict[str, Any]) -> dict[str, Any]:_POST_CONTENT_FIELDS = ("excerpt", "edit_reason")-def _public_activity_rows(batch: list[dict[str, Any]]) -> list[dict[str, Any]]:- """Reduce an activity page to public actions, without their post text.+def _is_public_activity(row: dict[str, Any]) -> bool:+ """`USER_ACTION_PUBLIC_TYPES` explains why the action types are narrowed here instead of in the request."""+ return row.get("action_type") in USER_ACTION_PUBLIC_TYPES- `USER_ACTION_PUBLIC_TYPES` explains why the action types are narrowed here instead of in the- request, and `_POST_CONTENT_FIELDS` why the text is dropped.- """- rows: list[dict[str, Any]] = []- for row in batch:- if row.get("action_type") not in USER_ACTION_PUBLIC_TYPES:- continue- for field_name in _POST_CONTENT_FIELDS:- row.pop(field_name, None)- rows.append(row)- return rows++def _without_post_content(row: dict[str, Any]) -> dict[str, Any]:+ """`_POST_CONTENT_FIELDS` explains why the text is dropped."""+ for field_name in _POST_CONTENT_FIELDS:+ row.pop(field_name, None)+ return rowdef _coerce_incremental_cursor(value: Any) -> Optional[int]:@@ -340,7 +337,7 @@ def discourse_source(returnresumable_source_manager.save_state(DiscourseResumeConfig(page=state.get("page"), before=state.get("before")))- def build_resource() -> Iterable[list[dict[str, Any]]]:+ def build_resource() -> Resource:if endpoint_config.fanout is not None:return _fanout_resource(endpoint_config=endpoint_config,@@ -365,14 +362,14 @@ def discourse_source(initial_paginator_state=_flat_initial_state(endpoint_config, resume_config),)- def items() -> Iterator[list[dict[str, Any]]]:+ def items() -> Resource:# Re-checked at run time (not just at source-create) in case the instance URL was edited# or now resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud._check_host(base_url, team_id)- for batch in build_resource():- rows = _public_activity_rows(batch) if endpoint == "user_actions" else batch- if rows:- yield rows+ resource = build_resource()+ if endpoint == "user_actions":+ resource.add_filter(_is_public_activity).add_map(_without_post_content)+ return resourcereturn SourceResponse(name=endpoint,@@ -407,7 +404,7 @@ def _flat_resource(should_use_incremental_field: bool,resume_hook: Callable[[Optional[dict[str, Any]]], None],initial_paginator_state: Optional[dict[str, Any]],-) -> Iterable[list[dict[str, Any]]]:+) -> Resource:endpoint_dict: Endpoint = {"path": endpoint_config.path,**_RETRY_MALFORMED_BODY,@@ -464,10 +461,10 @@ def _fanout_resource(incremental_field: Optional[str],resume_hook: Callable[[Optional[dict[str, Any]]], None],initial_paginator_state: Optional[dict[str, Any]],-) -> Iterable[list[dict[str, Any]]]:+) -> Resource:parent_config = DISCOURSE_ENDPOINTS[fanout.parent_name]- return build_dependent_resource(+ resource = build_dependent_resource(endpoint_configs=DISCOURSE_ENDPOINTS,child_endpoint=endpoint_config.name,fanout=fanout,@@ -486,6 +483,7 @@ def _fanout_resource(resume_hook=resume_hook,initial_paginator_state=initial_paginator_state,)+ return cast(Resource, resource)def _get_session(api_key: str, api_username: str) -> requests.Session:diff --git a/products/warehouse_sources/sources/dixa/dixa.py b/products/warehouse_sources/sources/dixa/dixa.pyindex 3493639d7c4c..90b267825691 100644--- a/products/warehouse_sources/sources/dixa/dixa.py+++ b/products/warehouse_sources/sources/dixa/dixa.py@@ -288,6 +288,8 @@ def _fan_out_rows(if next_state is not None:resumable_source_manager.save_state(next_state)+ # Every child row of this parent batch is yielded, so the staged parent cursor loses nothing.+ resumable_source_manager.safe_point()def get_rows(@@ -350,11 +352,12 @@ def get_rows(resume_config, logger, endpoint, should_use_incremental_field, db_incremental_field_last_value)for items, window_end in _iter_export_windows(fetch, config, window_start):+ # The whole window arrives in one response, so its end covers every row yielded below.+ resumable_source_manager.save_state(DixaResumeConfig(window_start_ms=window_end))if items:yield items- # Save state AFTER yielding the window so a crash re-yields it- # (merge dedupes on primary key) rather than skipping it.- resumable_source_manager.save_state(DixaResumeConfig(window_start_ms=window_end))+ else:+ resumable_source_manager.safe_point()returnstart_url = _main_start_url(diff --git a/products/warehouse_sources/sources/dockerhub/dockerhub.py b/products/warehouse_sources/sources/dockerhub/dockerhub.pyindex 6b416909f5e5..9fab61cadbfc 100644--- a/products/warehouse_sources/sources/dockerhub/dockerhub.py+++ b/products/warehouse_sources/sources/dockerhub/dockerhub.py@@ -7,7 +7,7 @@ from urllib.parse import quote, urlencode, urlsplitimport orjsonimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.dockerhub.settings import DOCKERHUB_ENDPOINTS, ORG_SCOPED_ENDPOINTSfrom products.warehouse_sources.sources.sdk import (@@ -15,6 +15,7 @@ from products.warehouse_sources.sources.sdk import (SourceResponse,frozen,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -101,10 +102,17 @@ def _tags_url(namespace: str, repository: str) -> str:return f"{DOCKERHUB_BASE_URL}/v2/namespaces/{quote(namespace)}/repositories/{quote(repository)}/tags?{query}"+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows+ # while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_jwt(@@ -165,6 +173,7 @@ def _request_json(retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -189,6 +198,7 @@ def _fetch_page(retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_object(@@ -266,6 +276,8 @@ def _cursor_pages(while url:items, next_url = client.get_page(url)+ if next_url:+ resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url))if items:yield items@@ -274,12 +286,14 @@ def _cursor_pages(breakurl = next_url- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted); merge dedupes the re-pulled page on the primary key.- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()-def _list_repository_names(client: DockerHubClient, namespace: str) -> list[str]:+def _list_repository_names(+ client: DockerHubClient,+ namespace: str,+ resumable_source_manager: ResumableSourceManager[DockerhubResumeConfig],+) -> list[str]:names: list[str] = []url: Optional[str] = _repositories_url(namespace)while url:@@ -287,6 +301,8 @@ def _list_repository_names(client: DockerHubClient, namespace: str) -> list[str]# `name` is the repository primary key and the fan-out key for tags: access it directly so a# malformed row (missing name) fails fast instead of being silently dropped from the sync.names.extend(item["name"] for item in items)+ # The names are not rows, and this attempt has yielded nothing yet.+ resumable_source_manager.safe_point()return names@@ -299,7 +315,7 @@ def _tag_pages(resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None# The repository list is small (names only) and alphabetically ordered, so materializing it keeps# resume deterministic: we slice from the saved repository instead of comparing names.- repositories = _list_repository_names(client, namespace)+ repositories = _list_repository_names(client, namespace, resumable_source_manager)start_index = 0resume_next_url: Optional[str] = None@@ -321,20 +337,19 @@ def _tag_pages(while url:items, next_url = client.get_page(url)+ # On the last page, pin state to the completed repository so a crash before the next+ # repository's first page re-syncs at most this one repository.+ resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url, repository=repository))if items:# Tag rows only carry a numeric repository id; inject the namespace and repository# name so rows are self-describing and the composite primary key is table-unique.yield [{**row, "namespace": namespace, "repository_name": repository} for row in items]+ resumable_source_manager.safe_point()if not next_url:breakurl = next_url- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url, repository=repository))-- # Pin state to the completed repository so a crash before the next repository's first page- # re-syncs at most this one repository.- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=None, repository=repository))def _audit_log_row(row: dict[str, Any]) -> dict[str, Any]:@@ -369,18 +384,19 @@ def _audit_log_pages(f"Docker Hub returned an unexpected audit log payload on page {page}: {type(logs).__name__}")+ # The endpoint returns neither a next link nor a total count, so a short page is the only+ # end-of-collection signal it gives us.+ has_next_page = len(logs) >= PAGE_SIZE+ if has_next_page:+ resumable_source_manager.save_state(DockerhubResumeConfig(page=page + 1))if logs:yield [_audit_log_row(row) for row in logs]- # The endpoint returns neither a next link nor a total count, so a short page is the only- # end-of-collection signal it gives us.- if len(logs) < PAGE_SIZE:+ if not has_next_page:breakpage += 1- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted); merge dedupes any re-pulled page on the primary key.- resumable_source_manager.save_state(DockerhubResumeConfig(page=page))+ resumable_source_manager.safe_point()def _audit_log_action_rows(client: DockerHubClient, namespace: str) -> Iterator[list[dict[str, Any]]]:diff --git a/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py b/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.pyindex d933eecaaec1..fca577b5d270 100644--- a/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py+++ b/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py@@ -152,6 +152,8 @@ def ecb_data_portal_source(resumable_source_manager.save_state(ECBResumeConfig(next_start_period=(window.end + timedelta(days=1)).isoformat()))+ # Rows go out one at a time, so the window is staged only after its last row.+ resumable_source_manager.safe_point()else:resumable_source_manager.clear_state()diff --git a/products/warehouse_sources/sources/fastly/fastly.py b/products/warehouse_sources/sources/fastly/fastly.pyindex b1e75ff245fe..9370e70f88a1 100644--- a/products/warehouse_sources/sources/fastly/fastly.py+++ b/products/warehouse_sources/sources/fastly/fastly.py@@ -376,7 +376,10 @@ def _flatten_origin_inspector(payload: dict[str, Any], service_id: str) -> list[def _iter_services(- session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[FastlyResumeConfig],) -> Iterator[dict[str, Any]]:"""Page through GET /service, yielding each service object. Used to drive fan-out endpoints."""url = _build_url(f"{FASTLY_BASE_URL}/service", {"per_page": SERVICE_PAGE_SIZE})@@ -385,6 +388,8 @@ def _iter_services(items = response.json()if isinstance(items, list):yield from items+ # The service list holds parents, not rows, so the walk can stop here and restart.+ resumable_source_manager.safe_point()next_url = _next_page_url(response)if not next_url:break@@ -465,6 +470,7 @@ def _get_json_api_rows(raise FastlyPaginationError(f"Fastly returned an unchanged next page link for {url}")# Saved AFTER yielding so a crash re-yields the last page rather than skipping it.resumable_source_manager.save_state(FastlyResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()url = next_url@@ -543,6 +549,8 @@ def _get_origin_inspector_rows(breakif next_cursor == cursor:raise FastlyPaginationError(f"Fastly returned an unchanged cursor for {url_base}")+ # The staged bookmark restarts at this service or an earlier one, so no row is lost.+ resumable_source_manager.safe_point()cursor = next_cursor@@ -628,6 +636,8 @@ def _get_platform_metrics_rows(breakif next_cursor == cursor:raise FastlyPaginationError(f"Fastly returned an unchanged cursor for {url_base}")+ # The staged bookmark restarts at this service or an earlier one, so no row is lost.+ resumable_source_manager.safe_point()cursor = next_cursor@@ -655,6 +665,7 @@ def _get_service_list_rows(break# Save AFTER yielding so a crash re-yields the last page rather than skipping it.resumable_source_manager.save_state(FastlyResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()url = next_url@@ -668,7 +679,7 @@ def _iter_services_from_bookmark("""Yield the service ids a fan-out endpoint still has to process, bookmarking each one once thecaller has yielded its rows. If the bookmarked service no longer exists (deleted between runs),start over — merge dedupes the re-pulled rows."""- services = list(_iter_services(session, headers, logger))+ services = list(_iter_services(session, headers, logger, resumable_source_manager))service_ids = [s["id"] for s in services if isinstance(s, dict) and s.get("id")]resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None@@ -682,6 +693,7 @@ def _iter_services_from_bookmark(# Bookmark the service AFTER yielding so a crash resumes here and re-yields. Advanced even for# a versionless (skipped) service so resume doesn't re-evaluate it every time.resumable_source_manager.save_state(FastlyResumeConfig(service_id=service_id))+ resumable_source_manager.safe_point()def _get_version_scoped_rows(@@ -759,6 +771,7 @@ def _get_version_resource_child_rows(]if rows:yield rows+ resumable_source_manager.safe_point()def _iter_billing_pages(@@ -795,6 +808,7 @@ def _iter_billing_pages(cursor = next_cursor# Saved AFTER yielding so a crash re-yields the last page rather than skipping it.resumable_source_manager.save_state(FastlyResumeConfig(cursor=cursor, cursor_request=request))+ resumable_source_manager.safe_point()def _get_billing_list_rows(diff --git a/products/warehouse_sources/sources/financial_modelling/financial_modelling.py b/products/warehouse_sources/sources/financial_modelling/financial_modelling.pyindex 6d0f89ef19b6..900f260c7346 100644--- a/products/warehouse_sources/sources/financial_modelling/financial_modelling.py+++ b/products/warehouse_sources/sources/financial_modelling/financial_modelling.py@@ -233,13 +233,17 @@ def get_rows(batcher.batch(row)if batcher.should_yield():yield batcher.get_table()+ # Flush after each request, before the bookmark advances past this symbol. A symbol can+ # take one request per quarter, and the pipeline acts only on a yield or a safe point.+ if batcher.should_yield(include_incomplete_chunk=True):+ yield batcher.get_table()+ else:+ # The staged index still points at this symbol, so a resume re-fetches it in full.+ resumable_source_manager.safe_point()- # Flush this symbol's rows before advancing the bookmark, so a crash re-fetches only from- # the next symbol (whose rows are not yet persisted) rather than dropping buffered rows.- if batcher.should_yield(include_incomplete_chunk=True):- yield batcher.get_table()if index + 1 < len(symbols):resumable_source_manager.save_state(FinancialModellingResumeConfig(symbol_index=index + 1))+ resumable_source_manager.safe_point()returnparams = {**config.extra_params, **window}diff --git a/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py b/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.pyindex a0621226f553..a0035f47d62b 100644--- a/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py+++ b/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py@@ -141,6 +141,9 @@ class _FakeResumableManager:def save_state(self, data: FinancialModellingResumeConfig) -> None:self.saved.append(data)+ def safe_point(self) -> None:+ pass+def _collect(endpoint: str, symbols: list[str], manager: _FakeResumableManager, by_symbol: dict[str, Any]diff --git a/products/warehouse_sources/sources/flagsmith/flagsmith.py b/products/warehouse_sources/sources/flagsmith/flagsmith.pyindex 13802dc33e49..cf5dd025e936 100644--- a/products/warehouse_sources/sources/flagsmith/flagsmith.py+++ b/products/warehouse_sources/sources/flagsmith/flagsmith.py@@ -19,6 +19,7 @@ from products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -345,7 +346,8 @@ def _iter_pages(budget: _PageBudget,) -> Iterator[list[dict[str, Any]]]:"""Walk a listing (paginated or plain-array) without touching resume state — used to- enumerate fan-out parents, which must be re-listed from scratch on every run."""+ enumerate fan-out parents, which must be re-listed from scratch on every run. Parents are+ enumerated before any row is yielded, so each page is a safe point."""url: str | None = start_urlwhile url:if not budget.take():@@ -355,6 +357,7 @@ def _iter_pages(rows, url = _extract_rows(base, data, url)if rows:yield rows+ reach_safe_point()def _extend_capped(@@ -616,6 +619,7 @@ def _paginate_resource(resumable_source_manager.save_state(FlagsmithResumeConfig(next_url=next_url or "", parent_key=parent.resume_key))+ resumable_source_manager.safe_point()url = next_urldiff --git a/products/warehouse_sources/sources/formbricks/formbricks.py b/products/warehouse_sources/sources/formbricks/formbricks.pyindex 57230a016938..9606d23761ae 100644--- a/products/warehouse_sources/sources/formbricks/formbricks.py+++ b/products/warehouse_sources/sources/formbricks/formbricks.py@@ -9,7 +9,7 @@ from urllib.parse import parse_qsl, urlencode, urlparseimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.formbricks.settings import FORMBRICKS_ENDPOINTS, FormbricksEndpointConfigfrom products.warehouse_sources.sources.sdk import (@@ -17,6 +17,7 @@ from products.warehouse_sources.sources.sdk import (SourceResponse,_is_host_safe,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -224,10 +225,16 @@ def _advance_skip(url: str) -> str:return parsed._replace(query=urlencode(params)).geturl()+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Each page is fetched only after the rows before it are yielded, and nothing is buffered.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((FormbricksRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(MAX_RETRIES),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -317,17 +324,15 @@ def get_rows(while True:items = _fetch_page(session, url, logger)+ # A short (or empty) page means we've reached the end of the collection.+ next_url = _advance_skip(url) if len(items) >= PAGE_SIZE else None+ if next_url is not None:+ resumable_source_manager.save_state(FormbricksResumeConfig(next_url=next_url))if items:yield items- # A short (or empty) page means we've reached the end of the collection.- if len(items) < PAGE_SIZE:+ if next_url is None:break-- next_url = _advance_skip(url)- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted); merge dedupes any re-pulled rows on the primary key.- resumable_source_manager.save_state(FormbricksResumeConfig(next_url=next_url))url = next_urldiff --git a/products/warehouse_sources/sources/g2/g2.py b/products/warehouse_sources/sources/g2/g2.pyindex 732f095f2340..096a6940dd61 100644--- a/products/warehouse_sources/sources/g2/g2.py+++ b/products/warehouse_sources/sources/g2/g2.py@@ -138,6 +138,10 @@ def get_rows(if next_url:resumable_source_manager.save_state(G2ResumeConfig(next_url=next_url))+ # The batcher spans pages, so only a drained batcher leaves no row behind the staged cursor.+ if not batcher.should_yield(include_incomplete_chunk=True):+ resumable_source_manager.safe_point()+if not next_url:breakurl = next_urldiff --git a/products/warehouse_sources/sources/gainsight_px/gainsight_px.py b/products/warehouse_sources/sources/gainsight_px/gainsight_px.pyindex 13ff37742a12..277e904654dd 100644--- a/products/warehouse_sources/sources/gainsight_px/gainsight_px.py+++ b/products/warehouse_sources/sources/gainsight_px/gainsight_px.py@@ -16,6 +16,7 @@ from products.warehouse_sources.sources.gainsight_px.settings import ()from products.warehouse_sources.sources.sdk import (BasePaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SourceResponse,@@ -287,8 +288,13 @@ def gainsight_px_source(}if config.incremental_fields:- return SourceResponse(- items=lambda: _event_pages(+ # The window loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the+ # framework's per-page safe points, so a run of empty windows can hand off.+ event_pages = Resource(+ _event_pages,+ name=endpoint,+ hints={},+ args=(api_key,region,endpoint,@@ -297,8 +303,8 @@ def gainsight_px_source(resumable_source_manager,db_incremental_field_last_value,),- **response_kwargs,)+ return SourceResponse(items=lambda: event_pages, **response_kwargs)paginator: BasePaginatorif config.pagination == "scroll":diff --git a/products/warehouse_sources/sources/gitguardian/gitguardian.py b/products/warehouse_sources/sources/gitguardian/gitguardian.pyindex 59746f7a7280..7cb10a977029 100644--- a/products/warehouse_sources/sources/gitguardian/gitguardian.py+++ b/products/warehouse_sources/sources/gitguardian/gitguardian.py@@ -239,7 +239,9 @@ def _get_fan_out_rows(config: GitGuardianEndpointConfig,parent: GitGuardianEndpointConfig,logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[GitGuardianResumeConfig],) -> Iterator[list[dict[str, Any]]]:+ # A fan-out keeps no resume state, so each safe point lets a hand-off restart from the first parent.parent_url = _build_url(base_url, parent.path, _list_params(parent))for page in _iter_pages(session, parent_url, headers, base_url, parent.name, logger):@@ -257,6 +259,7 @@ def _get_fan_out_rows(yield [{**row, config.parent_id_field: parent_id} for row in page.rows]else:yield page.rows+ resumable_source_manager.safe_point()except requests.HTTPError as e:# The parent can be deleted between listing it and fetching its children.if e.response is not None and e.response.status_code == 404:@@ -265,6 +268,7 @@ def _get_fan_out_rows()continueraise+ resumable_source_manager.safe_point()def get_rows(@@ -286,7 +290,7 @@ def get_rows(if config.parent_endpoint:parent = GITGUARDIAN_ENDPOINTS[config.parent_endpoint]- yield from _get_fan_out_rows(session, headers, base_url, config, parent, logger)+ yield from _get_fan_out_rows(session, headers, base_url, config, parent, logger, resumable_source_manager)returnparams = _list_params(config)@@ -322,6 +326,7 @@ def get_rows(# Checkpoint the CURRENT page's URL after yielding, so a crash re-fetches this page.if config.resumable:resumable_source_manager.save_state(GitGuardianResumeConfig(url=page.url))+ resumable_source_manager.safe_point()# The walk finished cleanly, so drop the checkpoint. Otherwise a retry that re-runs extract# after a completed walk would resume from the final page and skip everything before it.diff --git a/products/warehouse_sources/sources/hitpay/hitpay.py b/products/warehouse_sources/sources/hitpay/hitpay.pyindex 5eeacbd260bb..86ec48ce0b18 100644--- a/products/warehouse_sources/sources/hitpay/hitpay.py+++ b/products/warehouse_sources/sources/hitpay/hitpay.py@@ -123,6 +123,7 @@ def _recurring_billing_source(api_key: str,platform_api_key: Optional[str],environment: Optional[str],+ resumable_source_manager: ResumableSourceManager[HitpayResumeConfig],) -> SourceResponse:config = HITPAY_ENDPOINTS["RecurringBilling"]client = RESTClient(@@ -143,6 +144,9 @@ def _recurring_billing_source():if page:yield page+ # Every row of this status is yielded. The run has no cursor, so a hand-off here+ # restarts from the first status.+ resumable_source_manager.safe_point()return SourceResponse(name="RecurringBilling",@@ -168,7 +172,7 @@ def hitpay_source(db_incremental_field_last_value: Optional[Any],) -> SourceResponse:if endpoint == "RecurringBilling":- return _recurring_billing_source(api_key, platform_api_key, environment)+ return _recurring_billing_source(api_key, platform_api_key, environment, resumable_source_manager)config = HITPAY_ENDPOINTS[endpoint]rest_config: RESTAPIConfig = {diff --git a/products/warehouse_sources/sources/honeybadger/honeybadger.py b/products/warehouse_sources/sources/honeybadger/honeybadger.pyindex b001822999e2..54d9af7b71af 100644--- a/products/warehouse_sources/sources/honeybadger/honeybadger.py+++ b/products/warehouse_sources/sources/honeybadger/honeybadger.py@@ -146,19 +146,33 @@ def _parse_page(data: Any, data_selector: str = "results") -> tuple[list[Any], sreturn data.get(data_selector) or [], (data.get("links") or {}).get("next")-def _collect_items(session: requests.Session, first_url: str, logger: FilteringBoundLogger) -> list[dict]:- """Walk every page of a list endpoint and return the items, following `links.next`."""+def _collect_items(+ session: requests.Session,+ first_url: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[HoneybadgerResumeConfig],+) -> list[dict]:+ """Walk every page of a list endpoint and return the items, following `links.next`.++ It collects fan-out parents, not rows, and the caller holds no rows, so each page is a safe point.+ """items: list[dict] = []url: str | None = first_urlwhile url:results, next_url = _parse_page(_fetch_page(session, url, logger))items.extend(results)+ resumable_source_manager.safe_point()url = next_url if next_url != url else Nonereturn items-def _collect_ids(session: requests.Session, first_url: str, logger: FilteringBoundLogger) -> list[Any]:- return [item["id"] for item in _collect_items(session, first_url, logger)]+def _collect_ids(+ session: requests.Session,+ first_url: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[HoneybadgerResumeConfig],+) -> list[Any]:+ return [item["id"] for item in _collect_items(session, first_url, logger, resumable_source_manager)]def _build_params(@@ -244,13 +258,13 @@ def _get_project_rows(resume.next_url if resume and resume.next_url else _build_url(f"{HONEYBADGER_BASE_URL}/projects", params))for results, next_url in _iter_pages(session, first_url, logger):- if not results:- continue- yield results- # Save AFTER yielding (and only when more pages remain) so a crash re-yields the last- # page rather than skipping it — merge dedupes on the primary key.- if next_url:- resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url))+ if results:+ yield results+ # Save AFTER yielding (and only when more pages remain) so a crash re-yields the last+ # page rather than skipping it — merge dedupes on the primary key.+ if next_url:+ resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()def _get_project_child_rows(@@ -262,7 +276,9 @@ def _get_project_child_rows(params: dict[str, Any],) -> Iterator[list[dict]]:"""Fan out over every project and page through the per-project child endpoint."""- project_ids = _collect_ids(session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger)+ project_ids = _collect_ids(+ session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger, resumable_source_manager+ )# Resolve the saved project bookmark to the projects still to process. If the bookmarked# project no longer exists, start over — merge dedupes re-pulled rows on the primary key.@@ -279,15 +295,17 @@ def _get_project_child_rows(resume_url = None # only the resumed-into project uses the saved URLfor results, next_url in _iter_pages(session, first_url, logger, config.data_selector):- if not results:- continue- if config.time_series:- yield _time_series_rows(project_id, results)- else:- # The traversal determines the parent, so API rows cannot change their project attribution.- yield [{**item, "project_id": project_id} for item in results]- if next_url:- resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url, project_id=project_id))+ if results:+ if config.time_series:+ yield _time_series_rows(project_id, results)+ else:+ # The traversal determines the parent, so API rows cannot change their project attribution.+ yield [{**item, "project_id": project_id} for item in results]+ if next_url:+ resumable_source_manager.save_state(+ HoneybadgerResumeConfig(next_url=next_url, project_id=project_id)+ )+ resumable_source_manager.safe_point()# Advance the bookmark so a crash between projects resumes at the next one.if index + 1 < len(remaining):@@ -311,7 +329,9 @@ def _get_nested_rows(rows, which keeps the per-fault request count away from the 360 req/hour quota."""parent_key = _FAN_OUT_PARENT_KEYS[fan_out_parent]- project_ids = _collect_ids(session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger)+ project_ids = _collect_ids(+ session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger, resumable_source_manager+ )remaining_projects = project_idsresume_parent_id: Any = None@@ -326,6 +346,7 @@ def _get_nested_rows(session,_build_url(f"{HONEYBADGER_BASE_URL}/projects/{project_id}/{fan_out_parent}", parent_params),logger,+ resumable_source_manager,)parent_ids = [parent["id"] for parent in parents]parent_columns = {@@ -350,16 +371,16 @@ def _get_nested_rows(resume_url = Nonefor results, next_url in _iter_pages(session, first_url, logger, config.data_selector):- if not results:- continue- yield [- {**item, "project_id": project_id, parent_key: parent_id, **parent_columns[parent_id]}- for item in results- ]- if next_url:- resumable_source_manager.save_state(- HoneybadgerResumeConfig(next_url=next_url, project_id=project_id, **{parent_key: parent_id})- )+ if results:+ yield [+ {**item, "project_id": project_id, parent_key: parent_id, **parent_columns[parent_id]}+ for item in results+ ]+ if next_url:+ resumable_source_manager.save_state(+ HoneybadgerResumeConfig(next_url=next_url, project_id=project_id, **{parent_key: parent_id})+ )+ resumable_source_manager.safe_point()if parent_index + 1 < len(remaining_parents):resumable_source_manager.save_state(diff --git a/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py b/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.pyindex 311c889f2d18..cdf2cb153591 100644--- a/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py+++ b/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py@@ -327,13 +327,13 @@ class TestHoneybadger:result = self._driver().run("faults", answer)- # The first batch reaches the pipeline before the source saves its next-page state.- # A failed next request must not commit that state and skip the first batch on retry.+ # The first batch is written before the source saves its next-page state, so the safe point+ # after the save commits that state, and a retry starts at page 2 without losing the batch.assert isinstance(result.raised, requests.ConnectionError)assert result.items == [[{"id": 10, "project_id": 1}]]assert result.saved_states == [HoneybadgerResumeConfig(next_url=next_url, project_id=1)]- assert result.committed_states == []- assert result.final_state is None+ assert result.committed_states == [HoneybadgerResumeConfig(next_url=next_url, project_id=1)]+ assert result.final_state == HoneybadgerResumeConfig(next_url=next_url, project_id=1)assert result.params("page")[0:2] == [None, None]@pytest.mark.parametrize(diff --git a/products/warehouse_sources/sources/hyperspell/hyperspell.py b/products/warehouse_sources/sources/hyperspell/hyperspell.pyindex 2431a64adac8..4bfa02d48cd9 100644--- a/products/warehouse_sources/sources/hyperspell/hyperspell.py+++ b/products/warehouse_sources/sources/hyperspell/hyperspell.py@@ -196,6 +196,7 @@ def _get_list_rows(# Save state AFTER yielding so a crash re-yields the last page (deduped on the# primary key) rather than skipping it.resumable_source_manager.save_state(HyperspellResumeConfig(cursor=next_cursor, user_id=principal))+ resumable_source_manager.safe_point()cursor = next_cursor@@ -399,6 +400,7 @@ def get_rows(# Advance the bookmark to the next user so a crash between users resumes correctly.if index + 1 < len(remaining):resumable_source_manager.save_state(HyperspellResumeConfig(cursor=None, user_id=remaining[index + 1]))+ resumable_source_manager.safe_point()def hyperspell_source(diff --git a/products/warehouse_sources/sources/instana/instana.py b/products/warehouse_sources/sources/instana/instana.pyindex 6c3dc3473eac..7ec8fbe10bbf 100644--- a/products/warehouse_sources/sources/instana/instana.py+++ b/products/warehouse_sources/sources/instana/instana.py@@ -294,6 +294,7 @@ def _get_event_rows(# Save AFTER yielding, and only while more windows remain, so a crash re-fetches the last# chunk rather than skipping it — merge dedupes on the primary key.resumable_source_manager.save_state(InstanaResumeConfig(events_window_from=window_to))+ resumable_source_manager.safe_point()window_from = window_to@@ -498,6 +499,7 @@ def _get_metric_rows(window_from = window_toif window_from < end:resumable_source_manager.save_state(InstanaResumeConfig(metrics_window_from=window_from, next_page=1))+ resumable_source_manager.safe_point()def _get_fan_out_rows(diff --git a/products/warehouse_sources/sources/kapa_ai/kapa_ai.py b/products/warehouse_sources/sources/kapa_ai/kapa_ai.pyindex 406669a7990a..65050313f8d2 100644--- a/products/warehouse_sources/sources/kapa_ai/kapa_ai.py+++ b/products/warehouse_sources/sources/kapa_ai/kapa_ai.py@@ -134,11 +134,17 @@ def kapa_ai_source() -> SourceResponse:definition = schema_for_resource(ENDPOINTS, endpoint)- def get_rows() -> Iterator[list[dict[str, Any]]]:+ def get_rows() -> Resource:resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Noneresource = build_resource(api_key, project_id, endpoint, team_id, job_id, resumable_source_manager, resume)- yield from resource- resumable_source_manager.clear_state()++ def walk() -> Iterator[list[dict[str, Any]]]:+ yield from resource+ resumable_source_manager.clear_state()++ # The walk hands each page on unchanged and holds no rows. As a `Resource` it keeps the+ # framework's per-page safe points.+ return Resource(walk, name=endpoint, hints=resource._hints)return SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/sources/lacework/lacework.py b/products/warehouse_sources/sources/lacework/lacework.pyindex 7a0bf032629d..db7349a3cbdc 100644--- a/products/warehouse_sources/sources/lacework/lacework.py+++ b/products/warehouse_sources/sources/lacework/lacework.py@@ -381,12 +381,7 @@ def get_rows(logger.warning("Lacework: stopping pagination, next URL host does not match the configured account")next_page_url = None- if rows:- yield rows-- # Save AFTER yielding so a crash re-yields the current page rather than skipping it.- # When the window is done, checkpoint the NEXT window instead so a resume doesn't- # re-walk this one.+ # When the window is done, stage the NEXT window so a resume doesn't re-walk this one.if next_page_url is not None:resumable_source_manager.save_state(LaceworkResumeConfig(@@ -395,21 +390,26 @@ def get_rows(next_page_url=next_page_url,))- method, url, body = "GET", next_page_url, None+ elif window_end < now:+ resumable_source_manager.save_state(+ LaceworkResumeConfig(+ window_start=_format_datetime(window_end),+ window_end=_format_datetime(min(window_end + timedelta(days=config.window_days), now)),+ next_page_url=None,+ )+ )++ if rows:+ yield rowselse:+ resumable_source_manager.safe_point()++ if next_page_url is None:break+ method, url, body = "GET", next_page_url, Nonewindow_start = window_endwindow_end = None- if window_start < now:- next_window_end = min(window_start + timedelta(days=config.window_days), now)- resumable_source_manager.save_state(- LaceworkResumeConfig(- window_start=_format_datetime(window_start),- window_end=_format_datetime(next_window_end),- next_page_url=None,- )- )def lacework_source(diff --git a/products/warehouse_sources/sources/llama_cloud/llama_cloud.py b/products/warehouse_sources/sources/llama_cloud/llama_cloud.pyindex f3968635c4ac..fa46d53dc9b8 100644--- a/products/warehouse_sources/sources/llama_cloud/llama_cloud.py+++ b/products/warehouse_sources/sources/llama_cloud/llama_cloud.py@@ -292,6 +292,7 @@ def get_rows(# Save AFTER yielding so a crash re-yields the last page rather than skipping it —# merge dedupes the re-yielded rows on the primary key.resumable_source_manager.save_state(LlamaCloudResumeConfig(next_page_token=next_page_token))+ resumable_source_manager.safe_point()params["page_token"] = next_page_tokendiff --git a/products/warehouse_sources/sources/lovable/lovable.py b/products/warehouse_sources/sources/lovable/lovable.pyindex 40969cb01ec4..06f3ee7c3df9 100644--- a/products/warehouse_sources/sources/lovable/lovable.py+++ b/products/warehouse_sources/sources/lovable/lovable.py@@ -1,4 +1,4 @@-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, datetime, time, timedeltafrom typing import Any, Optional@@ -133,19 +133,25 @@ def _time_series_rows(time_series: dict[str, Any]) -> list[dict[str, Any]]:return [rows[timestamp] for timestamp in sorted(rows)]-def _iter_rows(client: RESTClient, path: str, params: dict[str, Any]) -> Iterator[dict[str, Any]]:+def _iter_rows(+ client: RESTClient, path: str, params: dict[str, Any], safe_point: Callable[[], None] | None = None+) -> Iterator[dict[str, Any]]:for page in client.paginate(path=path, params=params, paginator=_paginator(), data_selector="data"):yield from page+ if safe_point is not None:+ safe_point()-def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointConfig) -> Iterator[_ParentRef]:+def _iter_parents(+ client: RESTClient, api_version: str, config: LovableEndpointConfig, safe_point: Callable[[], None]+) -> Iterator[_ParentRef]:if config.scope == "account":yield _ParentRef()returnendpoints = version_config(api_version).endpointsworkspaces = endpoints["Workspaces"]- for workspace in _iter_rows(client, _path(workspaces, _ParentRef()), _params(workspaces, _ParentRef())):+ for workspace in _iter_rows(client, _path(workspaces, _ParentRef()), _params(workspaces, _ParentRef()), safe_point):workspace_id = workspace.get("id")if not workspace_id:continue@@ -155,7 +161,7 @@ def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointCprojects = endpoints["Projects"]workspace_ref = _ParentRef(workspace_id=workspace_id)- for project in _iter_rows(client, _path(projects, workspace_ref), _params(projects, workspace_ref)):+ for project in _iter_rows(client, _path(projects, workspace_ref), _params(projects, workspace_ref), safe_point):project_id = project.get("id")if not project_id or (config.published_projects_only and not project.get("is_published")):continue@@ -165,7 +171,7 @@ def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointCcontinuescans = endpoints["ProjectSecurityScans"]- for scan in _iter_rows(client, _path(scans, project_ref), _params(scans, project_ref)):+ for scan in _iter_rows(client, _path(scans, project_ref), _params(scans, project_ref), safe_point):scan_id = scan.get("id")if scan_id:yield _ParentRef(workspace_id=workspace_id, project_id=project_id, scan_id=scan_id)@@ -210,7 +216,8 @@ def _iter_endpoint(resumable_source_manager: ResumableSourceManager[LovableResumeConfig],extra_params: dict[str, Any],) -> Iterator[list[dict[str, Any]]]:- parents = list(_iter_parents(client, api_version, config))+ # Listing parents hands on no rows, so the staged cursor still covers every row handed on.+ parents = list(_iter_parents(client, api_version, config, resumable_source_manager.safe_point))resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Nonestart_index, cursor = resume_position(parents, resume)@@ -240,6 +247,8 @@ def _iter_endpoint(rows = [row for series in page for row in _time_series_rows(series)] if config.time_series else pageif rows:yield [_with_parent_ids(row, parent) for row in rows]+ # The staged cursor still fetches this page, so resuming from it loses no rows.+ resumable_source_manager.safe_point()cursor = Nonediff --git a/products/warehouse_sources/sources/luma/luma.py b/products/warehouse_sources/sources/luma/luma.pyindex 607513b459a2..ae670de43122 100644--- a/products/warehouse_sources/sources/luma/luma.py+++ b/products/warehouse_sources/sources/luma/luma.py@@ -10,8 +10,10 @@ from products.warehouse_sources.sources.luma.settings import EVENTS_PATH, LUMA_Efrom products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,+ interruptible_wait,make_tracked_session,max_retry_after_seconds,+ reach_safe_point,retry_with_deadline,)@@ -64,12 +66,23 @@ def _retry_wait(retry_state: RetryCallState) -> float:return _EXPONENTIAL_WAIT(retry_state)+def _before_retry_wait(_state: RetryCallState) -> None:+ # Callers send each request with no rows held back, so the run can stop before any retry wait.+ reach_safe_point()+++def _retry_sleep(seconds: float) -> None:+ interruptible_wait(seconds, safe_point=reach_safe_point)++# Luma holds a rate-limited key for one minute, so the waits must add up to more than that.# Eight attempts wait about 1+2+4+8+16+30+30 = 91 seconds in total.@retry_with_deadline(retry=retry_if_exception_type((LumaRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(8),wait=_retry_wait,+ sleep=_retry_sleep,+ before_sleep=_before_retry_wait,reraise=True,)def _fetch_page(@@ -177,6 +190,7 @@ def _get_fan_out_rows()for event_api_id in event_api_ids:yield from _get_child_rows_for_event(session, config, event_api_id, logger)+ resumable_source_manager.safe_point()if not next_events_cursor or not event_entries:break@@ -185,6 +199,7 @@ def _get_fan_out_rows(# Save AFTER yielding every child row of this events page, so a crash re-pulls at most one page# of parents; merge dedupes the re-pulled rows on the primary key.resumable_source_manager.save_state(LumaResumeConfig(pagination_cursor=events_cursor))+ resumable_source_manager.safe_point()def _get_top_level_rows(@@ -215,6 +230,7 @@ def _get_top_level_rows(# Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are# persisted); merge dedupes the re-pulled page on the primary key.resumable_source_manager.save_state(LumaResumeConfig(pagination_cursor=cursor))+ resumable_source_manager.safe_point()def get_rows(diff --git a/products/warehouse_sources/sources/mailersend/mailersend.py b/products/warehouse_sources/sources/mailersend/mailersend.pyindex 9a71b40b1703..1db1943ba1db 100644--- a/products/warehouse_sources/sources/mailersend/mailersend.py+++ b/products/warehouse_sources/sources/mailersend/mailersend.py@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (ClientConfig,EndpointResource,JSONResponsePaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SourceResponse,@@ -345,18 +346,33 @@ def mailersend_source(items: Callable[[], Any]if config.fan_out_over_domains:- items = partial(- _activity_pages,- api_token=api_token,- endpoint=endpoint,- config=config,- team_id=team_id,- job_id=job_id,- resume_hook=save_checkpoint,- initial_state=initial_state,- should_use_incremental_field=should_use_incremental_field,- db_incremental_field_last_value=db_incremental_field_last_value,+ widest_window = _date_window(+ should_use_incremental_field,+ db_incremental_field_last_value,+ (config.window_tiers_days or (MAILERSEND_DEFAULT_ACTIVITY_DAYS,))[0],+ )+ hints_resource = _activity_resource(+ api_token, endpoint, config, team_id, job_id, widest_window, save_checkpoint, initial_state+ )+ # The tier loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the+ # framework's per-page safe points.+ activity = Resource(+ partial(+ _activity_pages,+ api_token=api_token,+ endpoint=endpoint,+ config=config,+ team_id=team_id,+ job_id=job_id,+ resume_hook=save_checkpoint,+ initial_state=initial_state,+ should_use_incremental_field=should_use_incremental_field,+ db_incremental_field_last_value=db_incremental_field_last_value,+ ),+ name=endpoint,+ hints=hints_resource._hints,)+ items = lambda: activityelse:endpoint_resource: EndpointResourceif config.analytics_by_date:diff --git a/products/warehouse_sources/sources/mention/mention.py b/products/warehouse_sources/sources/mention/mention.pyindex e3bc3afc1be2..27fc6ad0762a 100644--- a/products/warehouse_sources/sources/mention/mention.py+++ b/products/warehouse_sources/sources/mention/mention.py@@ -6,7 +6,7 @@ from urllib.parse import urlencode, urljoinimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.mention.settings import MENTION_ENDPOINTSfrom products.warehouse_sources.sources.sdk import (@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (SourceResponse,frozen,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -104,10 +105,17 @@ def _more_url(payload: dict[str, Any]) -> Optional[str]:return resolved+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows+ # while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((MentionRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -195,19 +203,16 @@ def _alert_rows(while url:payload = _fetch_page(session, url, logger)rows = [_unwrap_alert(row) for row in _rows_from_payload(payload, "alerts", url)]- if rows:- yield rows# A missing ``_links.more`` marks the end of the collection. An empty page also terminates# defensively so a lingering cursor can never produce an infinite loop.- next_url = _more_url(payload)- if not next_url or not rows:- break+ next_url = _more_url(payload) if rows else None+ if next_url:+ resumable_source_manager.save_state(MentionResumeConfig(next_url=next_url))+ if rows:+ yield rowsurl = next_url- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted); merge dedupes the re-pulled page on the primary key.- resumable_source_manager.save_state(MentionResumeConfig(next_url=next_url))def _fan_out_alert_ids(@@ -224,40 +229,6 @@ def _fan_out_alert_ids(return _list_all_alert_ids(session, account_id, logger), None-def _mention_rows(- session: requests.Session,- logger: FilteringBoundLogger,- resume: MentionResumeConfig | None,- resumable_source_manager: ResumableSourceManager[MentionResumeConfig],-) -> Iterator[list[dict[str, Any]]]:- account_id = str(_get_account(session, logger)["id"])- alert_ids, next_url = _fan_out_alert_ids(session, account_id, logger, resume)-- while alert_ids:- alert_id = alert_ids[0]- url: Optional[str] = next_url or _list_url(f"/accounts/{account_id}/alerts/{alert_id}/mentions")-- while url:- payload = _fetch_page(session, url, logger)- rows = _rows_from_payload(payload, "mentions", url)- for row in rows:- # Normalized to a string so the merge key type is consistent across alerts.- row["alert_id"] = alert_id- if rows:- yield rows-- more = _more_url(payload)- if not more or not rows:- break-- url = more- resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids, next_url=url))-- alert_ids = alert_ids[1:]- next_url = None- resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids))--def _alert_tag_rows(session: requests.Session,logger: FilteringBoundLogger,@@ -275,11 +246,13 @@ def _alert_tag_rows(rows = _rows_from_payload(payload, "tags", url)for row in rows:row["alert_id"] = alert_id- if rows:- yield rowsalert_ids = alert_ids[1:]resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids))+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()def _paginated_alert_fan_out_rows(@@ -423,7 +396,9 @@ def get_rows(elif endpoint == "alert_tags":yield from _alert_tag_rows(session, logger, resume, resumable_source_manager)elif endpoint == "mentions":- yield from _mention_rows(session, logger, resume, resumable_source_manager)+ yield from _paginated_alert_fan_out_rows(+ session, logger, resume, resumable_source_manager, path="mentions", key="mentions"+ )elif endpoint == "alert_authors":yield from _paginated_alert_fan_out_rows(session, logger, resume, resumable_source_manager, path="authors", key="authors"diff --git a/products/warehouse_sources/sources/meteostat/meteostat.py b/products/warehouse_sources/sources/meteostat/meteostat.pyindex b05e44ab3a98..95c3d376daee 100644--- a/products/warehouse_sources/sources/meteostat/meteostat.py+++ b/products/warehouse_sources/sources/meteostat/meteostat.py@@ -278,6 +278,10 @@ def _get_rows(window_end = min(cursor + timedelta(days=endpoint.window_days - 1), end_boundary)data = _fetch_window(session, headers, endpoint, target, cursor, window_end, units)+ cursor = window_end + timedelta(days=1)+ resumable_source_manager.save_state(+ MeteostatResumeConfig(station_index=index, next_start=cursor.isoformat())+ )if data:rows = []for row in data:@@ -286,13 +290,8 @@ def _get_rows(row[endpoint.date_field] = _parse_timestamp(row.get(endpoint.date_field))rows.append(row)yield rows-- cursor = window_end + timedelta(days=1)- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window- # instead of skipping it.- resumable_source_manager.save_state(- MeteostatResumeConfig(station_index=index, next_start=cursor.isoformat())- )+ else:+ resumable_source_manager.safe_point()def _get_snapshot_rows(diff --git a/products/warehouse_sources/sources/moengage/moengage.py b/products/warehouse_sources/sources/moengage/moengage.pyindex db920c613ee5..f5370f827894 100644--- a/products/warehouse_sources/sources/moengage/moengage.py+++ b/products/warehouse_sources/sources/moengage/moengage.py@@ -325,6 +325,7 @@ def _iter_report_days(days: list[date],save_checkpoint: Callable[[date], None],clear_checkpoint: Callable[[], None],+ safe_point: Callable[[], None],) -> Iterator[list[dict[str, Any]]]:"""Yield every day's pages in ascending day order, checkpointing once a day is fully yielded.@@ -335,6 +336,8 @@ def _iter_report_days(yield from build_resource(day)if index + 1 < len(days):save_checkpoint(days[index + 1])+ # Most days hold no stats, so a backfill can make many requests without a yield.+ safe_point()# The walk finished: leaving the last checkpoint would make a later attempt resume mid-stream.clear_checkpoint()@@ -565,7 +568,12 @@ def moengage_source(resumable_source_manager.save_state(MoEngageResumeConfig(report_day_state={"start": next_day.isoformat()}))items = partial(- _iter_report_days, build_day_resource, days, save_day_checkpoint, resumable_source_manager.clear_state+ _iter_report_days,+ build_day_resource,+ days,+ save_day_checkpoint,+ resumable_source_manager.clear_state,+ resumable_source_manager.safe_point,)return SourceResponse(diff --git a/products/warehouse_sources/sources/monte_carlo/monte_carlo.py b/products/warehouse_sources/sources/monte_carlo/monte_carlo.pyindex 695c81916cd0..50f44fbd703f 100644--- a/products/warehouse_sources/sources/monte_carlo/monte_carlo.py+++ b/products/warehouse_sources/sources/monte_carlo/monte_carlo.py@@ -336,6 +336,7 @@ def _get_alert_rows(# Bookmark the next window so a crash between windows doesn't restart the backfill.if index + 1 < len(windows):resumable_source_manager.save_state(MonteCarloResumeConfig(window_after=time_range["before"]))+ resumable_source_manager.safe_point()def _get_dbt_test_result_rows(diff --git a/products/warehouse_sources/sources/new_relic/new_relic.py b/products/warehouse_sources/sources/new_relic/new_relic.pyindex e0a89e625f7d..f8cadb9d52a1 100644--- a/products/warehouse_sources/sources/new_relic/new_relic.py+++ b/products/warehouse_sources/sources/new_relic/new_relic.py@@ -373,6 +373,7 @@ def _get_event_rows(# rows a re-yield can append.if window_end_ms < until_ms:resumable_source_manager.save_state(NewRelicResumeConfig(window_start_ms=window_end_ms))+ resumable_source_manager.safe_point()window_start_ms = window_end_msdiff --git a/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py b/products/warehouse_sources/sources/new_relic/tests/test_new_relic.pyindex 86b90b6293ca..c9bca333a743 100644--- a/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py+++ b/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py@@ -47,6 +47,9 @@ class FakeResumableSourceManager:self.saved.append(state)self.state = state+ def safe_point(self) -> None:+ pass+def _nrql_data(rows: list[dict[str, Any]]) -> dict[str, Any]:return {"actor": {"account": {"nrql": {"results": rows}}}}diff --git a/products/warehouse_sources/sources/notion/notion.py b/products/warehouse_sources/sources/notion/notion.pyindex 32746dba0305..d6ff38e7b4ad 100644--- a/products/warehouse_sources/sources/notion/notion.py+++ b/products/warehouse_sources/sources/notion/notion.py@@ -365,6 +365,10 @@ def _search_stream(if not has_more or not next_cursor:break+ if not batcher.should_yield(include_incomplete_chunk=True):+ # Nothing is buffered, so every row before `next_cursor` is already yielded.+ resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))+ resumable_source_manager.safe_point()cursor = next_cursorif batcher.should_yield(include_incomplete_chunk=True):@@ -413,6 +417,9 @@ def _cursor_list_stream(if batcher.should_yield():resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))yield batcher.get_table()+ elif not batcher.should_yield(include_incomplete_chunk=True):+ resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))+ resumable_source_manager.safe_point()cursor = next_cursorif batcher.should_yield(include_incomplete_chunk=True):@@ -458,7 +465,10 @@ def _permission_groups_stream(def _iter_page_ids(- session: requests.Session, logger: FilteringBoundLogger, throttle: Optional["_RateLimiter"] = None+ session: requests.Session,+ logger: FilteringBoundLogger,+ throttle: Optional["_RateLimiter"] = None,+ resumable_source_manager: ResumableSourceManager[NotionResumeConfig] | None = None,) -> Iterator[str]:cursor: str | None = Nonewhile True:@@ -482,6 +492,9 @@ def _iter_page_ids(if not data.get("has_more") or not data.get("next_cursor"):breakcursor = data["next_cursor"]+ if resumable_source_manager is not None:+ # The enumeration buffers page ids, not rows, and stages no state, so a resume restarts it.+ resumable_source_manager.safe_point()def _iter_block_children(@@ -564,7 +577,7 @@ def _page_fan_out(else:# Materialize the full set of page IDs up front so the fan-out position can be persisted as a# shrinking queue (a crash during this initial enumeration restarts it, like other sources).- page_ids = list(_iter_page_ids(session, logger, throttle))+ page_ids = list(_iter_page_ids(session, logger, throttle, resumable_source_manager))last_flush_at = time.monotonic()last_empty_stage_at = last_flush_atdiff --git a/products/warehouse_sources/sources/npm_registry/npm_registry.py b/products/warehouse_sources/sources/npm_registry/npm_registry.pyindex d489f7972b0f..a03f1cb197e4 100644--- a/products/warehouse_sources/sources/npm_registry/npm_registry.py+++ b/products/warehouse_sources/sources/npm_registry/npm_registry.py@@ -187,16 +187,23 @@ def _iter_downloads(for entry in document.get("downloads") or []if isinstance(entry, dict) and entry.get("day")]+ last_window = window_end >= today+ if last_window:+ manager.save_state(NpmRegistryResumeConfig(package_index=package_index + 1, window_start=None))+ else:+ next_start = window_end + timedelta(days=1)+ manager.save_state(+ NpmRegistryResumeConfig(package_index=package_index, window_start=next_start.isoformat())+ )+if rows:yield rows+ else:+ manager.safe_point()- if window_end >= today:+ if last_window:break-window_start = window_end + timedelta(days=1)- # Save AFTER yielding the window so a crash re-fetches it rather than skipping it; merge- # dedupes the re-pulled days on the [package, day] primary key.- manager.save_state(NpmRegistryResumeConfig(package_index=package_index, window_start=window_start.isoformat()))def _iter_version_downloads(@@ -303,6 +310,7 @@ def get_rows(# Advance past this package before moving on, so a crash between packages resumes at the# next one rather than re-walking one we already finished.resumable_source_manager.save_state(NpmRegistryResumeConfig(package_index=index + 1, window_start=None))+ resumable_source_manager.safe_point()def validate_packages(packages_raw: str | None) -> tuple[bool, str | None]:diff --git a/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py b/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.pyindex 1ce8781711a6..b4afa7b30784 100644--- a/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py+++ b/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py@@ -96,16 +96,16 @@ class TestGetRowsDownloads:assert len(result.requests) == 1@time_machine.travel("2024-02-15", tick=False)- def test_saves_state_after_each_window_and_advances_package_on_completion(self) -> None:+ def test_stages_state_before_each_window_and_advances_package_on_completion(self) -> None:result = _driver().run("Downloads", always(ScriptedResponse(json={"downloads": [{"day": "2015-01-10", "downloads": 1}]})))- # Final call always advances to the next package index with no in-progress window.+ # The last window stages the next package, and the package loop stages it again.assert result.raised is Noneassert len(result.requests) > 1- assert len(result.saved_states) == len(result.requests)- assert result.saved_states[-1] == NpmRegistryResumeConfig(package_index=1)+ assert len(result.saved_states) == len(result.requests) + 1+ assert result.saved_states[-2:] == [NpmRegistryResumeConfig(package_index=1)] * 2assert result.committed_states == result.saved_states[:-1]diff --git a/products/warehouse_sources/sources/ortto/ortto.py b/products/warehouse_sources/sources/ortto/ortto.pyindex 2cd82f70f268..da73315dbccb 100644--- a/products/warehouse_sources/sources/ortto/ortto.py+++ b/products/warehouse_sources/sources/ortto/ortto.py@@ -140,8 +140,13 @@ def get_rows(if config.pagination == "cursor":if not isinstance(data, dict) or not data.get("has_more"):break- offset = data.get("next_offset", offset + len(items))- cursor_id = data.get("cursor_id") or cursor_id+ next_offset = data.get("next_offset", offset + len(items))+ next_cursor_id = data.get("cursor_id") or cursor_id+ if next_offset == offset and next_cursor_id == cursor_id:+ # The next request would repeat this one and get the same page forever.+ logger.warning(f"Ortto: {endpoint} reported has_more without advancing past offset {offset}")+ break+ offset, cursor_id = next_offset, next_cursor_idelse:if len(items) < config.page_size:break@@ -150,6 +155,7 @@ def get_rows(# Save state AFTER yielding so a crash re-yields the in-flight page# (merge dedupes on primary key) rather than skipping it.resumable_source_manager.save_state(OrttoResumeConfig(next_offset=offset, cursor_id=cursor_id))+ resumable_source_manager.safe_point()def ortto_source(diff --git a/products/warehouse_sources/sources/ortto/tests/test_ortto.py b/products/warehouse_sources/sources/ortto/tests/test_ortto.pyindex c7ec393156e2..7206637392e4 100644--- a/products/warehouse_sources/sources/ortto/tests/test_ortto.py+++ b/products/warehouse_sources/sources/ortto/tests/test_ortto.py@@ -48,6 +48,22 @@ class TestGetRowsCursorPagination:# State saved once, after the first page yielded.assert result.saved_states == [OrttoResumeConfig(next_offset=500, cursor_id="cur-1")]+ @pytest.mark.parametrize(+ "stuck_page",+ [+ {"contacts": [], "has_more": True},+ _contacts_page([], has_more=True, next_offset=0, cursor_id=None),+ ],+ ids=["no_next_offset", "next_offset_unchanged"],+ )+ def test_stops_when_has_more_does_not_advance(self, stuck_page: dict[str, Any]) -> None:+ result = _driver().run(+ "people", [ScriptedResponse(json={"fields": []}), *[ScriptedResponse(json=stuck_page)] * 3]+ )++ assert result.raised is None+ assert len(result.requests) == 2+def test_resumes_from_saved_offset_and_cursor(self) -> None:result = _driver().run("people",diff --git a/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py b/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.pyindex 779e423f3687..36431395deac 100644--- a/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py+++ b/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py@@ -4,7 +4,7 @@ from typing import Any, Optionalimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.pabbly_subscriptions_billing.settings import (PABBLY_ENDPOINTS,@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -59,10 +60,17 @@ def _is_no_data_message(message: str) -> bool:return lowered.startswith("no ") and "found" in lowered+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows+ # while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((PabblyRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -127,17 +135,16 @@ def _iter_top_level(while True:rows = _fetch_page(session, config.path, page, PAGE_SIZE, logger, config.ignore_no_data_errors)+ # A short page means we've reached the end of the list.+ has_next_page = len(rows) >= PAGE_SIZE+ if has_next_page:+ resumable_source_manager.save_state(PabblyResumeConfig(page=page + 1))if rows:yield rows- # A short page means we've reached the end of the list.- if len(rows) < PAGE_SIZE:+ if not has_next_page:break-page += 1- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted); merge dedupes any re-pulled rows on the primary key.- resumable_source_manager.save_state(PabblyResumeConfig(page=page))def _iter_children_of_parent(@@ -181,6 +188,9 @@ def _iter_fan_out(for parent in parents:parent_id = str(parent["id"])yield from _iter_children_of_parent(session, config, parent_id, logger)+ # The staged cursor is this parent page, and every child row read so far is yielded.+ # A parent with no children yields nothing, so the worker needs this point to stop.+ resumable_source_manager.safe_point()if len(parents) < PAGE_SIZE:breakdiff --git a/products/warehouse_sources/sources/pipeliner/pipeliner.py b/products/warehouse_sources/sources/pipeliner/pipeliner.pyindex b23fab863c44..e83a2b851a7f 100644--- a/products/warehouse_sources/sources/pipeliner/pipeliner.py+++ b/products/warehouse_sources/sources/pipeliner/pipeliner.py@@ -1,6 +1,5 @@import reimport dataclasses-from collections.abc import Iteratorfrom datetime import UTC, date, datetimefrom typing import Any, Optional@@ -10,6 +9,7 @@ from requests import Request, Responsefrom products.warehouse_sources.sources.pipeliner.settings import PIPELINER_ENDPOINTSfrom products.warehouse_sources.sources.sdk import (BasePaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SourceResponse,@@ -158,7 +158,7 @@ def pipeliner_source() -> SourceResponse:config = PIPELINER_ENDPOINTS[endpoint]- def items() -> Iterator[list[dict[str, Any]]]:+ def items() -> Resource:# Re-check at run time (not just at source-create) in case the service URL was edited or now# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.host_ok, host_err = _is_host_safe(normalize_service_url(service_url), team_id)@@ -190,8 +190,7 @@ def pipeliner_source(params[f"filter-op[{cursor_field}]"] = "gte"def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist AFTER a page is yielded and only while a next page remains, so a crash- # re-fetches from the next page (already-yielded pages are persisted); merge dedupes.+ # Persist only while a next page remains. The framework stages it with the page it covers.if state and state.get("cursor") is not None:resumable_source_manager.save_state(PipelinerResumeConfig(cursor=state["cursor"], filter_value=filter_value)@@ -224,7 +223,7 @@ def pipeliner_source(],}- resource = rest_api_resource(+ return rest_api_resource(rest_config,team_id,job_id,@@ -232,7 +231,6 @@ def pipeliner_source(resume_hook=save_checkpoint,initial_paginator_state=initial_paginator_state,)- yield from resourcereturn SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py b/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.pyindex ee8915e7329f..2f44c83a5052 100644--- a/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py+++ b/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py@@ -181,40 +181,42 @@ def _activity_event_pages(body = client.get("/activityevents", params=params)rows = body.get("activityEventEntities") or []- if rows:- yield [_normalize_activity_event(row) for row in rows]-continuation_token = body.get("continuationToken")pages += 1# A page can come back empty and still carry a token, so termination is# "token gone", never "no rows".- if not continuation_token:- break-- if continuation_token in seen_tokens:+ if continuation_token and continuation_token in seen_tokens:logger.warning(f"Power BI admin: repeated continuation token for {day.isoformat()}, stopping the day")continuation_token = None- break- seen_tokens.add(continuation_token)-- if pages >= MAX_CONTINUATION_PAGES:- logger.warning(- f"Power BI admin: hit the {MAX_CONTINUATION_PAGES}-page cap for {day.isoformat()}, "- "the rest of the day is skipped"+ elif continuation_token:+ seen_tokens.add(continuation_token)+ if pages >= MAX_CONTINUATION_PAGES:+ logger.warning(+ f"Power BI admin: hit the {MAX_CONTINUATION_PAGES}-page cap for {day.isoformat()}, "+ "the rest of the day is skipped"+ )+ continuation_token = None++ # Stage mid-day so a heartbeat timeout resumes inside the day rather than refetching it.+ if continuation_token:+ resumable_source_manager.save_state(+ PowerBiAdminResumeConfig(next_day=day.isoformat(), continuation_token=continuation_token)+ )+ else:+ resumable_source_manager.save_state(+ PowerBiAdminResumeConfig(next_day=(day + timedelta(days=1)).isoformat()))- continuation_token = None- break- # Save mid-day so a heartbeat timeout resumes inside the day rather than- # refetching it. Saved after yielding, so a crash re-yields the last page and- # merge dedupes on `Id`.- resumable_source_manager.save_state(- PowerBiAdminResumeConfig(next_day=day.isoformat(), continuation_token=continuation_token)- )+ if rows:+ yield [_normalize_activity_event(row) for row in rows]+ else:+ resumable_source_manager.safe_point()++ if not continuation_token:+ breakday += timedelta(days=1)- resumable_source_manager.save_state(PowerBiAdminResumeConfig(next_day=day.isoformat()))def _odata_pages(diff --git a/products/warehouse_sources/sources/pylon/pylon.py b/products/warehouse_sources/sources/pylon/pylon.pyindex a95cae7554b8..761575a9b497 100644--- a/products/warehouse_sources/sources/pylon/pylon.py+++ b/products/warehouse_sources/sources/pylon/pylon.py@@ -188,12 +188,12 @@ def _get_simple_rows(start_cursor = resume.cursor if resume else Nonefor items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):- if items:- yield items- # Save AFTER yielding so a crash re-yields the last page rather than skipping it; merge dedupes- # on the primary key.if next_cursor:manager.save_state(PylonResumeConfig(cursor=next_cursor))+ if items:+ yield items+ else:+ manager.safe_point()def _get_fan_out_rows(@@ -220,19 +220,21 @@ def _get_fan_out_rows(start_cursor = resume_cursor if index == 0 else Noneresume_cursor = None+ next_object_type = remaining[index + 1] if index + 1 < len(remaining) else Nonefor items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):# Stamp the queried object_type so a single table covers every type and the composite# [object_type, id] primary key stays unique even if two types reuse an id.stamped = [{**item, "object_type": item.get("object_type") or object_type} for item in items]- if stamped:- yield stampedif next_cursor:manager.save_state(PylonResumeConfig(cursor=next_cursor, object_type=object_type))-- # Advance the bookmark to the next object_type so a crash between types resumes correctly.- if index + 1 < len(remaining):- manager.save_state(PylonResumeConfig(cursor=None, object_type=remaining[index + 1]))+ elif next_object_type is not None:+ # The last page of this type: advance the bookmark so a crash between types resumes correctly.+ manager.save_state(PylonResumeConfig(cursor=None, object_type=next_object_type))+ if stamped:+ yield stamped+ else:+ manager.safe_point()def _get_windowed_rows(@@ -286,14 +288,17 @@ def _get_windowed_rows(resume_cursor = Nonefor items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):- if items:- yield itemsif next_cursor:manager.save_state(PylonResumeConfig(cursor=next_cursor, window_start=params["start_time"]))+ elif window_end < now:+ # The last page of this window: point the bookmark at the next window.+ manager.save_state(PylonResumeConfig(cursor=None, window_start=_format_rfc3339(window_end)))+ if items:+ yield items+ else:+ manager.safe_point()window_start = window_end- if window_start < now:- manager.save_state(PylonResumeConfig(cursor=None, window_start=_format_rfc3339(window_start)))def _iter_parent_pages(diff --git a/products/warehouse_sources/sources/replicate/replicate.py b/products/warehouse_sources/sources/replicate/replicate.pyindex 01e33bf46a2a..b20be88be2d7 100644--- a/products/warehouse_sources/sources/replicate/replicate.py+++ b/products/warehouse_sources/sources/replicate/replicate.py@@ -391,6 +391,7 @@ def get_rows(# dedupes on the primary key). Tag it with the watermark so a later run with an advanced# watermark rebuilds from the first page instead of trusting this cursor.resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, created_after=current_watermark))+ resumable_source_manager.safe_point()url = next_urldiff --git a/products/warehouse_sources/sources/runpod/runpod.py b/products/warehouse_sources/sources/runpod/runpod.pyindex 89dd5420e337..bad943b91abc 100644--- a/products/warehouse_sources/sources/runpod/runpod.py+++ b/products/warehouse_sources/sources/runpod/runpod.py@@ -212,14 +212,15 @@ def _iter_billing_rows(# Response ordering is undocumented; each window arrives whole, so sort client-side to keep# the ascending guarantee sort_mode="asc" promises the pipeline.rows = sorted((_normalize_billing_record(record) for record in records), key=lambda r: r.get("time") or "")+ if window_end is not None:+ resumable_source_manager.save_state(RunPodResumeConfig(window_start=_format_rfc3339(window_end)))if rows:yield rows+ else:+ resumable_source_manager.safe_point()if window_end is None:break- # Save AFTER yielding so a crash re-yields the last window rather than skipping it — merge- # dedupes the re-pulled rows on the surrogate id.- resumable_source_manager.save_state(RunPodResumeConfig(window_start=_format_rfc3339(window_end)))window_start = window_enddiff --git a/products/warehouse_sources/sources/sage_hr/sage_hr.py b/products/warehouse_sources/sources/sage_hr/sage_hr.pyindex d3719339d3c8..eb9674b90198 100644--- a/products/warehouse_sources/sources/sage_hr/sage_hr.py+++ b/products/warehouse_sources/sources/sage_hr/sage_hr.py@@ -7,7 +7,7 @@ from urllib.parse import urlencodeimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.sage_hr.settings import SAGE_HR_ENDPOINTS, SageHREndpointConfigfrom products.warehouse_sources.sources.sdk import (@@ -15,6 +15,7 @@ from products.warehouse_sources.sources.sdk import (SourceResponse,SyncWindow,make_tracked_session,+ reach_safe_point,retry_with_deadline,)@@ -93,10 +94,16 @@ def _extract_next_page(payload: dict[str, Any]) -> int | None:return next_page if isinstance(next_page, int) else None+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Each page is fetched only after the rows before it are yielded, and nothing is buffered.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((SageHRRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -158,6 +165,7 @@ def _get_windowed_rows(page = 1if resume is not None and resume.next_page is not None and resume.window_from == window_start.isoformat():page = resume.next_page+ next_window_start = window_end + timedelta(days=1)while True:params = {"page": page, "from": window_start.isoformat(), "to": window_end.isoformat()}@@ -172,28 +180,28 @@ def _get_windowed_rows(continueseen_ids.add(row_id)fresh.append(row)- if fresh:- yield fresh# `next_page` is null on the last page; an empty or non-advancing page also terminates# defensively so a bad meta block can never produce an infinite loop.- if not next_page or not items or next_page <= page:- break+ has_next_page = bool(next_page and items and next_page > page)+ if has_next_page:+ resumable_source_manager.save_state(+ SageHRResumeConfig(next_page=next_page, window_from=window_start.isoformat())+ )+ elif next_window_start <= window.end:+ # Window exhausted: a resume can skip straight to the next window.+ resumable_source_manager.save_state(+ SageHRResumeConfig(next_page=1, window_from=next_window_start.isoformat())+ )+ if fresh:+ yield fresh+ else:+ # Most windows are empty, so the walk needs a stop point that does not depend on rows.+ resumable_source_manager.safe_point()+ if not has_next_page or next_page is None:+ breakpage = next_page- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages- # are persisted).- resumable_source_manager.save_state(- SageHRResumeConfig(next_page=page, window_from=window_start.isoformat())- )-- # Window exhausted — everything up to it has been yielded, so a resume can skip straight to- # the next window.- next_window_start = window_end + timedelta(days=1)- if next_window_start <= window.end:- resumable_source_manager.save_state(- SageHRResumeConfig(next_page=1, window_from=next_window_start.isoformat())- )def get_rows(@@ -224,18 +232,17 @@ def get_rows(page = resume.next_page if (resume is not None and resume.next_page is not None) else 1while True:items, next_page = _fetch_page(session, _build_url(base_url, config.path, {"page": page}), logger)+ # `next_page` is null on the last page; an empty or non-advancing page also terminates+ # defensively so a bad meta block can never produce an infinite loop.+ has_next_page = bool(next_page and items and next_page > page)+ if has_next_page:+ resumable_source_manager.save_state(SageHRResumeConfig(next_page=next_page))if items:yield items- # `next_page` is null on the last page; an empty or non-advancing page also terminates- # defensively so a bad meta block can never produce an infinite loop.- if not next_page or not items or next_page <= page:+ if not has_next_page or next_page is None:break-page = next_page- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are- # persisted).- resumable_source_manager.save_state(SageHRResumeConfig(next_page=page))def sage_hr_source(diff --git a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.pyindex 780e06b94c55..17013c74c8b0 100644--- a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py+++ b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py@@ -40,6 +40,9 @@ class _FakeResumableManager:def save_state(self, data: SageHRResumeConfig) -> None:self.saved.append(data)+ def safe_point(self) -> None:+ pass+class TestNormalizeSubdomain:@parameterized.expand(diff --git a/products/warehouse_sources/sources/sevdesk/sevdesk.py b/products/warehouse_sources/sources/sevdesk/sevdesk.pyindex 0bbf542800b1..7d5e897cd94a 100644--- a/products/warehouse_sources/sources/sevdesk/sevdesk.py+++ b/products/warehouse_sources/sources/sevdesk/sevdesk.py@@ -1,10 +1,10 @@-from collections.abc import Iteratorfrom typing import Any, castfrom products.warehouse_sources.sources.sdk import (APIKeyAuth,Endpoint,EndpointResource,+ Resource,RESTAPIConfig,RESTClient,ResumableSourceManager,@@ -84,11 +84,11 @@ def sevdesk_source(SevdeskResumeConfig(offset=state["offset"]) if state is not None else SevdeskResumeConfig(completed=True))- def get_rows() -> Iterator[list[dict[str, Any]]]:+ def get_rows() -> Resource | list[list[dict[str, Any]]]:state = resumable_source_manager.load_state()if state is not None and state.completed:- return- resource = rest_api_resource(+ return []+ return rest_api_resource(config,team_id=team_id,job_id=job_id,@@ -96,7 +96,6 @@ def sevdesk_source(resume_hook=save_page_state,initial_paginator_state={"offset": state.offset} if state is not None else None,)- yield from resourcereturn SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/sources/shippo/shippo.py b/products/warehouse_sources/sources/shippo/shippo.pyindex 3452a36415f9..833612fb7ce9 100644--- a/products/warehouse_sources/sources/shippo/shippo.py+++ b/products/warehouse_sources/sources/shippo/shippo.py@@ -6,12 +6,13 @@ from urllib.parse import parse_qs, urlencode, urlsplitimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)from products.warehouse_sources.sources.shippo.settings import SHIPPO_ENDPOINTS@@ -94,10 +95,17 @@ def _build_url(path: str, params: dict[str, Any]) -> str:return f"{SHIPPO_BASE_URL}{path}/?{urlencode(clean_params)}"+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows+ # while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((ShippoRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(MAX_RETRIES),wait=wait_exponential_jitter(initial=5, max=90),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(session: requests.Session, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:@@ -177,6 +185,7 @@ def _paginate(# Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages# are persisted); merge dedupes the re-pulled page on the primary key.resumable_source_manager.save_state(ShippoResumeConfig(next_url=url, window_start=window_start))+ resumable_source_manager.safe_point()def get_rows(diff --git a/products/warehouse_sources/sources/smaily/smaily.py b/products/warehouse_sources/sources/smaily/smaily.pyindex f39022cb8568..f7cc71c0663c 100644--- a/products/warehouse_sources/sources/smaily/smaily.py+++ b/products/warehouse_sources/sources/smaily/smaily.py@@ -5,12 +5,13 @@ from typing import Any, Optionalimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.sdk import (ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)from products.warehouse_sources.sources.smaily.settings import (@@ -21,9 +22,6 @@ from products.warehouse_sources.sources.smaily.settings import ()REQUEST_TIMEOUT_SECONDS = 60-# Fan-out endpoints issue one request per parent; yield + checkpoint every N parents so a resumed-# job re-fetches at most one chunk.-STATS_CHUNK_SIZE = 100# Cheap endpoint used to confirm credentials are genuine: `list.php` takes no params and every API# user can read it, so one probe validates access to the account.DEFAULT_PROBE_PATH = "list.php"@@ -69,10 +67,17 @@ def _make_session(username: str, password: str) -> requests.Session:return session+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows+ # while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((SmailyRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch(@@ -129,19 +134,18 @@ def _list_rows(resume = manager.load_state() if manager.can_resume() else Nonepage = resume.page if resume else 0+ assert config.page_size is not Nonewhile True:rows = _fetch(session, url, _paged_params(config, page), logger, expect_list=True)+ has_next_page = len(rows) >= config.page_size+ if has_next_page:+ manager.save_state(SmailyResumeConfig(page=page + 1))if rows:yield rows- assert config.page_size is not None- if len(rows) < config.page_size:+ if not has_next_page:break-page += 1- # Save AFTER yielding so a crash re-fetches the last page (already-yielded pages are- # persisted); merge dedupes the re-pulled page on the primary key.- manager.save_state(SmailyResumeConfig(page=page))def _fetch_all_pages(@@ -189,6 +193,7 @@ def _segment_subscriber_rows(pending = sorted(str(segment["id"]) for segment in segments)page = 0+ assert config.page_size is not Nonewhile pending:segment_id = pending[0]@@ -196,19 +201,22 @@ def _segment_subscriber_rows(params = {"list": segment_id, **_paged_params(config, page)}rows = _fetch(session, url, params, logger, expect_list=True)normalized = [_normalize_subscriber(row, segment_id) for row in rows]+ has_next_page = len(rows) >= config.page_size+ if has_next_page:+ manager.save_state(SmailyResumeConfig(page=page + 1, pending_parent_ids=pending))+ else:+ manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending[1:]))if normalized:yield normalized+ else:+ manager.safe_point()- assert config.page_size is not None- if len(rows) < config.page_size:+ if not has_next_page:break-page += 1- manager.save_state(SmailyResumeConfig(page=page, pending_parent_ids=pending))pending = pending[1:]page = 0- manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))def _campaign_statistics_rows(@@ -227,24 +235,23 @@ def _campaign_statistics_rows(campaigns = _fetch_all_pages(session, base_url, SMAILY_ENDPOINTS["campaigns"], logger)pending = [str(campaign["id"]) for campaign in campaigns]- batch: list[dict[str, Any]] = []+ # One row per campaign is yielded at once: a local batch would hold rows across many requests,+ # where the run can neither stop nor commit its cursor.while pending:campaign_id = pending[0]stats = _fetch(session, url, {"id": campaign_id}, logger)if isinstance(stats, dict) and stats:- batch.append(stats)+ row = statselse:# Keep one row per campaign even when the stats payload is unusable (e.g. an empty# list for a never-sent campaign) so the table doesn't silently omit campaigns;# the stats columns stay null. Match the numeric id type of regular stats rows.logger.warning(f"Smaily: campaign {campaign_id} statistics had an unexpected payload shape")- batch.append({"id": int(campaign_id) if campaign_id.isdigit() else campaign_id})+ row = {"id": int(campaign_id) if campaign_id.isdigit() else campaign_id}pending = pending[1:]- if batch and (len(batch) >= STATS_CHUNK_SIZE or not pending):- yield batch- batch = []- manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))+ manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))+ yield [row]def get_rows(diff --git a/products/warehouse_sources/sources/square/square.py b/products/warehouse_sources/sources/square/square.pyindex 24a74d62dfcf..98baf649fdb3 100644--- a/products/warehouse_sources/sources/square/square.py+++ b/products/warehouse_sources/sources/square/square.py@@ -268,10 +268,13 @@ def get_rows(# resume into.if config.paginated and next_cursor:resumable_source_manager.save_state(SquareResumeConfig(cursor=next_cursor))+ elif config.paginated and next_cursor:+ resumable_source_manager.save_state(SquareResumeConfig(cursor=next_cursor))if not config.paginated or not next_cursor:break+ resumable_source_manager.safe_point()cursor = next_cursordiff --git a/products/warehouse_sources/sources/sumo_logic/sumo_logic.py b/products/warehouse_sources/sources/sumo_logic/sumo_logic.pyindex e1f8fd3e7b4b..0e1fadee2cb8 100644--- a/products/warehouse_sources/sources/sumo_logic/sumo_logic.py+++ b/products/warehouse_sources/sources/sumo_logic/sumo_logic.py@@ -212,6 +212,7 @@ def _get_token_paginated_rows(# Save AFTER yielding the batch — a crash re-yields the last page (merge dedupes on the# primary key) instead of skipping it.resumable_source_manager.save_state(SumoLogicResumeConfig(token=next_token))+ resumable_source_manager.safe_point()token = next_tokendiff --git a/products/warehouse_sources/sources/swarmia/swarmia.py b/products/warehouse_sources/sources/swarmia/swarmia.pyindex 785c56f28494..b08c72b27349 100644--- a/products/warehouse_sources/sources/swarmia/swarmia.py+++ b/products/warehouse_sources/sources/swarmia/swarmia.py@@ -28,10 +28,10 @@ class SwarmiaRetryableError(Exception):pass[email protected](frozen=False)[email protected](frozen=True)class SwarmiaResumeConfig:- # ISO date (YYYY-MM-DD) of the start of the next window to fetch. Saved after each window's rows- # are yielded, so a crash resumes at the first window whose rows may not have been committed.+ # ISO date (YYYY-MM-DD) of the start of the next window to fetch. Staged before each window's rows+ # are yielded, so the pipeline commits it together with those rows.next_window_start: str@@ -377,14 +377,14 @@ def get_rows(params = _window_params(config, window_start, window_end)raw_rows = _fetch_csv(session, config.path, params, headers, logger)rows = _rows_from_csv(config, raw_rows)- if rows:- yield rows- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window rather than- # skipping it.if index + 1 < len(windows):resumable_source_manager.save_state(SwarmiaResumeConfig(next_window_start=windows[index + 1][0].isoformat()))+ if rows:+ yield rows+ else:+ resumable_source_manager.safe_point()def swarmia_source(diff --git a/products/warehouse_sources/sources/tailscale/tailscale.py b/products/warehouse_sources/sources/tailscale/tailscale.pyindex 45122e272478..3159f0cebb4d 100644--- a/products/warehouse_sources/sources/tailscale/tailscale.py+++ b/products/warehouse_sources/sources/tailscale/tailscale.py@@ -204,7 +204,8 @@ def _key_rows(response.raise_for_status()listed = _extract_rows(response.json(), config.data_key)- rows: list[dict[str, Any]] = []+ logger.debug(f"Tailscale: listed {len(listed)} rows from {config.name}")+ # Each detail is yielded as it arrives: a local batch would hold rows across one request per key.for key in listed:key_id = key.get("id") if isinstance(key, dict) else Noneif not key_id:@@ -216,11 +217,7 @@ def _key_rows(continuedetail_response.raise_for_status()detail = detail_response.json()- rows.append(detail if isinstance(detail, dict) else key)-- logger.debug(f"Tailscale: fetched {len(rows)} rows from {config.name}")- if rows:- yield rows+ yield [detail if isinstance(detail, dict) else key]def _audit_log_rows(@@ -280,15 +277,15 @@ def _audit_log_rows(# must be yielded in ascending eventTime order.fresh.sort(key=lambda item: item[0] or datetime.min.replace(tzinfo=UTC))+ resumable_source_manager.save_state(TailscaleResumeConfig(window_start=_format_rfc3339(window_end)))if fresh:- yield [row for _, row in fresh]parsed_times = [event_time for event_time, _ in fresh if event_time is not None]if parsed_times:cursor = max(parsed_times)-- # Checkpoint AFTER yielding so a crash re-fetches (and dedupes) the last window- # instead of skipping it.- resumable_source_manager.save_state(TailscaleResumeConfig(window_start=_format_rfc3339(window_end)))+ yield [row for _, row in fresh]+ else:+ # Most windows are empty, so the walk needs a stop point that does not depend on rows.+ resumable_source_manager.safe_point()start = window_enddiff --git a/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py b/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.pyindex 37ec055b9d6e..85f47eae4083 100644--- a/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py+++ b/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py@@ -185,13 +185,18 @@ def _page_predates_cutoff(rows: list[dict[str, Any]], cutoff: datetime | None) -def _iter_workspaces(- session: requests.Session, organization: str, logger: FilteringBoundLogger+ session: requests.Session,+ organization: str,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[TerraformCloudResumeConfig],) -> Iterator[dict[str, Any]]:"""Page through the organization's workspaces, yielding raw JSON:API workspace objects."""url: str | None = _build_url(f"/organizations/{quote(organization, safe='')}/workspaces", {"page[size]": PAGE_SIZE})while url:data = _fetch_json(session, url, logger)yield from _data_items(data)+ # The workspace list holds parents, not rows, so the walk can stop here and restart.+ resumable_source_manager.safe_point()url = _next_url(data)@@ -220,6 +225,7 @@ def _get_top_level_rows(# page rather than skipping it — merge dedupes on the primary key.if next_url:resumable_source_manager.save_state(TerraformCloudResumeConfig(next_url=next_url))+ resumable_source_manager.safe_point()url = next_url@@ -249,7 +255,7 @@ def _get_fan_out_rows(parent workspace without unpacking relationships. Run and state-version ids are globallyunique, so `id` alone stays a valid table-wide primary key."""- workspaces = list(_iter_workspaces(session, organization, logger))+ workspaces = list(_iter_workspaces(session, organization, logger, resumable_source_manager))# Resolve the saved workspace bookmark to the slice still to process. If the bookmarked# workspace no longer exists (deleted between attempts), start over from the first one —@@ -297,6 +303,7 @@ def _get_fan_out_rows(# sync re-walks each workspace's full history.if _page_predates_cutoff(rows, cutoff):break+ resumable_source_manager.safe_point()url = next_urlexcept requests.HTTPError as exc:# A workspace deleted between enumeration and this fetch 404s. Skip it rather than@@ -311,6 +318,7 @@ def _get_fan_out_rows(resumable_source_manager.save_state(TerraformCloudResumeConfig(next_url=None, workspace_id=remaining[index + 1]["id"]))+ resumable_source_manager.safe_point()def get_rows(diff --git a/products/warehouse_sources/sources/ubidots/ubidots.py b/products/warehouse_sources/sources/ubidots/ubidots.pyindex f9ba81a61159..599bbdb9db48 100644--- a/products/warehouse_sources/sources/ubidots/ubidots.py+++ b/products/warehouse_sources/sources/ubidots/ubidots.py@@ -8,13 +8,14 @@ from urllib.parse import urlencode, urlparse, urlunparseimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.sdk import (UNVERSIONED_API_VERSION,ResumableSourceManager,SourceResponse,make_tracked_session,+ reach_safe_point,retry_with_deadline,)from products.warehouse_sources.sources.ubidots.settings import (@@ -147,10 +148,17 @@ def _start_timestamp_ms(value: Any) -> Optional[int]:return None+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # Every request runs between pages or windows: the rows before it are yielded, and no caller+ # holds rows while it fetches.+ reach_safe_point()++@retry_with_deadline(retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_page(@@ -210,12 +218,14 @@ def get_rows(# Save AFTER yielding so a crash re-fetches from the next cursor (already-yielded pages are# persisted); merge dedupes the re-pulled page on the primary key.resumable_source_manager.save_state(UbidotsResumeConfig(next_url=url))+ resumable_source_manager.safe_point()def _iter_variable_ids(session: requests.Session,base_url: str,logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[UbidotsResumeConfig],) -> Iterator[str]:"""List every variable id via the v2.0 API — the parent set the values stream fans out over."""url: Optional[str] = f"{base_url}{VARIABLES_LIST_PATH}?{urlencode({'page_size': PAGE_SIZE})}"@@ -226,6 +236,8 @@ def _iter_variable_ids(# Direct access on purpose: a variable without an id would otherwise silently drop its# whole time series from the sync — better to fail loudly on a malformed page.yield str(item["id"])+ # The caller has finished every variable on this page and holds no rows.+ resumable_source_manager.safe_point()def _initial_values_url(base_url: str, variable_id: str, start: Optional[int]) -> str:@@ -255,7 +267,7 @@ def get_values_rows(start = _start_timestamp_ms(db_incremental_field_last_value) if should_use_incremental_field else None- for variable_id in _iter_variable_ids(session, base_url, logger):+ for variable_id in _iter_variable_ids(session, base_url, logger, resumable_source_manager):if variable_id in completed:continue@@ -293,9 +305,11 @@ def get_values_rows(completed_variable_ids=sorted(completed),))+ resumable_source_manager.safe_point()completed.add(variable_id)resumable_source_manager.save_state(UbidotsResumeConfig(completed_variable_ids=sorted(completed)))+ resumable_source_manager.safe_point()def _read_capped_body(response: requests.Response) -> bytes:@@ -328,6 +342,7 @@ def _read_capped_body(response: requests.Response) -> bytes:retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_data_series(@@ -428,7 +443,7 @@ def get_values_rows_v2(floor = min(watermark, now_ms) if watermark is not None else Noneurl = f"{base_url}{DATA_SERIES_PATH}?{urlencode({'results_format': 'object'})}"- for variable_id in _iter_variable_ids(session, base_url, logger):+ for variable_id in _iter_variable_ids(session, base_url, logger, resumable_source_manager):if variable_id in completed:continue@@ -480,9 +495,11 @@ def get_values_rows_v2(completed_variable_ids=sorted(completed),))+ resumable_source_manager.safe_point()completed.add(variable_id)resumable_source_manager.save_state(UbidotsResumeConfig(completed_variable_ids=sorted(completed)))+ resumable_source_manager.safe_point()def ubidots_source(diff --git a/products/warehouse_sources/sources/unleash/unleash.py b/products/warehouse_sources/sources/unleash/unleash.pyindex dd0d8bef1b8c..0273c6bb8af4 100644--- a/products/warehouse_sources/sources/unleash/unleash.py+++ b/products/warehouse_sources/sources/unleash/unleash.py@@ -1,6 +1,5 @@import reimport dataclasses-from collections.abc import Iteratorfrom typing import Any, Optionalfrom urllib.parse import urlparse@@ -9,6 +8,7 @@ import requestsfrom products.warehouse_sources.sources.sdk import (Endpoint,OffsetPaginator,+ Resource,RESTAPIConfig,ResumableSourceManager,SinglePagePaginator,@@ -170,16 +170,15 @@ def unleash_source(initial_paginator_state = {"offset": resume.offset}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields- # the last page (merge dedupes) rather than skipping it.+ # Persist only when a next page remains. The framework stages it with the page it covers.if state and state.get("offset") is not None:resumable_source_manager.save_state(UnleashResumeConfig(offset=int(state["offset"])))- def items() -> Iterator[list[dict[str, Any]]]:+ def items() -> Resource:# Re-check at run time (not just at source-create) in case the instance URL was edited or# now resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud._check_host(instance_url, team_id)- resource = rest_api_resource(+ return rest_api_resource(rest_config,team_id,job_id,@@ -187,10 +186,6 @@ def unleash_source(resume_hook=save_checkpoint if config.paginated else None,initial_paginator_state=initial_paginator_state,)- for batch in resource:- # An empty collection/page must not push an empty batch into the pipeline.- if batch:- yield batchreturn SourceResponse(name=endpoint,diff --git a/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py b/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.pyindex 15b504f61777..3154c7b6b4a4 100644--- a/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py+++ b/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py@@ -1,6 +1,6 @@import reimport dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, date, datetime, timedeltafrom typing import Any, Optionalfrom urllib.parse import quote@@ -161,6 +161,7 @@ def _article_rows(window_start: date,window_end: date,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> list[dict[str, Any]]:rows: list[dict[str, Any]] = []for article in articles:@@ -169,6 +170,8 @@ def _article_rows(f"/{quote(article, safe='')}/daily/{window_start:%Y%m%d}/{window_end:%Y%m%d}")rows.extend(_with_date(item) for item in _get_items(session, url, logger))+ if not rows:+ safe_point()return rows@@ -210,6 +213,7 @@ def _country_pageview_rows(window_start: date,window_end: date,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> list[dict[str, Any]]:rows: list[dict[str, Any]] = []# One request per month whose first day falls in the window, so consecutive windows never@@ -234,6 +238,8 @@ def _country_pageview_rows("rank": country.get("rank"),})+ if not rows:+ safe_point()month = _next_month(month)return rows@@ -272,6 +278,7 @@ def _unique_device_rows(window_start: date,window_end: date,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> list[dict[str, Any]]:rows: list[dict[str, Any]] = []for access_site in UNIQUE_DEVICES_ACCESS_SITES:@@ -286,6 +293,8 @@ def _unique_device_rows(if item_date is not None:item["date"] = datetime(item_date.year, item_date.month, item_date.day, tzinfo=UTC)rows.append(item)+ if not rows:+ safe_point()return rows@@ -296,6 +305,7 @@ def _media_request_rows(window_start: date,window_end: date,logger: FilteringBoundLogger,+ safe_point: Callable[[], None],) -> list[dict[str, Any]]:media_agent = agent if agent in MEDIA_REQUEST_AGENTS else "all-agents"rows: list[dict[str, Any]] = []@@ -305,6 +315,8 @@ def _media_request_rows(f"/daily/{window_start:%Y%m%d}/{window_end:%Y%m%d}")rows.extend(_with_date(item) for item in _get_items(session, url, logger))+ if not rows:+ safe_point()return rows@@ -381,41 +393,50 @@ def _get_rows(window_days = TOP_WINDOW_DAYS if endpoint in DAILY_REQUEST_ENDPOINTS else WINDOW_DAYS+ # The staged cursor points at the window in progress until the window is yielded. While the+ # window holds no rows, a hand-off therefore loses nothing, so the per-request helpers reach a+ # safe point only before the window has any rows.+ safe_point = resumable_source_manager.safe_point+cursor = startwhile cursor <= end_boundary:window_end = min(cursor + timedelta(days=window_days - 1), end_boundary)+ rows: list[dict[str, Any]] = []if endpoint == TOP_ARTICLES_ENDPOINT:- rows = []day = cursorwhile day <= window_end:rows.extend(_top_rows(session, normalized_project, access, day, logger))+ if not rows:+ safe_point()day += timedelta(days=1)elif endpoint == COUNTRY_TOP_ARTICLES_ENDPOINT:- rows = []day = cursorwhile day <= window_end:for country in country_codes:rows.extend(_country_top_rows(session, country, access, day, logger))+ if not rows:+ safe_point()day += timedelta(days=1)elif endpoint == COUNTRY_PAGEVIEWS_ENDPOINT:- rows = _country_pageview_rows(session, normalized_project, access, cursor, window_end, logger)+ rows = _country_pageview_rows(session, normalized_project, access, cursor, window_end, logger, safe_point)elif endpoint == UNIQUE_DEVICES_ENDPOINT:- rows = _unique_device_rows(session, normalized_project, cursor, window_end, logger)+ rows = _unique_device_rows(session, normalized_project, cursor, window_end, logger, safe_point)elif endpoint == MEDIA_REQUESTS_ENDPOINT:- rows = _media_request_rows(session, normalized_project, agent, cursor, window_end, logger)+ rows = _media_request_rows(session, normalized_project, agent, cursor, window_end, logger, safe_point)elif endpoint == ARTICLE_PAGEVIEWS_ENDPOINT:- rows = _article_rows(session, normalized_project, access, agent, articles, cursor, window_end, logger)+ rows = _article_rows(+ session, normalized_project, access, agent, articles, cursor, window_end, logger, safe_point+ )else:rows = _aggregate_rows(session, normalized_project, access, agent, cursor, window_end, logger)- if rows:- yield rows-cursor = window_end + timedelta(days=1)- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window- # instead of skipping it.resumable_source_manager.save_state(WikipediaPageviewsResumeConfig(next_start=cursor.isoformat()))+ if rows:+ yield rows+ else:+ safe_point()def wikipedia_pageviews_source(diff --git a/products/warehouse_sources/sources/writesonic/writesonic.py b/products/warehouse_sources/sources/writesonic/writesonic.pyindex fe71109dd4e4..6a85537aa28b 100644--- a/products/warehouse_sources/sources/writesonic/writesonic.py+++ b/products/warehouse_sources/sources/writesonic/writesonic.py@@ -245,8 +245,8 @@ def _iter_daily("""Sync a daily export endpoint one UTC day at a time.The required `date` param is both the incremental granularity and the resume granularity.- State is saved *after* each page's rows are yielded, so a crash re-fetches the in-flight- page (merge dedupes on the primary key) rather than skipping it."""+ The next page is staged before each page's rows are yielded, so the pipeline commits it+ together with those rows."""resume = manager.load_state() if manager.can_resume() else Noneresumed_date = _to_date(resume.date) if resume and resume.date else Nonecurrent = resumed_date or start_date@@ -264,11 +264,13 @@ def _iter_daily(# Content export rows don't carry the export date; stamp it in since it's part# of the primary key, the partition key, and the incremental cursor.items = [{**item, "date": day} for item in items]- yield itemsmanager.save_state(WritesonicResumeConfig(date=day, page=page + 1))+ yield itemsnext_day = current + timedelta(days=1)manager.save_state(WritesonicResumeConfig(date=next_day.isoformat(), page=1))+ # Every row of the day is yielded, so the staged day loses nothing.+ manager.safe_point()current = next_daystart_page = 1@@ -292,8 +294,8 @@ def _iter_config(for page, items in _iter_pages(endpoint_config, api_key=api_key, params=params, start_page=start_page, logger=logger):- yield itemsmanager.save_state(WritesonicResumeConfig(page=page + 1))+ yield itemsdef get_rows(diff --git a/products/warehouse_sources/sources/zenduty/zenduty.py b/products/warehouse_sources/sources/zenduty/zenduty.pyindex 0ffeb2e462fc..730bf1b514ed 100644--- a/products/warehouse_sources/sources/zenduty/zenduty.py+++ b/products/warehouse_sources/sources/zenduty/zenduty.py@@ -119,9 +119,16 @@ def _with_team_id(row: dict[str, Any], team_id: str) -> dict[str, Any]:def _list_team_ids(- session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger, page_size: int+ session: requests.Session,+ headers: dict[str, str],+ logger: FilteringBoundLogger,+ page_size: int,+ resumable_source_manager: ResumableSourceManager[ZendutyResumeConfig],) -> list[str]:- """Enumerate every team's `unique_id` — the parents fan-out endpoints iterate over."""+ """Enumerate every team's `unique_id` — the parents fan-out endpoints iterate over.++ No rows are held while it runs, so each page is a safe point.+ """url: str | None = _initial_url(ZENDUTY_ENDPOINTS["teams"].path, page_size)team_ids: list[str] = []while url:@@ -131,6 +138,7 @@ def _list_team_ids(unique_id = row.get("unique_id")if isinstance(unique_id, str) and unique_id:team_ids.append(unique_id)+ resumable_source_manager.safe_point()url = _ensure_zenduty_url(next_url) if next_url else Nonereturn team_ids@@ -156,6 +164,7 @@ def _paginate_top_level(# dedupes on the primary key. Advance the URL before the next fetch to avoid re-looping it.# Validate before saving so an off-origin URL never lands in the resume checkpoint.resumable_source_manager.save_state(ZendutyResumeConfig(next_url=_ensure_zenduty_url(next_url)))+ resumable_source_manager.safe_point()url = next_url@@ -168,7 +177,7 @@ def _paginate_fan_out(resume: ZendutyResumeConfig | None,page_size: int,) -> Iterator[list[dict[str, Any]]]:- team_ids = _list_team_ids(session, headers, logger, page_size)+ team_ids = _list_team_ids(session, headers, logger, page_size, resumable_source_manager)if not team_ids:return@@ -194,12 +203,14 @@ def _paginate_fan_out(resumable_source_manager.save_state(ZendutyResumeConfig(next_url=_ensure_zenduty_url(next_url), team_id=team_id))+ resumable_source_manager.safe_point()url = next_urlcontinue# Team done — checkpoint at the next team's start so a resume skips completed teams# instead of re-walking them.if idx + 1 < len(team_ids):resumable_source_manager.save_state(ZendutyResumeConfig(next_url=None, team_id=team_ids[idx + 1]))+ resumable_source_manager.safe_point()break