Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): give control back during empty pages in 71 sources (#127475)

Tom Owers35af22be7703
diff --git a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
index 2cdec71af8e6..172d5fe6c04b 100644
--- a/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
+++ b/posthog/test/repo_invariants/resume_state_after_yield_baseline.txt
@@ -4,7 +4,6 @@ adjust/adjust.py::get_rows
adobe_analytics/adobe_analytics.py::_metadata_batches
adobe_analytics/adobe_analytics.py::_report_batches
adobe_commerce/adobe_commerce.py::get_rows
-adyen/adyen.py::_iter_cursor_pages
adyen/adyen.py::_iter_fanout_pages
adyen/adyen.py::_iter_offset_pages
adyen/adyen.py::_iter_page_number_pages
@@ -31,7 +30,6 @@ appfollow/appfollow.py::_get_app_fanout
appfollow/appfollow.py::_get_ratings
appfollow/appfollow.py::_get_reviews
apple_search_ads/apple_search_ads.py::_iter_rows
-applovin/applovin.py::get_rows
appsignal/appsignal.py::_get_incident_rows
appsignal/appsignal.py::_get_log_line_rows
appsignal/appsignal.py::_get_metric_timeseries_rows
@@ -92,8 +90,6 @@ codecov/codecov.py::_get_top_level_rows
cody/cody.py::_get_windowed_rows
coin_api/coin_api.py::_get_timeseries_rows
coingecko/coingecko.py::_chart_pages
-coinmarketcap/coinmarketcap.py::get_batch_rows
-coinmarketcap/coinmarketcap.py::get_snapshot_rows
commercetools/commercetools.py::get_rows
common/rest_source/rest_client.py::RESTClient.paginate
common/sql/keyset.py::iter_keyset_pages
@@ -114,7 +110,6 @@ cronitor/cronitor.py::_get_paginated_rows
cronitor/cronitor.py::_get_site_error_rows
crossref/crossref.py::get_rows
cursor/cursor.py::_get_spend_rows
-cursor/cursor.py::_paginate_window
customerly/customerly.py::_get_article_rows
dagster_cloud/dagster_cloud.py::_iter_fanout_rows
dagster_cloud/dagster_cloud.py::_iter_insights_rows
@@ -132,9 +127,6 @@ deepsource/deepsource.py::_root_connection_rows
descope/descope.py::_user_history_rows
dixa/dixa.py::_fan_out_rows
dixa/dixa.py::get_rows
-dockerhub/dockerhub.py::_audit_log_pages
-dockerhub/dockerhub.py::_cursor_pages
-dockerhub/dockerhub.py::_tag_pages
docusign/docusign.py::get_rows
doppler/doppler.py::_get_fan_out_rows
doppler/doppler.py::get_rows
@@ -163,7 +155,6 @@ firebase/firebase.py::iter_realtime_database
fireworks_ai/fireworks_ai.py::_iter_usage_windows
flagsmith/flagsmith.py::_paginate_resource
float_app/float_app.py::_report_window_pages
-formbricks/formbricks.py::get_rows
fred/fred.py::get_rows
freshsales/freshsales.py::_get_paginated_fanout_rows
freshsales/freshsales.py::get_rows
@@ -243,7 +234,6 @@ klaviyo/klaviyo.py::_get_fan_out_rows
klaviyo/klaviyo.py::get_rows
kong_konnect/kong_konnect.py::get_rows
kubecost/kubecost.py::get_rows
-lacework/lacework.py::get_rows
langfuse/langfuse.py::get_rows
langsmith/langsmith.py::_emit_batches
linear/linear.py::_make_paginated_request
@@ -261,13 +251,9 @@ marketo/marketo.py::_asset_offset_rows
marketo/marketo.py::_bulk_rows
marketo/marketo.py::_rest_token_rows
matomo/matomo.py::get_rows
-mention/mention.py::_alert_rows
-mention/mention.py::_alert_tag_rows
-mention/mention.py::_mention_rows
meta_ads/meta_ads.py::_iter_simple_pagination
meta_ads/meta_ads.py::_iter_time_range_pagination
metaplane/metaplane.py::_get_evaluation_rows
-meteostat/meteostat.py::_get_rows
mistral_ai/mistral_ai.py::get_rows
mixpanel/mixpanel.py::_iter_engage
mixpanel/mixpanel.py::_iter_export
@@ -282,7 +268,6 @@ new_relic/new_relic.py::_get_event_rows
new_york_times/new_york_times.py::_get_article_search_rows
notion/notion.py::_page_fan_out
notion/notion.py::_search_stream
-npm_registry/npm_registry.py::_iter_downloads
npm_registry/npm_registry.py::get_rows
nuget/nuget.py::_get_catalog_event_rows
nuget/nuget.py::_get_package_rows
@@ -298,7 +283,6 @@ openrouter/openrouter.py::_get_offset_rows
ortto/ortto.py::get_rows
outbrain/outbrain.py::get_rows
pabbly_subscriptions_billing/pabbly_subscriptions_billing.py::_iter_fan_out
-pabbly_subscriptions_billing/pabbly_subscriptions_billing.py::_iter_top_level
packagist/packagist.py::get_rows
pardot/pardot.py::get_rows
payfit/payfit.py::_get_payslip_rows
@@ -311,7 +295,6 @@ pinterest_ads/pinterest_ads.py::_iter_fanned_out_analytics
platform_sh/platform_sh.py::get_rows
polar/polar.py::get_rows
postgres/postgres.py::postgres_source.get_rows.offset_chunking
-power_bi_admin/power_bi_admin.py::_activity_event_pages
power_bi_admin/power_bi_admin.py::_odata_pages
printify/printify.py::_iter_endpoint_pages
profound/profound.py::_report_pages
@@ -320,9 +303,6 @@ pulumi_cloud/pulumi_cloud.py::_get_deployment_rows
pulumi_cloud/pulumi_cloud.py::_get_resource_rows
pulumi_cloud/pulumi_cloud.py::_get_stack_rows
pulumi_cloud/pulumi_cloud.py::_get_stack_update_rows
-pylon/pylon.py::_get_fan_out_rows
-pylon/pylon.py::_get_simple_rows
-pylon/pylon.py::_get_windowed_rows
qualtrics/qualtrics.py::_iter_collection
qualtrics/qualtrics.py::get_rows
qualys_vmdr/qualys_vmdr.py::get_rows
@@ -333,9 +313,6 @@ render/render.py::_get_top_level_rows
replicate/replicate.py::get_rows
rokt_ads/rokt_ads.py::rokt_ads_source
rollbar/rollbar.py::get_rows
-runpod/runpod.py::_iter_billing_rows
-sage_hr/sage_hr.py::_get_windowed_rows
-sage_hr/sage_hr.py::get_rows
scaleway/scaleway.py::get_rows
segment/segment.py::get_rows
sentry/sentry.py::_iter_issue_tag_values_rows
@@ -346,9 +323,6 @@ signoz/signoz.py::get_rows
similarweb/similarweb.py::_paginated_rows
singular/singular.py::_report_rows
slack/slack.py::_channel_messages_generator
-smaily/smaily.py::_campaign_statistics_rows
-smaily/smaily.py::_list_rows
-smaily/smaily.py::_segment_subscriber_rows
snapchat_ads/snapchat_ads.py::_iter_rows
snowplow/snowplow.py::_get_failed_event_metric_rows
snowplow/snowplow.py::_get_windowed_job_rows
@@ -376,9 +350,7 @@ sumo_logic/sumo_logic.py::_get_collector_fan_out_rows
sumo_logic/sumo_logic.py::_get_log_rows
sumo_logic/sumo_logic.py::_get_offset_paginated_rows
sumo_logic/sumo_logic.py::_get_token_paginated_rows
-swarmia/swarmia.py::get_rows
taboola/taboola.py::get_rows
-tailscale/tailscale.py::_audit_log_rows
tawk_to/tawk_to.py::_paged_property_rows
tawk_to/tawk_to.py::get_rows
teamcity/teamcity.py::_fan_out_rows
@@ -406,9 +378,7 @@ vercel/vercel.py::get_rows
weights_and_biases/weights_and_biases.py::_get_fan_out_rows
weights_and_biases/weights_and_biases.py::_get_project_rows
weights_and_biases/weights_and_biases.py::get_rows
-wikipedia_pageviews/wikipedia_pageviews.py::_get_rows
wordpress/wordpress.py::get_rows
-writesonic/writesonic.py::_iter_config
writesonic/writesonic.py::_iter_daily
youtube_analytics/youtube_analytics.py::get_rows
zendesk_sunshine/zendesk_sunshine.py::_object_records_incremental
diff --git a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
index c832115fe4f9..ee2aeff8e6e2 100644
--- a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
+++ b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
@@ -5,10 +5,8 @@ ashby/ashby.py::_iter_pages
assemblyai/assemblyai.py::assemblyai_source.items
aviationstack/aviationstack.py::aviationstack_source.items
browse_ai/browse_ai.py::browse_ai_source.rows
-cal_com/cal_com.py::_booking_attendees_items
calendly/calendly.py::calendly_source.get_rows
callrail/callrail.py::get_rows
-canvas_lms/canvas_lms.py::_non_fanout_source.items
chargedesk/chargedesk.py::_run_fanout_pass
chargedesk/chargedesk.py::_run_pass
checkly/checkly.py::checkly_source.items
@@ -38,21 +36,16 @@ noaa_cdo/noaa_cdo.py::noaa_cdo_source.items
okendo/okendo.py::okendo_source.items
okta/okta.py::okta_source.items
paymongo/paymongo.py::pull_rows
-pipeliner/pipeliner.py::pipeliner_source.items
-plunk/plunk.py::plunk_source.items
poplar/poplar.py::poplar_source.rows
profound/profound.py::_report_pages
promptwatch/promptwatch.py::promptwatch_source
retell_ai/retell_ai.py::retell_ai_source.items
-revenuecat/revenuecat.py::revenuecat_api_source.items
sentinelone/sentinelone.py::sentinelone_source._items
sentry/sentry.py::_skip_endpoint_on_forbidden
-sevdesk/sevdesk.py::sevdesk_source.get_rows
shutterstock/shutterstock.py::get_rows
telli/telli.py::telli_source.get_rows
twenty/twenty.py::twenty_source.items
twitter/twitter.py::_iter_pages
-unleash/unleash.py::unleash_source.items
vitally/vitally.py::vitally_source
who_gho/who_gho.py::_catalog_pages
who_gho/who_gho.py::_dimension_values_pages
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py
index 7662122b4418..369e5949c360 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/klaviyo/klaviyo.py
@@ -1,5 +1,5 @@
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, timedelta, tzinfo
from email.utils import parsedate_to_datetime
from typing import Any, Optional
@@ -382,8 +382,12 @@ def _iter_resource_ids(
logger: FilteringBoundLogger,
path: str,
page_size: int,
+ safe_point: Callable[[], None] | None = None,
) -> Iterator[str]:
- """Page through a Klaviyo collection and yield each row's id, following the cursor links."""
+ """Page through a Klaviyo collection and yield each row's id, following the cursor links.
+
+ `safe_point` runs after each page. Pass it only when the caller buffers no rows.
+ """
# Some collections (e.g. /object-types) reject page[size]; page_size <= 0 means "omit it and
# page by cursor links alone", mirroring _build_initial_params for top-level endpoints.
params = {"page[size]": page_size} if page_size > 0 else {}
@@ -392,6 +396,8 @@ def _iter_resource_ids(
data = _fetch_page(session, url, headers, logger)
for item in data.get("data", []):
yield item["id"]
+ if safe_point is not None:
+ safe_point()
next_url = data.get("links", {}).get("next")
if not next_url:
@@ -404,6 +410,7 @@ def _iter_fan_out_parents(
headers: dict[str, str],
logger: FilteringBoundLogger,
fan_out: KlaviyoFanOutConfig,
+ safe_point: Callable[[], None],
) -> Iterator[tuple[dict[str, str], str]]:
"""Yield (ancestor columns, parent id) for every parent the child endpoint fans out over.
@@ -412,17 +419,19 @@ def _iter_fan_out_parents(
formats the parent path with each grandparent id, tagging every parent with that id.
"""
if fan_out.grandparent is None:
- for parent_id in _iter_resource_ids(session, headers, logger, fan_out.parent_path, fan_out.parent_page_size):
+ for parent_id in _iter_resource_ids(
+ session, headers, logger, fan_out.parent_path, fan_out.parent_page_size, safe_point
+ ):
yield {}, parent_id
return
grandparent = fan_out.grandparent
for grandparent_id in _iter_resource_ids(
- session, headers, logger, grandparent.parent_path, grandparent.parent_page_size
+ session, headers, logger, grandparent.parent_path, grandparent.parent_page_size, safe_point
):
path = fan_out.parent_path.format(**{grandparent.parent_id_column: grandparent_id})
try:
- for parent_id in _iter_resource_ids(session, headers, logger, path, fan_out.parent_page_size):
+ for parent_id in _iter_resource_ids(session, headers, logger, path, fan_out.parent_page_size, safe_point):
yield {grandparent.parent_id_column: grandparent_id}, parent_id
except requests.HTTPError as exc:
# A grandparent deleted between enumeration and this fetch 404s; its children are gone
@@ -476,7 +485,8 @@ def _get_fan_out_rows(
"""
fan_out = config.fan_out
assert fan_out is not None
- parents = list(_iter_fan_out_parents(session, headers, logger, fan_out))
+ # Parents are listed before any row is batched, so each parent page is a safe point.
+ parents = list(_iter_fan_out_parents(session, headers, logger, fan_out, resumable_source_manager.safe_point))
parent_ids = [parent_id for _, parent_id in parents]
# Resolve the saved parent-ID bookmark to the slice of parents still to process. If the
@@ -515,6 +525,9 @@ def _get_fan_out_rows(
KlaviyoResumeConfig(next_url=next_url, list_id=parent_id)
)
+ if not batcher.should_yield(include_incomplete_chunk=True):
+ resumable_source_manager.safe_point()
+
if not next_url:
break
url = next_url
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
index 97fad6640413..972e1e7ad49f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
@@ -10,6 +10,7 @@ it is redacted from logs and raised error messages.
import dataclasses
from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime
+from functools import partial
from typing import Any, Optional
from urllib.parse import urljoin, urlparse
@@ -29,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
JSONResponsePaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
EndpointResource,
@@ -321,7 +323,7 @@ def _memories_pages(
db_incremental_field_last_value: Any,
resume_hook: Optional[Callable[[Optional[dict[str, Any]]], None]] = None,
initial_paginator_state: dict[str, Any] | None = None,
-) -> Iterator[list[Any]]:
+) -> Resource:
config = MEM0_ENDPOINTS[MEMORIES_ENDPOINT]
rest_config: RESTAPIConfig = {
@@ -351,7 +353,13 @@ def _memories_pages(
initial_paginator_state=initial_paginator_state,
)
- return _pages_until_page_rejected(resource, resumed=initial_paginator_state is not None)
+ # The rejection guard hands each page on unchanged and holds no rows. As a `Resource` it keeps
+ # the framework's per-page safe points.
+ return Resource(
+ partial(_pages_until_page_rejected, resource, resumed=initial_paginator_state is not None),
+ name=MEMORIES_ENDPOINT,
+ hints=resource._hints,
+ )
def _entities_resource(
@@ -455,7 +463,7 @@ def _memory_history_resource(
{
"name": MEMORIES_ENDPOINT,
"endpoint": {"path": MEM0_ENDPOINTS[MEMORIES_ENDPOINT].path, "method": "POST"},
- "data_iterator": lambda: _memories_pages(client, team_id, job_id, filters, None),
+ "data_iterator": lambda: iter(_memories_pages(client, team_id, job_id, filters, None)),
},
{
"name": MEMORY_HISTORY_ENDPOINT,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py
index aade691b8edb..477b824c0984 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py
@@ -34,6 +34,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
PageNumberPaginator,
SinglePagePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import Endpoint
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -165,10 +166,9 @@ def _make_bounded_session(
return session
[email protected](frozen=True)
class PlunkResumeConfig:
- # Exactly one of these is set, matching the endpoint's pagination style. Persisted after each
- # page is yielded, so a crash before the write re-yields the last page (merge dedupes on `id`).
+ # Exactly one of these is set, matching the endpoint's pagination style.
cursor: str = ""
page: int = 0
@@ -309,8 +309,7 @@ def plunk_source(
initial_paginator_state = {"page": resume.page}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist only when a next page remains; the hook fires AFTER a page is yielded so a crash
- # re-yields the last page (merge dedupes on the primary key) rather than skipping it.
+ # Persist only when a next page remains. The framework stages it with the page it covers.
if not state:
return
if state.get("cursor"):
@@ -348,7 +347,7 @@ def plunk_source(
],
}
- def items() -> Any:
+ def items() -> Resource:
# Re-check at run time (not just at source-create) in case the URL was edited or now
# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud. Refuse
# plaintext HTTP before the key is used. Both raise before any request leaves the process.
@@ -358,7 +357,7 @@ def plunk_source(
if not _is_https(resolved_base_url):
raise PlunkHostNotAllowedError(HTTP_NOT_ALLOWED_ERROR)
- yield from rest_api_resource(
+ return rest_api_resource(
rest_config,
team_id,
job_id,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
index 7d74ffab858c..541e4ad51649 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
@@ -27,6 +27,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
JSONResponsePaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
@@ -407,15 +408,21 @@ def revenuecat_api_source(
initial_paginator_state=initial_paginator_state,
)
- def items() -> Iterator[list[dict[str, Any]]]:
+ def dict_rows() -> Iterator[list[dict[str, Any]]]:
# The framework passes non-dict page entries through untouched (data_map only runs on
# dicts), so drop them here — matching the old client's per-row dict guard — before a
# malformed row can reach the delta writer.
for page in resource:
yield [row for row in page if isinstance(row, dict)]
+ # The framework staged the cursor after this page before it handed the page on, and the
+ # guard holds no rows, so the run can stop here.
+ resumable_source_manager.safe_point()
+
+ # As a `Resource` the guard also keeps the framework's safe points before each retry wait.
+ pages = Resource(dict_rows, name=schema_name, hints=resource._hints)
return SourceResponse(
- items=items,
+ items=lambda: pages,
primary_keys=endpoint.primary_keys,
name=schema_name,
partition_keys=endpoint.partition_keys,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index 45d5d308b3e9..eba4f8224e68 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -1,67 +1,39 @@
-Adjust fail:safe-point
-Adyen fail:safe-point
AmazonS3 not-checkable:schemas
-Anthropic fail:safe-point
-AppLovin fail:safe-point
AppStoreConnect not-checkable:no-request
AppleSearchAds not-checkable:no-request
AppsFlyer fail:safe-point
-Appsignal fail:safe-point
Attentive not-checkable:database
AwsAthena not-checkable:schemas
-AwsCostExplorer fail:safe-point
AwsDocumentDB not-checkable:driver:pymongo
AzureBlob not-checkable:schemas
AzureCostManagement not-checkable:no-request
AzureLogAnalytics not-checkable:schemas
Baserow not-checkable:schemas
-BigMailer fail:safe-point
BigQuery not-checkable:driver:google.cloud
BingAds not-checkable:database
-Bitbucket fail:safe-point
Breezometer not-checkable:no-request
-Bugsnag fail:safe-point
-CalCom fail:safe-point
-CanvasLms fail:safe-point
Chatwoot not-checkable:database
-CircleCI fail:safe-point
-CircleciInsights fail:safe-point
CiscoDuo not-checkable:no-request
ClickHouse not-checkable:schemas
CloudflareD1 not-checkable:schemas
CloudflareR2 not-checkable:schemas
Coda fail:safe-point
-Codecov fail:safe-point
-CoinMarketCap fail:safe-point
Convex not-checkable:schemas
CosmosDB not-checkable:schemas
Coveralls not-checkable:no-request
-Cursor fail:safe-point
Custom not-checkable:schemas
Databricks not-checkable:driver:databricks
-Datadog fail:safe-point
Depot not-checkable:no-request
-Discourse fail:safe-point
-Dixa fail:safe-point
-Dockerhub fail:safe-point
Docusign not-checkable:no-request
Dynamics365 not-checkable:no-request
DynamoDB not-checkable:schemas
-EcbDataPortal fail:safe-point
Elasticsearch not-checkable:schemas
-Fastly fail:safe-point
Featurebase not-checkable:database
FileUpload not-checkable:no-request
-FinancialModelling fail:safe-point
Firebase not-checkable:schemas
-Flagsmith fail:safe-point
-Formbricks fail:safe-point
Framer not-checkable:no-request
-G2 fail:safe-point
-GainsightPx fail:safe-point
GcpCloudBilling not-checkable:no-request
Ghost not-checkable:no-request
-Gitguardian fail:safe-point
Github not-checkable:schemas
GoogleAds not-checkable:driver:google.ads
GoogleAnalytics not-checkable:database
@@ -73,94 +45,51 @@ GrafanaIRM not-checkable:no-request
Groq fail:safe-point
Hatchet not-checkable:no-request
HelpScout not-checkable:database
-Hitpay fail:safe-point
-Honeybadger fail:safe-point
Hubspot not-checkable:database
-Hyperspell fail:safe-point
Instagram not-checkable:database
-Instana fail:safe-point
Intercom not-checkable:database
Kafka not-checkable:driver:confluent_kafka
Kandji fail:safe-point
-KapaAI fail:safe-point
Kernel fail:safe-point
Kestra not-checkable:no-request
-Klaviyo fail:safe-point
-Lacework fail:safe-point
Linear not-checkable:database
LinkedinAds not-checkable:database
-LlamaCloud fail:safe-point
Logicmonitor not-checkable:no-request
-Lovable fail:safe-point
-Luma fail:safe-point
MSSQL not-checkable:driver:pymssql
-MailerSend fail:safe-point
-Mem0 fail:safe-point
-Mention fail:safe-point
MetaAds not-checkable:database
-Meteostat fail:safe-point
-MoEngage fail:safe-point
Modal not-checkable:driver:grpclib
MongoDB not-checkable:driver:pymongo
-MonteCarlo fail:safe-point
Motherduck not-checkable:driver:duckdb
MyHours fail:safe-point
MySQL not-checkable:driver:psycopg
Neo4j not-checkable:schemas
Neon not-checkable:driver:psycopg
-NewRelic fail:safe-point
-Notion fail:safe-point
-NpmRegistry fail:safe-point
OpenMeteo not-checkable:no-request
OpenWeather not-checkable:no-request
Optimizely fail:safe-point
-Ortto fail:safe-point
-PabblySubscriptionsBilling fail:safe-point
Pardot not-checkable:database
PinterestAds not-checkable:database
Pipedrive not-checkable:database
-Pipeliner fail:safe-point
PlanetScaleMySQL not-checkable:driver:psycopg
PlanetScalePostgres not-checkable:driver:psycopg
-Plunk fail:safe-point
Postgres not-checkable:driver:psycopg
-PowerBiAdmin fail:safe-point
ProofpointTap not-checkable:no-request
-Pylon fail:safe-point
RedditAds not-checkable:database
Redshift not-checkable:driver:psycopg
-Replicate fail:safe-point
-RevenueCat fail:safe-point
-RunPod fail:safe-point
SFTP not-checkable:transport
-SageHR fail:safe-point
Salesforce not-checkable:database
-Sevdesk fail:safe-point
-Shippo fail:safe-point
Slack not-checkable:schemas
-Smaily fail:safe-point
SnapchatAds not-checkable:database
Snowflake not-checkable:driver:snowflake
-Square fail:safe-point
Stripe not-checkable:database
-SumoLogic fail:safe-point
Supabase not-checkable:driver:psycopg
-Swarmia fail:safe-point
-Tailscale fail:safe-point
TemporalIO not-checkable:driver:temporalio.client
-TerraformCloud fail:safe-point
TikTokAds not-checkable:database
Trino not-checkable:schemas
TwitterAds not-checkable:database
-Ubidots fail:safe-point
-Unleash fail:safe-point
Vitally fail:safe-point
-WikipediaPageviews fail:safe-point
Wrike not-checkable:no-request
-Writesonic fail:safe-point
YouTubeAnalytics not-checkable:database
-ZapSign fail:safe-point
ZapierSupportedStorage fail:safe-point
-Zenduty fail:safe-point
shared-path:rest-client-paginate fail:safe-point
shared-path:rest-framework-wrapped-resource fail:safe-point
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
index acaae9b1d21d..b93fd76e54c1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resources,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
ClientConfig,
EndpointResource,
@@ -371,15 +372,17 @@ def zapsign_source(
if endpoint == DOCUMENTS_RESOURCE:
webhook_enabled = async_to_sync(webhook_source_manager.webhook_enabled)()
+ def dict_rows() -> Iterator[list[dict[str, Any]]]:
+ for page in resource:
+ yield [row for row in page if isinstance(row, dict)]
+
+ # The guard holds no rows between pages. As a `Resource` it keeps the framework's per-page safe points.
+ pages = Resource(dict_rows, name=endpoint, hints=resource._hints)
+
def items() -> Any:
if webhook_enabled:
return webhook_source_manager.get_items(table_transformer=_webhook_table_transformer)
-
- def pages() -> Iterator[list[dict[str, Any]]]:
- for page in resource:
- yield [row for row in page if isinstance(row, dict)]
-
- return pages()
+ return pages
partition_keys = PARTITION_KEYS.get(endpoint)
return SourceResponse(
diff --git a/products/warehouse_sources/sources/adjust/adjust.py b/products/warehouse_sources/sources/adjust/adjust.py
index e3e52cac5a2e..041b4c90fa0b 100644
--- a/products/warehouse_sources/sources/adjust/adjust.py
+++ b/products/warehouse_sources/sources/adjust/adjust.py
@@ -424,6 +424,8 @@ def get_rows(
params = build_report_params(report, window_start, window_end, tokens, metrics)
for rows in _iter_report_pages(session, params, report, logger):
window_rows.extend(unpivot_event_rows(rows, config.dimensions, event_names))
+ if not window_rows:
+ resumable_source_manager.safe_point()
if window_rows:
window_rows.sort(key=lambda row: str(row.get("day") or ""))
yield window_rows
@@ -432,6 +434,9 @@ def get_rows(
for rows in _iter_report_pages(session, params, report, logger):
if rows:
yield rows
+ # The staged cursor points at this window's start or earlier, so a resume re-reads
+ # the window and loses no rows.
+ resumable_source_manager.safe_point()
# Save AFTER yielding the window so a crash re-yields it rather than skipping it (merge
# dedupes on the dimension key).
diff --git a/products/warehouse_sources/sources/adyen/adyen.py b/products/warehouse_sources/sources/adyen/adyen.py
index 5c6a3efe7313..9a83d91c5e03 100644
--- a/products/warehouse_sources/sources/adyen/adyen.py
+++ b/products/warehouse_sources/sources/adyen/adyen.py
@@ -304,15 +304,13 @@ def _iter_cursor_pages(
resume: Optional[AdyenResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
resume_window = _to_datetime(resume.window_start) if resume is not None else None
- cursor = resume.cursor if resume is not None else None
+ page_cursor = resume.cursor if resume is not None else None
+ # Start the windows at the saved one. The default start moves with the clock, so the window
+ # grid of this attempt may not line up with the saved window start.
+ if resume_window is not None:
+ start = resume_window
for window_start, window_end in iter_windows(start, end):
- # Skip windows already walked to completion before the interruption.
- if resume_window is not None and window_start < resume_window:
- continue
- page_cursor = cursor if resume_window is not None and window_start == resume_window else None
- resume_window = None
-
while True:
params: dict[str, Any] = {
"balancePlatform": balance_platform,
@@ -326,19 +324,22 @@ def _iter_cursor_pages(
payload = _request(session, _build_url(host, config.path, params), logger).json()
items = extract_items(payload, config.data_key)
+ page_cursor = next_cursor(payload)
+ if page_cursor:
+ manager.save_state(
+ AdyenResumeConfig(window_start=_format_timestamp(window_start), cursor=page_cursor),
+ )
+ else:
+ manager.save_state(AdyenResumeConfig(window_start=_format_timestamp(window_end)))
+
if items:
yield items
+ else:
+ manager.safe_point()
- page_cursor = next_cursor(payload)
if not page_cursor:
break
- # Checkpoint after yielding, so a crash re-yields the last batch (merge dedupes on
- # the primary key) instead of skipping it.
- manager.save_state(
- AdyenResumeConfig(window_start=_format_timestamp(window_start), cursor=page_cursor),
- )
-
def _iter_offset_pages(
session: requests.Session,
diff --git a/products/warehouse_sources/sources/adyen/tests/test_adyen.py b/products/warehouse_sources/sources/adyen/tests/test_adyen.py
index 422f96cdfe56..613e8cc62861 100644
--- a/products/warehouse_sources/sources/adyen/tests/test_adyen.py
+++ b/products/warehouse_sources/sources/adyen/tests/test_adyen.py
@@ -309,7 +309,7 @@ class TestCursorPagination:
)
)
- def test_state_is_saved_after_each_yielded_page(self) -> None:
+ def test_state_is_staged_for_each_page_and_finished_window(self) -> None:
session = _FakeSession(
[
_FakeResponse(json_data={"data": [{"id": "t1"}], "_links": {"next": {"href": "https://x?cursor=c2"}}}),
@@ -320,8 +320,10 @@ class TestCursorPagination:
self._pages(session, manager)
- assert [state.cursor for state in manager.saved_states] == ["c2"]
- assert manager.saved_states[0].window_start == "2026-01-01T00:00:00Z"
+ assert manager.saved_states == [
+ AdyenResumeConfig(window_start="2026-01-01T00:00:00Z", cursor="c2"),
+ AdyenResumeConfig(window_start="2026-01-02T00:00:00Z"),
+ ]
def test_resume_starts_from_the_saved_cursor(self) -> None:
session = _FakeSession([_FakeResponse(json_data={"data": [{"id": "t9"}]})])
diff --git a/products/warehouse_sources/sources/anthropic/anthropic.py b/products/warehouse_sources/sources/anthropic/anthropic.py
index 16901cb8d09a..ac31b3a2a0fa 100644
--- a/products/warehouse_sources/sources/anthropic/anthropic.py
+++ b/products/warehouse_sources/sources/anthropic/anthropic.py
@@ -962,7 +962,18 @@ def anthropic_source(
# Built for the first day only to carry the resource's column hints; the fan-out below
# rebuilds a resource per day.
resource = build_analytics_resource(analytics_windows[0])
- items = partial(_iter_analytics_windows, build_analytics_resource, analytics_windows, save_analytics_checkpoint)
+ # The day loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the
+ # framework's per-page safe points, so a run of empty days can hand off and commit its day.
+ analytics_days = Resource(
+ partial(_iter_analytics_windows, build_analytics_resource, analytics_windows, save_analytics_checkpoint),
+ name=endpoint,
+ hints=resource._hints,
+ )
+
+ def analytics_items() -> Iterable[list[dict[str, Any]]]:
+ return analytics_days
+
+ items = analytics_items
elif config.fan_out_over_days:
# Claude Code analytics: one windowed request per calendar day, driven entirely by the
# paginator (there is no parent API resource to resolve days from).
diff --git a/products/warehouse_sources/sources/applovin/applovin.py b/products/warehouse_sources/sources/applovin/applovin.py
index 48335a44dbbf..4e0a627abe6c 100644
--- a/products/warehouse_sources/sources/applovin/applovin.py
+++ b/products/warehouse_sources/sources/applovin/applovin.py
@@ -234,27 +234,31 @@ def get_rows(
while True:
rows = _fetch_page(session, config, api_key, start, window_end, offset, logger)
+ # A short page means the window is exhausted: AppLovin returns an empty set once
+ # `offset` passes the total row count.
+ window_done = len(rows) < REPORT_PAGE_SIZE
+ if window_done:
+ next_start = window_end + timedelta(days=1)
+ if next_start <= today:
+ resumable_source_manager.save_state(
+ AppLovinResumeConfig(next_window_start=next_start.isoformat(), next_offset=0)
+ )
+ else:
+ offset += REPORT_PAGE_SIZE
+ resumable_source_manager.save_state(
+ AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=offset)
+ )
+
if rows:
yield rows
+ else:
+ resumable_source_manager.safe_point()
- # A short page means the window is exhausted: AppLovin returns an empty set once
- # `offset` passes the total row count.
- if len(rows) < REPORT_PAGE_SIZE:
+ if window_done:
break
- offset += REPORT_PAGE_SIZE
- # Saved after yielding so a crash re-yields the in-flight page instead of skipping
- # it (the merge dedupes on the dimension primary key).
- resumable_source_manager.save_state(
- AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=offset)
- )
-
start = window_end + timedelta(days=1)
offset = 0
- if start <= today:
- resumable_source_manager.save_state(
- AppLovinResumeConfig(next_window_start=start.isoformat(), next_offset=0)
- )
# The whole window is walked, so drop the checkpoint rather than leaving a later attempt to
# resume from the final report window.
diff --git a/products/warehouse_sources/sources/applovin/tests/test_applovin.py b/products/warehouse_sources/sources/applovin/tests/test_applovin.py
index 9ce3683395aa..be39b1092c87 100644
--- a/products/warehouse_sources/sources/applovin/tests/test_applovin.py
+++ b/products/warehouse_sources/sources/applovin/tests/test_applovin.py
@@ -120,7 +120,9 @@ class TestWindowWalking:
assert isinstance(result.raised, AppLovinAPIError)
assert result.saved_states
- assert result.final_state == AppLovinResumeConfig(next_window_start=second_window.isoformat())
+ # Both windows were handed on, so the commit points at the window that failed.
+ third_window = second_window + timedelta(days=REPORT_WINDOW_DAYS)
+ assert result.final_state == AppLovinResumeConfig(next_window_start=third_window.isoformat())
class TestPagination:
diff --git a/products/warehouse_sources/sources/appsignal/appsignal.py b/products/warehouse_sources/sources/appsignal/appsignal.py
index be57eae8cdac..eb531e2dc68a 100644
--- a/products/warehouse_sources/sources/appsignal/appsignal.py
+++ b/products/warehouse_sources/sources/appsignal/appsignal.py
@@ -335,6 +335,8 @@ def _get_windowed_rows(
windows: deque[tuple[int, int]] = deque([(start, now)])
while windows:
+ # Every row of the windows before this one is yielded, and the staged start is not past them.
+ resumable_source_manager.safe_point()
window_since, window_before = windows.popleft()
count = _window_count(session, api_token, url, config, window_since, window_before, logger)
if count == 0:
@@ -823,6 +825,7 @@ def _get_metric_timeseries_rows(
)
if not rows:
+ resumable_source_manager.safe_point()
continue
rows.sort(key=lambda row: _to_epoch(row.get("timestamp")) or 0)
@@ -963,6 +966,7 @@ def _get_performance_trace_rows(
for window in _iter_windows(start, now, TRACES_WINDOW_SECONDS):
traces = _window_traces(session, api_token, app_id, window.since, window.before, logger)
if not traces:
+ resumable_source_manager.safe_point()
continue
yield traces
resumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))
@@ -1033,6 +1037,8 @@ def _get_trace_span_rows(
AppsignalResumeConfig(window_start=_to_epoch(trace_time) or window.since)
)
spans, batch_traces = [], 0
+ if not spans:
+ resumable_source_manager.safe_point()
if spans:
yield spans
@@ -1042,6 +1048,7 @@ def _get_trace_span_rows(
)
return
resumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))
+ resumable_source_manager.safe_point()
def _fetch_deploy_stats(
@@ -1105,6 +1112,7 @@ def _walk_aggregate_windows(
if rows:
yield rows
resumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))
+ resumable_source_manager.safe_point()
def _fetch_tracing_range_or_empty_if_aged_out(
@@ -1364,7 +1372,6 @@ def _get_slow_event_action_rows(
return
bucket = _to_iso(window.since)
- rows: list[dict[str, Any]] = []
for event in _fetch_slow_events(session, api_token, app_id, window, logger):
digests_fetched += 1
@@ -1381,15 +1388,19 @@ def _get_slow_event_action_rows(
"digest": digest,
},
)
- rows.extend(
+ rows = [
{**action, "digest": digest, "timestamp": bucket}
for action in actions
if isinstance(action, dict) and action.get("namespace") and action.get("action_name")
- )
+ ]
+ # The staged start still points at this bucket, so a resume re-reads it in full.
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
- if rows:
- yield rows
resumable_source_manager.save_state(AppsignalResumeConfig(window_start=window.before))
+ resumable_source_manager.safe_point()
buckets_done += 1
diff --git a/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py b/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py
index 69046507261b..073548b2e132 100644
--- a/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py
+++ b/products/warehouse_sources/sources/aws_cost_explorer/aws_cost_explorer.py
@@ -487,6 +487,7 @@ def get_rows(
window_services = list_window_services(session, credentials, window)
if not window_services:
page_token = None
+ resumable_source_manager.safe_point()
continue
while True:
@@ -508,6 +509,7 @@ def get_rows(
window_start=window.start.isoformat(), next_page_token=page_token, slice_key=slice_key
)
)
+ resumable_source_manager.safe_point()
if not page_token:
break
diff --git a/products/warehouse_sources/sources/bigmailer/bigmailer.py b/products/warehouse_sources/sources/bigmailer/bigmailer.py
index 91ff79edf050..0ee00fa75854 100644
--- a/products/warehouse_sources/sources/bigmailer/bigmailer.py
+++ b/products/warehouse_sources/sources/bigmailer/bigmailer.py
@@ -1,5 +1,6 @@
import dataclasses
from collections.abc import Iterator
+from functools import partial
from typing import Any, Optional
from requests import Response
@@ -228,7 +229,8 @@ def bigmailer_source(
return SourceResponse(
name=endpoint,
- items=lambda: _raise_auth_errors(resource),
+ # The wrapper hands each page on unchanged, so as a `Resource` it keeps the framework safe points.
+ items=lambda: Resource(partial(_raise_auth_errors, resource), name=endpoint, hints=resource._hints),
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/sources/bitbucket/bitbucket.py b/products/warehouse_sources/sources/bitbucket/bitbucket.py
index 46e44e817d0f..659acd65a409 100644
--- a/products/warehouse_sources/sources/bitbucket/bitbucket.py
+++ b/products/warehouse_sources/sources/bitbucket/bitbucket.py
@@ -299,7 +299,10 @@ def _map_rows(endpoint: str, items: list[dict[str, Any]], repo: dict[str, Any] |
def _iter_repositories(
- session: requests.Session, workspace: str, logger: FilteringBoundLogger
+ session: requests.Session,
+ workspace: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[BitbucketResumeConfig],
) -> Iterator[dict[str, Any]]:
"""Page through the workspace's repositories, oldest-first (created_on is immutable,
so the enumeration order is stable across resume attempts)."""
@@ -307,6 +310,8 @@ def _iter_repositories(
while True:
data = _fetch_page(session, url, logger)
yield from data.get("values", [])
+ # The repo list holds parents, not rows, so the walk can stop here and restart.
+ resumable_source_manager.safe_point()
next_url = data.get("next")
if not next_url:
break
@@ -351,6 +356,7 @@ def _get_top_level_rows(
# the last page rather than skipping it — merge dedupes on the primary key.
if next_url:
resumable_source_manager.save_state(BitbucketResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
if not next_url:
break
@@ -367,7 +373,9 @@ def _get_fan_out_rows(
cutoff: datetime | None,
cursor_field: str | None,
) -> Iterator[list[dict[str, Any]]]:
- repos = [repo for repo in _iter_repositories(session, workspace, logger) if repo.get("slug")]
+ repos = [
+ repo for repo in _iter_repositories(session, workspace, logger, resumable_source_manager) if repo.get("slug")
+ ]
# Resolve the saved repo bookmark to the slice still to process. If the bookmarked
# repo no longer exists, start over from the first repo — merge dedupes re-pulled rows.
@@ -405,6 +413,7 @@ def _get_fan_out_rows(
resumable_source_manager.save_state(
BitbucketResumeConfig(next_url=next_url, repo_slug=repo["slug"])
)
+ resumable_source_manager.safe_point()
if not next_url:
break
@@ -449,6 +458,7 @@ def _iter_fan_out_parents(
repo_slug: str,
logger: FilteringBoundLogger,
since: Any,
+ resumable_source_manager: ResumableSourceManager[BitbucketResumeConfig],
) -> Iterator[dict[str, Any]]:
"""Walk one repository's parent rows for a two-level fan-out, reusing the parent
endpoint's own page size and params so the list stays defined in one place."""
@@ -485,6 +495,8 @@ def _iter_fan_out_parents(
)
return
+ # The caller has handled every parent yielded so far and holds no rows.
+ resumable_source_manager.safe_point()
if not next_url:
return
url = next_url
@@ -506,7 +518,9 @@ def _get_second_level_fan_out_rows(
An incremental sync narrows the parent walk to what changed since the watermark. Without
that bound every sync would issue one request per parent that ever existed, which no
workspace's rate budget survives."""
- repos = [repo for repo in _iter_repositories(session, workspace, logger) if repo.get("slug")]
+ repos = [
+ repo for repo in _iter_repositories(session, workspace, logger, resumable_source_manager) if repo.get("slug")
+ ]
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
remaining = repos
@@ -532,7 +546,9 @@ def _get_second_level_fan_out_rows(
bookmark_url = resume_url if index == 0 else None
try:
- for parent in _iter_fan_out_parents(session, fan_out, workspace, repo["slug"], logger, parent_since):
+ for parent in _iter_fan_out_parents(
+ session, fan_out, workspace, repo["slug"], logger, parent_since, resumable_source_manager
+ ):
cursor = parent.get(fan_out.order_field)
if bookmark_cursor is not None and _parent_precedes_bookmark(fan_out.order, cursor, bookmark_cursor):
continue
@@ -569,6 +585,7 @@ def _get_second_level_fan_out_rows(
if rows:
yield rows
resumable_source_manager.save_state(dataclasses.replace(bookmark, next_url=next_url))
+ resumable_source_manager.safe_point()
if not next_url:
break
diff --git a/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py b/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py
index e0400e3b72bc..11a00758206d 100644
--- a/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py
+++ b/products/warehouse_sources/sources/bitbucket/tests/test_bitbucket.py
@@ -491,7 +491,7 @@ def test_fan_out_parent_walk_stops_at_the_cap_and_warns():
logger = MagicMock()
with scripted_network([_page(page)]):
- parents = list(_iter_fan_out_parents(requests.Session(), fan_out, "ws", "repo-a", logger, None))
+ parents = list(_iter_fan_out_parents(requests.Session(), fan_out, "ws", "repo-a", logger, None, MagicMock()))
assert [parent["uuid"] for parent in parents] == ["{p1}", "{p2}"]
logger.warning.assert_called_once()
diff --git a/products/warehouse_sources/sources/bugsnag/bugsnag.py b/products/warehouse_sources/sources/bugsnag/bugsnag.py
index 332bbf66e249..039cf3cbbb6b 100644
--- a/products/warehouse_sources/sources/bugsnag/bugsnag.py
+++ b/products/warehouse_sources/sources/bugsnag/bugsnag.py
@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
from products.warehouse_sources.sources.sdk.internals import Batcher
@@ -234,7 +235,10 @@ def _iter_all_pages(
tolerated_statuses: tuple[int, ...] = (),
paginate_by_offset: bool = False,
) -> Iterator[dict[str, Any]]:
- """Yield every item across all pages of a list endpoint, following its next-page cursor."""
+ """Yield every item across all pages of a list endpoint, following its next-page cursor.
+
+ Only parent listings use this, and they run before any row is batched, so each page is a safe point.
+ """
while True:
items, next_url = _fetch_list_page(
session,
@@ -245,6 +249,7 @@ def _iter_all_pages(
paginate_by_offset=paginate_by_offset,
)
yield from items
+ reach_safe_point()
if not next_url:
return
url = next_url
@@ -500,6 +505,12 @@ def _resolve_parents(
return parents
+def _safe_point_if_drained(batcher: Batcher, manager: ResumableSourceManager[BugsnagResumeConfig]) -> None:
+ """Reach a safe point only when the shared batcher holds no rows the staged cursor skips."""
+ if not batcher.should_yield(include_incomplete_chunk=True):
+ manager.safe_point()
+
+
def _iter_top_level(
session: requests.Session,
headers: dict[str, str],
@@ -538,6 +549,7 @@ def _iter_top_level(
yield batcher.get_table()
# Save AFTER yielding so a crash resumes at this page rather than losing buffered rows.
manager.save_state(BugsnagResumeConfig(next_url=checkpoint_url))
+ _safe_point_if_drained(batcher, manager)
if not next_url:
break
url = next_url
@@ -584,6 +596,7 @@ def _iter_fan_out(
if batcher.should_yield():
yield batcher.get_table()
manager.save_state(BugsnagResumeConfig(next_url=None, parent_id=parent.resume_id))
+ _safe_point_if_drained(batcher, manager)
continue
page_count = 0
@@ -610,6 +623,7 @@ def _iter_fan_out(
if batcher.should_yield():
yield batcher.get_table()
manager.save_state(BugsnagResumeConfig(next_url=checkpoint_url, parent_id=parent.resume_id))
+ _safe_point_if_drained(batcher, manager)
if not next_url:
break
if config.max_pages is not None and page_count >= config.max_pages:
diff --git a/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py b/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py
index 5c4583d2a14c..0c67767cf126 100644
--- a/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py
+++ b/products/warehouse_sources/sources/bugsnag/tests/test_bugsnag.py
@@ -36,6 +36,9 @@ class _FakeResumableManager:
def save_state(self, data: BugsnagResumeConfig) -> None:
self.saved.append(data)
+ def safe_point(self) -> None:
+ pass
+
def _make_response(status_code: int, body: Any = None, link: str | None = None) -> requests.Response:
response = requests.Response()
diff --git a/products/warehouse_sources/sources/cal_com/cal_com.py b/products/warehouse_sources/sources/cal_com/cal_com.py
index ce4112e75523..14d6093d1bd1 100644
--- a/products/warehouse_sources/sources/cal_com/cal_com.py
+++ b/products/warehouse_sources/sources/cal_com/cal_com.py
@@ -25,10 +25,12 @@ from products.warehouse_sources.sources.sdk import (
JSONResponseCursorPaginator,
OffsetPaginator,
RESTAPIConfig,
+ RESTClient,
ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
build_dependent_resource,
+ create_auth,
frozen,
make_tracked_session,
rest_api_resource,
@@ -460,40 +462,32 @@ def _booking_attendees_items(
parent_params["limit"] = parent_config.page_size
parent_client_config = _client_config(region, api_key, parent_config)
- parent_client_config["paginator"] = (
- CalComBookingsWindowPaginator(now=datetime.now(UTC)) if windowed else _make_paginator(parent_config)
+ bookings_client = RESTClient(
+ base_url=parent_client_config["base_url"],
+ headers=parent_client_config.get("headers"),
+ auth=create_auth(parent_client_config.get("auth")),
+ paginator=CalComBookingsWindowPaginator(now=datetime.now(UTC)) if windowed else _make_paginator(parent_config),
+ capture=parent_client_config.get("capture", True),
)
- parent_rest_config: RESTAPIConfig = {
- "client": parent_client_config,
- "resources": [
- {
- "name": BOOKING_ATTENDEES_PARENT,
- "endpoint": {
- "path": parent_config.path,
- "params": parent_params,
- "data_selector": "data",
- "data_selector_required": True,
- },
- }
- ],
- }
-
initial_paginator_state: Optional[dict[str, Any]] = None
if resumable_source_manager.can_resume():
initial_paginator_state = _resume_paginator_state(resumable_source_manager.load_state(), windowed)
- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # The hook runs when this generator asks for the next page, so after its rows are out.
- _save_cursor_state(resumable_source_manager, state)
+ # The bookings are paged here rather than through a `Resource`: the next-page state must stay
+ # unstaged until every attendee request of the current page is done.
+ next_page: dict[str, Optional[dict[str, Any]]] = {"state": None}
- bookings = rest_api_resource(
- parent_rest_config,
- team_id,
- job_id,
- None,
- resume_hook=save_checkpoint,
+ def remember_next_page(state: Optional[dict[str, Any]], _has_next_page: bool) -> None:
+ next_page["state"] = state
+
+ bookings = bookings_client.paginate(
+ path=parent_config.path,
+ params=parent_params,
+ data_selector="data",
+ data_selector_required=True,
initial_paginator_state=initial_paginator_state,
+ page_state_hook=remember_next_page,
)
# capture=False for the same reason as `_client_config`: attendee rows are contact details.
@@ -529,8 +523,11 @@ def _booking_attendees_items(
"bookingUpdatedAt": booking.get("updatedAt"),
}
)
+ _save_cursor_state(resumable_source_manager, next_page["state"])
if rows:
yield rows
+ else:
+ resumable_source_manager.safe_point()
def cal_com_source(
diff --git a/products/warehouse_sources/sources/canvas_lms/canvas_lms.py b/products/warehouse_sources/sources/canvas_lms/canvas_lms.py
index cbddb19430a2..a58ab267a79f 100644
--- a/products/warehouse_sources/sources/canvas_lms/canvas_lms.py
+++ b/products/warehouse_sources/sources/canvas_lms/canvas_lms.py
@@ -224,6 +224,12 @@ def validate_credentials(
return False, f"Could not connect to Canvas (HTTP {response.status_code})"
+def _check_host(domain: str, team_id: int) -> None:
+ host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)
+ if not host_ok:
+ raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)
+
+
def _non_fanout_source(
config: CanvasEndpointConfig,
domain: str,
@@ -274,10 +280,10 @@ def _non_fanout_source(
def items() -> Iterable[Any]:
# Re-check at run time (not just at source-create) in case the domain was edited or now
# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.
- host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)
- if not host_ok:
- raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)
- yield from resource
+ _check_host(domain, team_id)
+ # Return the framework resource itself, not a generator around it, so the framework
+ # reaches its per-page safe points.
+ return resource
return _make_source_response(config, items)
@@ -340,10 +346,8 @@ def _fanout_source(
)
def items() -> Iterable[Any]:
- host_ok, host_err = _is_host_safe(normalize_domain(domain), team_id)
- if not host_ok:
- raise CanvasHostNotAllowedError(host_err or HOST_NOT_ALLOWED_ERROR)
- yield from dependent_resource
+ _check_host(domain, team_id)
+ return dependent_resource
return _make_source_response(config, items)
diff --git a/products/warehouse_sources/sources/circleci/circleci.py b/products/warehouse_sources/sources/circleci/circleci.py
index 6f220d14e7f8..a2838ada5b76 100644
--- a/products/warehouse_sources/sources/circleci/circleci.py
+++ b/products/warehouse_sources/sources/circleci/circleci.py
@@ -405,14 +405,21 @@ def _jobs_for_workflow(
def _workflow_rows(
- fetch_page: FetchPageFn, pipelines: list[dict[str, Any]], logger: FilteringBoundLogger
+ fetch_page: FetchPageFn,
+ pipelines: list[dict[str, Any]],
+ logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
for pipeline in pipelines:
yield from _workflows_for_pipeline(fetch_page, pipeline["id"], logger)
+ safe_point()
def _job_rows(
- fetch_page: FetchPageFn, pipelines: list[dict[str, Any]], logger: FilteringBoundLogger
+ fetch_page: FetchPageFn,
+ pipelines: list[dict[str, Any]],
+ logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
for pipeline in pipelines:
for workflows in _workflows_for_pipeline(fetch_page, pipeline["id"], logger):
@@ -431,6 +438,8 @@ def _job_rows(
}
for job in jobs
]
+ safe_point()
+ safe_point()
def _project_rows(
@@ -448,7 +457,10 @@ def _project_rows(
def _component_version_rows(
- fetch_page: FetchPageFn, components: list[dict[str, Any]], logger: FilteringBoundLogger
+ fetch_page: FetchPageFn,
+ components: list[dict[str, Any]],
+ logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
for component in components:
component_id = component["id"]
@@ -469,6 +481,7 @@ def _component_version_rows(
}
for version in versions
]
+ safe_point()
def _user_rows(
@@ -476,6 +489,7 @@ def _user_rows(
pipelines: list[dict[str, Any]],
logger: FilteringBoundLogger,
seen_user_ids: set[str],
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
# v2 has no "list users in org" endpoint, so the actor ids carried on workflows are
# resolved one at a time via GET /user/{id}.
@@ -496,6 +510,7 @@ def _user_rows(
continue
raise
yield [user]
+ safe_point()
def _iter_v3_pages(
@@ -673,6 +688,8 @@ def _get_v3_rows(
batches = _v3_run_rows(fetch_page, endpoint, projects, logger)
yield from _stage_cursor_before_last_batch(batches, resumable_source_manager, next_cursor)
+ # The page's batches are all handed on, so the staged projects cursor covers every row.
+ resumable_source_manager.safe_point()
def get_rows(
@@ -713,25 +730,30 @@ def get_rows(
else:
pages = _iter_pipeline_pages(fetch_page, org_slug, logger, start_token)
+ # A fan-out parent's children are all handed on before this runs, and the staged token still
+ # reads the current page again, so it loses no rows.
+ safe_point = resumable_source_manager.safe_point
+
for items, next_token in pages:
if endpoint in DIRECT_LIST_ENDPOINTS:
if items:
yield items
elif endpoint == "workflows":
- yield from _workflow_rows(fetch_page, items, logger)
+ yield from _workflow_rows(fetch_page, items, logger, safe_point)
elif endpoint == "jobs":
- yield from _job_rows(fetch_page, items, logger)
+ yield from _job_rows(fetch_page, items, logger, safe_point)
elif endpoint == "projects":
yield from _project_rows(fetch_page, items, seen_project_slugs)
elif endpoint == "users":
- yield from _user_rows(fetch_page, items, logger, seen_user_ids)
+ yield from _user_rows(fetch_page, items, logger, seen_user_ids, safe_point)
elif endpoint == "component_versions":
- yield from _component_version_rows(fetch_page, items, logger)
+ yield from _component_version_rows(fetch_page, items, logger, safe_point)
# Save state after the page's rows (and any fan-out children) have been yielded, so a
# crash re-yields the in-progress page instead of skipping it.
if next_token:
resumable_source_manager.save_state(CircleCIResumeConfig(next_page_token=next_token))
+ safe_point()
def circleci_source(
diff --git a/products/warehouse_sources/sources/circleci/tests/test_circleci.py b/products/warehouse_sources/sources/circleci/tests/test_circleci.py
index bde17d5a27b0..2f405c2c0ef7 100644
--- a/products/warehouse_sources/sources/circleci/tests/test_circleci.py
+++ b/products/warehouse_sources/sources/circleci/tests/test_circleci.py
@@ -145,7 +145,8 @@ class TestPipelinesRows:
assert result.rows == [{"id": "p1"}]
assert result.params("page-token") == [None, "token-2"]
assert result.saved_states == [CircleCIResumeConfig(next_page_token="token-2")]
- assert result.committed_states == []
+ # The safe point after the page commits the cursor, because its rows are already handed on.
+ assert result.committed_states == [CircleCIResumeConfig(next_page_token="token-2")]
def test_unknown_endpoint_raises(self):
with pytest.raises(ValueError, match="Unknown CircleCI endpoint"):
diff --git a/products/warehouse_sources/sources/circleci_insights/circleci_insights.py b/products/warehouse_sources/sources/circleci_insights/circleci_insights.py
index f3fe196ca584..a370e56cab54 100644
--- a/products/warehouse_sources/sources/circleci_insights/circleci_insights.py
+++ b/products/warehouse_sources/sources/circleci_insights/circleci_insights.py
@@ -290,6 +290,7 @@ def _discover_workflow_names(
project_slug: str,
all_branches: bool,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> list[str]:
"""List the workflow names Insights knows for a project, from the workflow metrics listing.
Always uses the widest reporting window so fan-out doesn't miss workflows that only ran
@@ -307,6 +308,8 @@ def _discover_workflow_names(
name = item.get("name")
if name and name not in names:
names.append(name)
+ # The names are parents, not rows, so the staged cursor still covers every row handed on.
+ safe_point()
return names
@@ -384,6 +387,7 @@ def get_rows(
# Mark the slug fully synced so a resume moves straight to the next one.
resumable_source_manager.save_state(CircleciInsightsResumeConfig(slug=slug, slug_done=True))
+ resumable_source_manager.safe_point()
def _project_workflow_metrics_rows(
@@ -413,6 +417,7 @@ def _project_workflow_metrics_rows(
resumable_source_manager.save_state(
CircleciInsightsResumeConfig(slug=project_slug, next_page_token=next_token)
)
+ resumable_source_manager.safe_point()
def _project_workflow_fan_out_rows(
@@ -428,7 +433,9 @@ def _project_workflow_fan_out_rows(
db_incremental_field_last_value: Any,
) -> Iterator[list[dict[str, Any]]]:
config = CIRCLECI_INSIGHTS_ENDPOINTS[endpoint]
- workflow_names = _discover_workflow_names(fetch_page, project_slug, all_branches, logger)
+ workflow_names = _discover_workflow_names(
+ fetch_page, project_slug, all_branches, logger, resumable_source_manager.safe_point
+ )
params: dict[str, Any] = _endpoint_params(config, window, all_branches)
if all_branches and not config.takes_branch_params:
@@ -479,6 +486,7 @@ def _project_workflow_fan_out_rows(
slug=project_slug, workflow_name=workflow_name, next_page_token=next_token
)
)
+ resumable_source_manager.safe_point()
def _workflow_summary_row(data: dict[str, Any], project_slug: str, workflow_name: str) -> dict[str, Any]:
@@ -519,7 +527,9 @@ def _project_workflow_object_fan_out_rows(
"""Fan out over a project's workflows for endpoints that return a single object per workflow
instead of a paginated listing, so resume granularity is one workflow."""
config = CIRCLECI_INSIGHTS_ENDPOINTS[endpoint]
- workflow_names = _discover_workflow_names(fetch_page, project_slug, all_branches, logger)
+ workflow_names = _discover_workflow_names(
+ fetch_page, project_slug, all_branches, logger, resumable_source_manager.safe_point
+ )
params = _endpoint_params(config, window, all_branches)
resume_workflow = slug_resume.workflow_name if slug_resume is not None else None
@@ -547,6 +557,7 @@ def _project_workflow_object_fan_out_rows(
resumable_source_manager.save_state(
CircleciInsightsResumeConfig(slug=project_slug, workflow_name=workflow_name)
)
+ resumable_source_manager.safe_point()
def _project_flaky_tests_rows(
diff --git a/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py b/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py
index 80ed7672dd1e..d6ac09e121ad 100644
--- a/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py
+++ b/products/warehouse_sources/sources/circleci_insights/tests/test_circleci_insights.py
@@ -207,8 +207,11 @@ class TestWorkflowMetricsRows:
CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2"),
CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True),
]
- assert result.committed_states == [CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2")]
- assert result.final_state == CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2")
+ assert result.committed_states == [
+ CircleciInsightsResumeConfig(slug="gh/a/one", next_page_token="token-2"),
+ CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True),
+ ]
+ assert result.final_state == CircleciInsightsResumeConfig(slug="gh/a/one", slug_done=True)
def test_resume_skips_completed_project(self):
result = _driver("gh/a/one, gh/a/two").run(
diff --git a/products/warehouse_sources/sources/codecov/codecov.py b/products/warehouse_sources/sources/codecov/codecov.py
index 790855798343..62821d380dd4 100644
--- a/products/warehouse_sources/sources/codecov/codecov.py
+++ b/products/warehouse_sources/sources/codecov/codecov.py
@@ -218,7 +218,11 @@ def _iter_pages(
def _iter_repo_names(
- session: requests.Session, owner_base_url: str, headers: dict[str, str], logger: FilteringBoundLogger
+ session: requests.Session,
+ owner_base_url: str,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[CodecovResumeConfig],
) -> Iterator[str]:
"""Page through the owner's active repos and yield each name. Inactive repos (never
received an upload) have no coverage data, so fanning out over them is wasted API cost."""
@@ -226,6 +230,8 @@ def _iter_repo_names(
for results, _next_url in _iter_pages(session, url, headers, logger):
for item in results:
yield item["name"]
+ # The repo list holds parents, not rows, so the walk can stop here and restart.
+ resumable_source_manager.safe_point()
def _get_top_level_rows(
@@ -256,6 +262,7 @@ def _get_top_level_rows(
# merge dedupes on the primary key.
if next_url:
resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
def _remaining_repos(
@@ -292,7 +299,9 @@ def _get_fan_out_rows(
) -> Iterator[list[dict[str, Any]]]:
"""Fan out one paginated walk per repository, injecting the repo name onto every row (it
feeds each endpoint's composite primary key)."""
- repo_names = repositories or list(_iter_repo_names(session, owner_base_url, headers, logger))
+ repo_names = repositories or list(
+ _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
+ )
cutoff = _normalize_cutoff(db_incremental_field_last_value) if should_use_incremental_field else None
params: dict[str, Any] = {**config.extra_params}
@@ -327,6 +336,7 @@ def _get_fan_out_rows(
# re-yields the last page rather than skipping it.
if next_url and not stop_after_this_page:
resumable_source_manager.save_state(CodecovResumeConfig(next_url=next_url, repo=repo))
+ resumable_source_manager.safe_point()
if stop_after_this_page:
break
except requests.HTTPError as exc:
@@ -340,6 +350,7 @@ def _get_fan_out_rows(
# Advance the bookmark to the next repo so a crash between repos resumes correctly.
if index + 1 < len(remaining):
resumable_source_manager.save_state(CodecovResumeConfig(next_url=None, repo=remaining[index + 1]))
+ resumable_source_manager.safe_point()
def _iter_child_rows(
@@ -392,7 +403,9 @@ def _iter_parent_fan_out_batches(
) -> Iterator[tuple[list[dict[str, Any]], CodecovResumeConfig | None]]:
"""Yield (rows, state) per child response. `state` is set, with no rows, once every row of a
parent page has been yielded: it is the position to resume from after that page."""
- repo_names = repositories or list(_iter_repo_names(session, owner_base_url, headers, logger))
+ repo_names = repositories or list(
+ _iter_repo_names(session, owner_base_url, headers, logger, resumable_source_manager)
+ )
parent_params: dict[str, Any] = {"page_size": PAGE_SIZE}
child_params: dict[str, Any] = {**config.extra_params}
diff --git a/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py b/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py
index 7149a39ee4b3..06414c052c4a 100644
--- a/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py
+++ b/products/warehouse_sources/sources/coinmarketcap/coinmarketcap.py
@@ -241,12 +241,11 @@ def get_batch_rows(
batch = coin_ids[index : index + config.batch_size]
body = _get(session, config.path, {**params, config.id_param: ",".join(str(coin_id) for coin_id in batch)})
rows = _rows_from(config.row_shape, body.get("data"))
+ resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_coin_id=batch[-1]))
if rows:
yield rows
-
- # Saved after the batch is yielded, so a crash re-yields it rather than skipping it;
- # the merge de-duplicates on the endpoint's primary key.
- resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_coin_id=batch[-1]))
+ else:
+ resumable_source_manager.safe_point()
def _batch_source(
@@ -333,11 +332,11 @@ def get_snapshot_rows(
{**config.extra_params, "date": day.isoformat(), "start": 1, "limit": config.limit},
)
rows = [row for row in (body.get("data") or []) if isinstance(row, dict)]
+ resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_snapshot_date=day.isoformat()))
if rows:
yield [{**row, config.date_field: day.isoformat()} for row in rows]
-
- # Saved after the day is yielded, so a crash re-yields that day rather than skipping it.
- resumable_source_manager.save_state(CoinMarketCapResumeConfig(last_snapshot_date=day.isoformat()))
+ else:
+ resumable_source_manager.safe_point()
day += datetime.timedelta(days=1)
diff --git a/products/warehouse_sources/sources/cursor/cursor.py b/products/warehouse_sources/sources/cursor/cursor.py
index 47a5ac203884..7974554e69b0 100644
--- a/products/warehouse_sources/sources/cursor/cursor.py
+++ b/products/warehouse_sources/sources/cursor/cursor.py
@@ -570,16 +570,19 @@ def _paginate_window(
# no rows — so termination has to read the page's own size, not the row count.
page_items = _page_item_count(config, data, items)
has_next = config.paginated and _has_next_page(data, page, page_items, config.page_size)
+ window_done = not page_items or not has_next
+
+ if not window_done:
+ resumable_source_manager.save_state(CursorResumeConfig(window_start=window_start, page=page + 1))
+ elif window_end + 1 <= end_ms:
+ resumable_source_manager.save_state(CursorResumeConfig(window_start=window_end + 1, page=1))
if rows:
yield rows
- # Save AFTER yielding so a crash re-yields the last batch rather than skipping it.
- if has_next:
- resumable_source_manager.save_state(CursorResumeConfig(window_start=window_start, page=page + 1))
- elif window_end + 1 <= end_ms:
- resumable_source_manager.save_state(CursorResumeConfig(window_start=window_end + 1, page=1))
+ else:
+ resumable_source_manager.safe_point()
- if not page_items or not has_next:
+ if window_done:
break
page += 1
diff --git a/products/warehouse_sources/sources/cursor/tests/test_cursor.py b/products/warehouse_sources/sources/cursor/tests/test_cursor.py
index e9a0f93d6bb3..da2339c98303 100644
--- a/products/warehouse_sources/sources/cursor/tests/test_cursor.py
+++ b/products/warehouse_sources/sources/cursor/tests/test_cursor.py
@@ -442,7 +442,7 @@ class TestCursorTransport:
assert body["startDate"] == window_start
assert body["page"] == 4
- def test_windowed_saves_state_after_each_batch(self):
+ def test_windowed_stages_state_before_each_batch(self):
now_ms = 1_700_000_000_000
manager = _manager()
pages = [
@@ -481,12 +481,8 @@ class TestCursorTransport:
first_batch = next(iterator)
assert first_batch[0]["id"] # synthetic primary key was added
- # The generator is suspended at the yield — nothing saved until the consumer has
- # taken the batch and pulls the next one, so a crash mid-batch re-fetches it.
- assert manager.save_state.call_count == 0
-
- next(iterator)
- # Resuming past the first yield saved a bookmark at page 2 of the same window.
+ # The pipeline commits the staged cursor with the batch, so page 2 is staged before the yield.
+ assert manager.save_state.call_count == 1
saved = manager.save_state.call_args.args[0]
assert saved.page == 2
assert saved.window_start is not None
diff --git a/products/warehouse_sources/sources/datadog/datadog.py b/products/warehouse_sources/sources/datadog/datadog.py
index ce7dd1371b57..8da915d458ae 100644
--- a/products/warehouse_sources/sources/datadog/datadog.py
+++ b/products/warehouse_sources/sources/datadog/datadog.py
@@ -297,11 +297,15 @@ def _walk(
fetch_page: Callable[..., Any],
host: str,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
save_state: Callable[[str], None] | None = None,
allow_missing: bool = False,
max_pages: int | None = None,
) -> Iterator[list[dict[str, Any]]]:
- """Page through one endpoint URL, yielding a normalized batch per response."""
+ """Page through one endpoint URL, yielding a normalized batch per response.
+
+ `safe_point` runs after each page, once the caller has taken every row and the cursor is staged.
+ """
url = start_url
pages = 0
while True:
@@ -333,6 +337,7 @@ def _walk(
# Save state AFTER yielding the batch — a crash re-yields the last batch (merge dedupes
# on primary key) instead of skipping it.
save_state(next_url)
+ safe_point()
url = next_url
@@ -341,6 +346,7 @@ def _fan_out_rows(
fetch_page: Callable[..., Any],
host: str,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
"""Walk a parent endpoint and query the child endpoint once per parent id."""
fan_out = config.parent
@@ -353,7 +359,7 @@ def _fan_out_rows(
)
parents_seen = 0
- for parent_batch in _walk(parent_config, parent_url, fetch_page, host, logger):
+ for parent_batch in _walk(parent_config, parent_url, fetch_page, host, logger, safe_point):
for parent in parent_batch:
if parents_seen >= fan_out.max_parents:
# Returning here would write a truncated table that looks like a complete sync.
@@ -376,7 +382,7 @@ def _fan_out_rows(
)
# A parent deleted between the list call and its child call answers 404; skip it
# rather than failing the whole sync.
- for child_batch in _walk(config, child_url, fetch_page, host, logger, allow_missing=True):
+ for child_batch in _walk(config, child_url, fetch_page, host, logger, safe_point, allow_missing=True):
for row in child_batch:
row[fan_out.child_id_field] = parent_id
yield child_batch
@@ -415,7 +421,7 @@ def get_rows(
if config.parent is not None:
# A fan-out position is a parent cursor plus a child page, which the single-URL resume
# state cannot express, so these endpoints restart from the first parent instead.
- yield from _fan_out_rows(config, fetch_page, host, logger)
+ yield from _fan_out_rows(config, fetch_page, host, logger, resumable_source_manager.safe_point)
return
resume_config = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
@@ -440,6 +446,7 @@ def get_rows(
fetch_page,
host,
logger,
+ resumable_source_manager.safe_point,
save_state=lambda next_url: resumable_source_manager.save_state(DatadogResumeConfig(next_url=next_url)),
max_pages=max_pages,
)
diff --git a/products/warehouse_sources/sources/discourse/discourse.py b/products/warehouse_sources/sources/discourse/discourse.py
index ba3cb640604b..9219aadba634 100644
--- a/products/warehouse_sources/sources/discourse/discourse.py
+++ b/products/warehouse_sources/sources/discourse/discourse.py
@@ -1,7 +1,7 @@
import re
-from collections.abc import Callable, Iterable, Iterator
+from collections.abc import Callable
from datetime import datetime
-from typing import Any, Optional
+from typing import Any, Optional, cast
from urllib.parse import urlparse
import requests
@@ -23,6 +23,7 @@ from products.warehouse_sources.sources.sdk import (
IncrementalConfig,
OffsetPaginator,
PageNumberPaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SinglePagePaginator,
@@ -230,20 +231,16 @@ def _flatten_directory_item(item: dict[str, Any]) -> dict[str, Any]:
_POST_CONTENT_FIELDS = ("excerpt", "edit_reason")
-def _public_activity_rows(batch: list[dict[str, Any]]) -> list[dict[str, Any]]:
- """Reduce an activity page to public actions, without their post text.
+def _is_public_activity(row: dict[str, Any]) -> bool:
+ """`USER_ACTION_PUBLIC_TYPES` explains why the action types are narrowed here instead of in the request."""
+ return row.get("action_type") in USER_ACTION_PUBLIC_TYPES
- `USER_ACTION_PUBLIC_TYPES` explains why the action types are narrowed here instead of in the
- request, and `_POST_CONTENT_FIELDS` why the text is dropped.
- """
- rows: list[dict[str, Any]] = []
- for row in batch:
- if row.get("action_type") not in USER_ACTION_PUBLIC_TYPES:
- continue
- for field_name in _POST_CONTENT_FIELDS:
- row.pop(field_name, None)
- rows.append(row)
- return rows
+
+def _without_post_content(row: dict[str, Any]) -> dict[str, Any]:
+ """`_POST_CONTENT_FIELDS` explains why the text is dropped."""
+ for field_name in _POST_CONTENT_FIELDS:
+ row.pop(field_name, None)
+ return row
def _coerce_incremental_cursor(value: Any) -> Optional[int]:
@@ -340,7 +337,7 @@ def discourse_source(
return
resumable_source_manager.save_state(DiscourseResumeConfig(page=state.get("page"), before=state.get("before")))
- def build_resource() -> Iterable[list[dict[str, Any]]]:
+ def build_resource() -> Resource:
if endpoint_config.fanout is not None:
return _fanout_resource(
endpoint_config=endpoint_config,
@@ -365,14 +362,14 @@ def discourse_source(
initial_paginator_state=_flat_initial_state(endpoint_config, resume_config),
)
- def items() -> Iterator[list[dict[str, Any]]]:
+ def items() -> Resource:
# Re-checked at run time (not just at source-create) in case the instance URL was edited
# or now resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.
_check_host(base_url, team_id)
- for batch in build_resource():
- rows = _public_activity_rows(batch) if endpoint == "user_actions" else batch
- if rows:
- yield rows
+ resource = build_resource()
+ if endpoint == "user_actions":
+ resource.add_filter(_is_public_activity).add_map(_without_post_content)
+ return resource
return SourceResponse(
name=endpoint,
@@ -407,7 +404,7 @@ def _flat_resource(
should_use_incremental_field: bool,
resume_hook: Callable[[Optional[dict[str, Any]]], None],
initial_paginator_state: Optional[dict[str, Any]],
-) -> Iterable[list[dict[str, Any]]]:
+) -> Resource:
endpoint_dict: Endpoint = {
"path": endpoint_config.path,
**_RETRY_MALFORMED_BODY,
@@ -464,10 +461,10 @@ def _fanout_resource(
incremental_field: Optional[str],
resume_hook: Callable[[Optional[dict[str, Any]]], None],
initial_paginator_state: Optional[dict[str, Any]],
-) -> Iterable[list[dict[str, Any]]]:
+) -> Resource:
parent_config = DISCOURSE_ENDPOINTS[fanout.parent_name]
- return build_dependent_resource(
+ resource = build_dependent_resource(
endpoint_configs=DISCOURSE_ENDPOINTS,
child_endpoint=endpoint_config.name,
fanout=fanout,
@@ -486,6 +483,7 @@ def _fanout_resource(
resume_hook=resume_hook,
initial_paginator_state=initial_paginator_state,
)
+ return cast(Resource, resource)
def _get_session(api_key: str, api_username: str) -> requests.Session:
diff --git a/products/warehouse_sources/sources/dixa/dixa.py b/products/warehouse_sources/sources/dixa/dixa.py
index 3493639d7c4c..90b267825691 100644
--- a/products/warehouse_sources/sources/dixa/dixa.py
+++ b/products/warehouse_sources/sources/dixa/dixa.py
@@ -288,6 +288,8 @@ def _fan_out_rows(
if next_state is not None:
resumable_source_manager.save_state(next_state)
+ # Every child row of this parent batch is yielded, so the staged parent cursor loses nothing.
+ resumable_source_manager.safe_point()
def get_rows(
@@ -350,11 +352,12 @@ def get_rows(
resume_config, logger, endpoint, should_use_incremental_field, db_incremental_field_last_value
)
for items, window_end in _iter_export_windows(fetch, config, window_start):
+ # The whole window arrives in one response, so its end covers every row yielded below.
+ resumable_source_manager.save_state(DixaResumeConfig(window_start_ms=window_end))
if items:
yield items
- # Save state AFTER yielding the window so a crash re-yields it
- # (merge dedupes on primary key) rather than skipping it.
- resumable_source_manager.save_state(DixaResumeConfig(window_start_ms=window_end))
+ else:
+ resumable_source_manager.safe_point()
return
start_url = _main_start_url(
diff --git a/products/warehouse_sources/sources/dockerhub/dockerhub.py b/products/warehouse_sources/sources/dockerhub/dockerhub.py
index 6b416909f5e5..9fab61cadbfc 100644
--- a/products/warehouse_sources/sources/dockerhub/dockerhub.py
+++ b/products/warehouse_sources/sources/dockerhub/dockerhub.py
@@ -7,7 +7,7 @@ from urllib.parse import quote, urlencode, urlsplit
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.dockerhub.settings import DOCKERHUB_ENDPOINTS, ORG_SCOPED_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
@@ -15,6 +15,7 @@ from products.warehouse_sources.sources.sdk import (
SourceResponse,
frozen,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -101,10 +102,17 @@ def _tags_url(namespace: str, repository: str) -> str:
return f"{DOCKERHUB_BASE_URL}/v2/namespaces/{quote(namespace)}/repositories/{quote(repository)}/tags?{query}"
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows
+ # while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_jwt(
@@ -165,6 +173,7 @@ def _request_json(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -189,6 +198,7 @@ def _fetch_page(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_object(
@@ -266,6 +276,8 @@ def _cursor_pages(
while url:
items, next_url = client.get_page(url)
+ if next_url:
+ resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url))
if items:
yield items
@@ -274,12 +286,14 @@ def _cursor_pages(
break
url = next_url
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted); merge dedupes the re-pulled page on the primary key.
- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
-def _list_repository_names(client: DockerHubClient, namespace: str) -> list[str]:
+def _list_repository_names(
+ client: DockerHubClient,
+ namespace: str,
+ resumable_source_manager: ResumableSourceManager[DockerhubResumeConfig],
+) -> list[str]:
names: list[str] = []
url: Optional[str] = _repositories_url(namespace)
while url:
@@ -287,6 +301,8 @@ def _list_repository_names(client: DockerHubClient, namespace: str) -> list[str]
# `name` is the repository primary key and the fan-out key for tags: access it directly so a
# malformed row (missing name) fails fast instead of being silently dropped from the sync.
names.extend(item["name"] for item in items)
+ # The names are not rows, and this attempt has yielded nothing yet.
+ resumable_source_manager.safe_point()
return names
@@ -299,7 +315,7 @@ def _tag_pages(
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
# The repository list is small (names only) and alphabetically ordered, so materializing it keeps
# resume deterministic: we slice from the saved repository instead of comparing names.
- repositories = _list_repository_names(client, namespace)
+ repositories = _list_repository_names(client, namespace, resumable_source_manager)
start_index = 0
resume_next_url: Optional[str] = None
@@ -321,20 +337,19 @@ def _tag_pages(
while url:
items, next_url = client.get_page(url)
+ # On the last page, pin state to the completed repository so a crash before the next
+ # repository's first page re-syncs at most this one repository.
+ resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url, repository=repository))
if items:
# Tag rows only carry a numeric repository id; inject the namespace and repository
# name so rows are self-describing and the composite primary key is table-unique.
yield [{**row, "namespace": namespace, "repository_name": repository} for row in items]
+ resumable_source_manager.safe_point()
if not next_url:
break
url = next_url
- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=next_url, repository=repository))
-
- # Pin state to the completed repository so a crash before the next repository's first page
- # re-syncs at most this one repository.
- resumable_source_manager.save_state(DockerhubResumeConfig(next_url=None, repository=repository))
def _audit_log_row(row: dict[str, Any]) -> dict[str, Any]:
@@ -369,18 +384,19 @@ def _audit_log_pages(
f"Docker Hub returned an unexpected audit log payload on page {page}: {type(logs).__name__}"
)
+ # The endpoint returns neither a next link nor a total count, so a short page is the only
+ # end-of-collection signal it gives us.
+ has_next_page = len(logs) >= PAGE_SIZE
+ if has_next_page:
+ resumable_source_manager.save_state(DockerhubResumeConfig(page=page + 1))
if logs:
yield [_audit_log_row(row) for row in logs]
- # The endpoint returns neither a next link nor a total count, so a short page is the only
- # end-of-collection signal it gives us.
- if len(logs) < PAGE_SIZE:
+ if not has_next_page:
break
page += 1
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted); merge dedupes any re-pulled page on the primary key.
- resumable_source_manager.save_state(DockerhubResumeConfig(page=page))
+ resumable_source_manager.safe_point()
def _audit_log_action_rows(client: DockerHubClient, namespace: str) -> Iterator[list[dict[str, Any]]]:
diff --git a/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py b/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py
index d933eecaaec1..fca577b5d270 100644
--- a/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py
+++ b/products/warehouse_sources/sources/ecb_data_portal/ecb_data_portal.py
@@ -152,6 +152,8 @@ def ecb_data_portal_source(
resumable_source_manager.save_state(
ECBResumeConfig(next_start_period=(window.end + timedelta(days=1)).isoformat())
)
+ # Rows go out one at a time, so the window is staged only after its last row.
+ resumable_source_manager.safe_point()
else:
resumable_source_manager.clear_state()
diff --git a/products/warehouse_sources/sources/fastly/fastly.py b/products/warehouse_sources/sources/fastly/fastly.py
index b1e75ff245fe..9370e70f88a1 100644
--- a/products/warehouse_sources/sources/fastly/fastly.py
+++ b/products/warehouse_sources/sources/fastly/fastly.py
@@ -376,7 +376,10 @@ def _flatten_origin_inspector(payload: dict[str, Any], service_id: str) -> list[
def _iter_services(
- session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[FastlyResumeConfig],
) -> Iterator[dict[str, Any]]:
"""Page through GET /service, yielding each service object. Used to drive fan-out endpoints."""
url = _build_url(f"{FASTLY_BASE_URL}/service", {"per_page": SERVICE_PAGE_SIZE})
@@ -385,6 +388,8 @@ def _iter_services(
items = response.json()
if isinstance(items, list):
yield from items
+ # The service list holds parents, not rows, so the walk can stop here and restart.
+ resumable_source_manager.safe_point()
next_url = _next_page_url(response)
if not next_url:
break
@@ -465,6 +470,7 @@ def _get_json_api_rows(
raise FastlyPaginationError(f"Fastly returned an unchanged next page link for {url}")
# Saved AFTER yielding so a crash re-yields the last page rather than skipping it.
resumable_source_manager.save_state(FastlyResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
url = next_url
@@ -543,6 +549,8 @@ def _get_origin_inspector_rows(
break
if next_cursor == cursor:
raise FastlyPaginationError(f"Fastly returned an unchanged cursor for {url_base}")
+ # The staged bookmark restarts at this service or an earlier one, so no row is lost.
+ resumable_source_manager.safe_point()
cursor = next_cursor
@@ -628,6 +636,8 @@ def _get_platform_metrics_rows(
break
if next_cursor == cursor:
raise FastlyPaginationError(f"Fastly returned an unchanged cursor for {url_base}")
+ # The staged bookmark restarts at this service or an earlier one, so no row is lost.
+ resumable_source_manager.safe_point()
cursor = next_cursor
@@ -655,6 +665,7 @@ def _get_service_list_rows(
break
# Save AFTER yielding so a crash re-yields the last page rather than skipping it.
resumable_source_manager.save_state(FastlyResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
url = next_url
@@ -668,7 +679,7 @@ def _iter_services_from_bookmark(
"""Yield the service ids a fan-out endpoint still has to process, bookmarking each one once the
caller has yielded its rows. If the bookmarked service no longer exists (deleted between runs),
start over — merge dedupes the re-pulled rows."""
- services = list(_iter_services(session, headers, logger))
+ services = list(_iter_services(session, headers, logger, resumable_source_manager))
service_ids = [s["id"] for s in services if isinstance(s, dict) and s.get("id")]
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
@@ -682,6 +693,7 @@ def _iter_services_from_bookmark(
# Bookmark the service AFTER yielding so a crash resumes here and re-yields. Advanced even for
# a versionless (skipped) service so resume doesn't re-evaluate it every time.
resumable_source_manager.save_state(FastlyResumeConfig(service_id=service_id))
+ resumable_source_manager.safe_point()
def _get_version_scoped_rows(
@@ -759,6 +771,7 @@ def _get_version_resource_child_rows(
]
if rows:
yield rows
+ resumable_source_manager.safe_point()
def _iter_billing_pages(
@@ -795,6 +808,7 @@ def _iter_billing_pages(
cursor = next_cursor
# Saved AFTER yielding so a crash re-yields the last page rather than skipping it.
resumable_source_manager.save_state(FastlyResumeConfig(cursor=cursor, cursor_request=request))
+ resumable_source_manager.safe_point()
def _get_billing_list_rows(
diff --git a/products/warehouse_sources/sources/financial_modelling/financial_modelling.py b/products/warehouse_sources/sources/financial_modelling/financial_modelling.py
index 6d0f89ef19b6..900f260c7346 100644
--- a/products/warehouse_sources/sources/financial_modelling/financial_modelling.py
+++ b/products/warehouse_sources/sources/financial_modelling/financial_modelling.py
@@ -233,13 +233,17 @@ def get_rows(
batcher.batch(row)
if batcher.should_yield():
yield batcher.get_table()
+ # Flush after each request, before the bookmark advances past this symbol. A symbol can
+ # take one request per quarter, and the pipeline acts only on a yield or a safe point.
+ if batcher.should_yield(include_incomplete_chunk=True):
+ yield batcher.get_table()
+ else:
+ # The staged index still points at this symbol, so a resume re-fetches it in full.
+ resumable_source_manager.safe_point()
- # Flush this symbol's rows before advancing the bookmark, so a crash re-fetches only from
- # the next symbol (whose rows are not yet persisted) rather than dropping buffered rows.
- if batcher.should_yield(include_incomplete_chunk=True):
- yield batcher.get_table()
if index + 1 < len(symbols):
resumable_source_manager.save_state(FinancialModellingResumeConfig(symbol_index=index + 1))
+ resumable_source_manager.safe_point()
return
params = {**config.extra_params, **window}
diff --git a/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py b/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py
index a0621226f553..a0035f47d62b 100644
--- a/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py
+++ b/products/warehouse_sources/sources/financial_modelling/tests/test_financial_modelling.py
@@ -141,6 +141,9 @@ class _FakeResumableManager:
def save_state(self, data: FinancialModellingResumeConfig) -> None:
self.saved.append(data)
+ def safe_point(self) -> None:
+ pass
+
def _collect(
endpoint: str, symbols: list[str], manager: _FakeResumableManager, by_symbol: dict[str, Any]
diff --git a/products/warehouse_sources/sources/flagsmith/flagsmith.py b/products/warehouse_sources/sources/flagsmith/flagsmith.py
index 13802dc33e49..cf5dd025e936 100644
--- a/products/warehouse_sources/sources/flagsmith/flagsmith.py
+++ b/products/warehouse_sources/sources/flagsmith/flagsmith.py
@@ -19,6 +19,7 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -345,7 +346,8 @@ def _iter_pages(
budget: _PageBudget,
) -> Iterator[list[dict[str, Any]]]:
"""Walk a listing (paginated or plain-array) without touching resume state — used to
- enumerate fan-out parents, which must be re-listed from scratch on every run."""
+ enumerate fan-out parents, which must be re-listed from scratch on every run. Parents are
+ enumerated before any row is yielded, so each page is a safe point."""
url: str | None = start_url
while url:
if not budget.take():
@@ -355,6 +357,7 @@ def _iter_pages(
rows, url = _extract_rows(base, data, url)
if rows:
yield rows
+ reach_safe_point()
def _extend_capped(
@@ -616,6 +619,7 @@ def _paginate_resource(
resumable_source_manager.save_state(
FlagsmithResumeConfig(next_url=next_url or "", parent_key=parent.resume_key)
)
+ resumable_source_manager.safe_point()
url = next_url
diff --git a/products/warehouse_sources/sources/formbricks/formbricks.py b/products/warehouse_sources/sources/formbricks/formbricks.py
index 57230a016938..9606d23761ae 100644
--- a/products/warehouse_sources/sources/formbricks/formbricks.py
+++ b/products/warehouse_sources/sources/formbricks/formbricks.py
@@ -9,7 +9,7 @@ from urllib.parse import parse_qsl, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.formbricks.settings import FORMBRICKS_ENDPOINTS, FormbricksEndpointConfig
from products.warehouse_sources.sources.sdk import (
@@ -17,6 +17,7 @@ from products.warehouse_sources.sources.sdk import (
SourceResponse,
_is_host_safe,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -224,10 +225,16 @@ def _advance_skip(url: str) -> str:
return parsed._replace(query=urlencode(params)).geturl()
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Each page is fetched only after the rows before it are yielded, and nothing is buffered.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((FormbricksRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -317,17 +324,15 @@ def get_rows(
while True:
items = _fetch_page(session, url, logger)
+ # A short (or empty) page means we've reached the end of the collection.
+ next_url = _advance_skip(url) if len(items) >= PAGE_SIZE else None
+ if next_url is not None:
+ resumable_source_manager.save_state(FormbricksResumeConfig(next_url=next_url))
if items:
yield items
- # A short (or empty) page means we've reached the end of the collection.
- if len(items) < PAGE_SIZE:
+ if next_url is None:
break
-
- next_url = _advance_skip(url)
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted); merge dedupes any re-pulled rows on the primary key.
- resumable_source_manager.save_state(FormbricksResumeConfig(next_url=next_url))
url = next_url
diff --git a/products/warehouse_sources/sources/g2/g2.py b/products/warehouse_sources/sources/g2/g2.py
index 732f095f2340..096a6940dd61 100644
--- a/products/warehouse_sources/sources/g2/g2.py
+++ b/products/warehouse_sources/sources/g2/g2.py
@@ -138,6 +138,10 @@ def get_rows(
if next_url:
resumable_source_manager.save_state(G2ResumeConfig(next_url=next_url))
+ # The batcher spans pages, so only a drained batcher leaves no row behind the staged cursor.
+ if not batcher.should_yield(include_incomplete_chunk=True):
+ resumable_source_manager.safe_point()
+
if not next_url:
break
url = next_url
diff --git a/products/warehouse_sources/sources/gainsight_px/gainsight_px.py b/products/warehouse_sources/sources/gainsight_px/gainsight_px.py
index 13ff37742a12..277e904654dd 100644
--- a/products/warehouse_sources/sources/gainsight_px/gainsight_px.py
+++ b/products/warehouse_sources/sources/gainsight_px/gainsight_px.py
@@ -16,6 +16,7 @@ from products.warehouse_sources.sources.gainsight_px.settings import (
)
from products.warehouse_sources.sources.sdk import (
BasePaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
@@ -287,8 +288,13 @@ def gainsight_px_source(
}
if config.incremental_fields:
- return SourceResponse(
- items=lambda: _event_pages(
+ # The window loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the
+ # framework's per-page safe points, so a run of empty windows can hand off.
+ event_pages = Resource(
+ _event_pages,
+ name=endpoint,
+ hints={},
+ args=(
api_key,
region,
endpoint,
@@ -297,8 +303,8 @@ def gainsight_px_source(
resumable_source_manager,
db_incremental_field_last_value,
),
- **response_kwargs,
)
+ return SourceResponse(items=lambda: event_pages, **response_kwargs)
paginator: BasePaginator
if config.pagination == "scroll":
diff --git a/products/warehouse_sources/sources/gitguardian/gitguardian.py b/products/warehouse_sources/sources/gitguardian/gitguardian.py
index 59746f7a7280..7cb10a977029 100644
--- a/products/warehouse_sources/sources/gitguardian/gitguardian.py
+++ b/products/warehouse_sources/sources/gitguardian/gitguardian.py
@@ -239,7 +239,9 @@ def _get_fan_out_rows(
config: GitGuardianEndpointConfig,
parent: GitGuardianEndpointConfig,
logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[GitGuardianResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
+ # A fan-out keeps no resume state, so each safe point lets a hand-off restart from the first parent.
parent_url = _build_url(base_url, parent.path, _list_params(parent))
for page in _iter_pages(session, parent_url, headers, base_url, parent.name, logger):
@@ -257,6 +259,7 @@ def _get_fan_out_rows(
yield [{**row, config.parent_id_field: parent_id} for row in page.rows]
else:
yield page.rows
+ resumable_source_manager.safe_point()
except requests.HTTPError as e:
# The parent can be deleted between listing it and fetching its children.
if e.response is not None and e.response.status_code == 404:
@@ -265,6 +268,7 @@ def _get_fan_out_rows(
)
continue
raise
+ resumable_source_manager.safe_point()
def get_rows(
@@ -286,7 +290,7 @@ def get_rows(
if config.parent_endpoint:
parent = GITGUARDIAN_ENDPOINTS[config.parent_endpoint]
- yield from _get_fan_out_rows(session, headers, base_url, config, parent, logger)
+ yield from _get_fan_out_rows(session, headers, base_url, config, parent, logger, resumable_source_manager)
return
params = _list_params(config)
@@ -322,6 +326,7 @@ def get_rows(
# Checkpoint the CURRENT page's URL after yielding, so a crash re-fetches this page.
if config.resumable:
resumable_source_manager.save_state(GitGuardianResumeConfig(url=page.url))
+ resumable_source_manager.safe_point()
# The walk finished cleanly, so drop the checkpoint. Otherwise a retry that re-runs extract
# after a completed walk would resume from the final page and skip everything before it.
diff --git a/products/warehouse_sources/sources/hitpay/hitpay.py b/products/warehouse_sources/sources/hitpay/hitpay.py
index 5eeacbd260bb..86ec48ce0b18 100644
--- a/products/warehouse_sources/sources/hitpay/hitpay.py
+++ b/products/warehouse_sources/sources/hitpay/hitpay.py
@@ -123,6 +123,7 @@ def _recurring_billing_source(
api_key: str,
platform_api_key: Optional[str],
environment: Optional[str],
+ resumable_source_manager: ResumableSourceManager[HitpayResumeConfig],
) -> SourceResponse:
config = HITPAY_ENDPOINTS["RecurringBilling"]
client = RESTClient(
@@ -143,6 +144,9 @@ def _recurring_billing_source(
):
if page:
yield page
+ # Every row of this status is yielded. The run has no cursor, so a hand-off here
+ # restarts from the first status.
+ resumable_source_manager.safe_point()
return SourceResponse(
name="RecurringBilling",
@@ -168,7 +172,7 @@ def hitpay_source(
db_incremental_field_last_value: Optional[Any],
) -> SourceResponse:
if endpoint == "RecurringBilling":
- return _recurring_billing_source(api_key, platform_api_key, environment)
+ return _recurring_billing_source(api_key, platform_api_key, environment, resumable_source_manager)
config = HITPAY_ENDPOINTS[endpoint]
rest_config: RESTAPIConfig = {
diff --git a/products/warehouse_sources/sources/honeybadger/honeybadger.py b/products/warehouse_sources/sources/honeybadger/honeybadger.py
index b001822999e2..54d9af7b71af 100644
--- a/products/warehouse_sources/sources/honeybadger/honeybadger.py
+++ b/products/warehouse_sources/sources/honeybadger/honeybadger.py
@@ -146,19 +146,33 @@ def _parse_page(data: Any, data_selector: str = "results") -> tuple[list[Any], s
return data.get(data_selector) or [], (data.get("links") or {}).get("next")
-def _collect_items(session: requests.Session, first_url: str, logger: FilteringBoundLogger) -> list[dict]:
- """Walk every page of a list endpoint and return the items, following `links.next`."""
+def _collect_items(
+ session: requests.Session,
+ first_url: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[HoneybadgerResumeConfig],
+) -> list[dict]:
+ """Walk every page of a list endpoint and return the items, following `links.next`.
+
+ It collects fan-out parents, not rows, and the caller holds no rows, so each page is a safe point.
+ """
items: list[dict] = []
url: str | None = first_url
while url:
results, next_url = _parse_page(_fetch_page(session, url, logger))
items.extend(results)
+ resumable_source_manager.safe_point()
url = next_url if next_url != url else None
return items
-def _collect_ids(session: requests.Session, first_url: str, logger: FilteringBoundLogger) -> list[Any]:
- return [item["id"] for item in _collect_items(session, first_url, logger)]
+def _collect_ids(
+ session: requests.Session,
+ first_url: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[HoneybadgerResumeConfig],
+) -> list[Any]:
+ return [item["id"] for item in _collect_items(session, first_url, logger, resumable_source_manager)]
def _build_params(
@@ -244,13 +258,13 @@ def _get_project_rows(
resume.next_url if resume and resume.next_url else _build_url(f"{HONEYBADGER_BASE_URL}/projects", params)
)
for results, next_url in _iter_pages(session, first_url, logger):
- if not results:
- continue
- yield results
- # Save AFTER yielding (and only when more pages remain) so a crash re-yields the last
- # page rather than skipping it — merge dedupes on the primary key.
- if next_url:
- resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url))
+ if results:
+ yield results
+ # Save AFTER yielding (and only when more pages remain) so a crash re-yields the last
+ # page rather than skipping it — merge dedupes on the primary key.
+ if next_url:
+ resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
def _get_project_child_rows(
@@ -262,7 +276,9 @@ def _get_project_child_rows(
params: dict[str, Any],
) -> Iterator[list[dict]]:
"""Fan out over every project and page through the per-project child endpoint."""
- project_ids = _collect_ids(session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger)
+ project_ids = _collect_ids(
+ session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger, resumable_source_manager
+ )
# Resolve the saved project bookmark to the projects still to process. If the bookmarked
# project no longer exists, start over — merge dedupes re-pulled rows on the primary key.
@@ -279,15 +295,17 @@ def _get_project_child_rows(
resume_url = None # only the resumed-into project uses the saved URL
for results, next_url in _iter_pages(session, first_url, logger, config.data_selector):
- if not results:
- continue
- if config.time_series:
- yield _time_series_rows(project_id, results)
- else:
- # The traversal determines the parent, so API rows cannot change their project attribution.
- yield [{**item, "project_id": project_id} for item in results]
- if next_url:
- resumable_source_manager.save_state(HoneybadgerResumeConfig(next_url=next_url, project_id=project_id))
+ if results:
+ if config.time_series:
+ yield _time_series_rows(project_id, results)
+ else:
+ # The traversal determines the parent, so API rows cannot change their project attribution.
+ yield [{**item, "project_id": project_id} for item in results]
+ if next_url:
+ resumable_source_manager.save_state(
+ HoneybadgerResumeConfig(next_url=next_url, project_id=project_id)
+ )
+ resumable_source_manager.safe_point()
# Advance the bookmark so a crash between projects resumes at the next one.
if index + 1 < len(remaining):
@@ -311,7 +329,9 @@ def _get_nested_rows(
rows, which keeps the per-fault request count away from the 360 req/hour quota.
"""
parent_key = _FAN_OUT_PARENT_KEYS[fan_out_parent]
- project_ids = _collect_ids(session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger)
+ project_ids = _collect_ids(
+ session, _build_url(f"{HONEYBADGER_BASE_URL}/projects", {"limit": PAGE_LIMIT}), logger, resumable_source_manager
+ )
remaining_projects = project_ids
resume_parent_id: Any = None
@@ -326,6 +346,7 @@ def _get_nested_rows(
session,
_build_url(f"{HONEYBADGER_BASE_URL}/projects/{project_id}/{fan_out_parent}", parent_params),
logger,
+ resumable_source_manager,
)
parent_ids = [parent["id"] for parent in parents]
parent_columns = {
@@ -350,16 +371,16 @@ def _get_nested_rows(
resume_url = None
for results, next_url in _iter_pages(session, first_url, logger, config.data_selector):
- if not results:
- continue
- yield [
- {**item, "project_id": project_id, parent_key: parent_id, **parent_columns[parent_id]}
- for item in results
- ]
- if next_url:
- resumable_source_manager.save_state(
- HoneybadgerResumeConfig(next_url=next_url, project_id=project_id, **{parent_key: parent_id})
- )
+ if results:
+ yield [
+ {**item, "project_id": project_id, parent_key: parent_id, **parent_columns[parent_id]}
+ for item in results
+ ]
+ if next_url:
+ resumable_source_manager.save_state(
+ HoneybadgerResumeConfig(next_url=next_url, project_id=project_id, **{parent_key: parent_id})
+ )
+ resumable_source_manager.safe_point()
if parent_index + 1 < len(remaining_parents):
resumable_source_manager.save_state(
diff --git a/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py b/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py
index 311c889f2d18..cdf2cb153591 100644
--- a/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py
+++ b/products/warehouse_sources/sources/honeybadger/tests/test_honeybadger.py
@@ -327,13 +327,13 @@ class TestHoneybadger:
result = self._driver().run("faults", answer)
- # The first batch reaches the pipeline before the source saves its next-page state.
- # A failed next request must not commit that state and skip the first batch on retry.
+ # The first batch is written before the source saves its next-page state, so the safe point
+ # after the save commits that state, and a retry starts at page 2 without losing the batch.
assert isinstance(result.raised, requests.ConnectionError)
assert result.items == [[{"id": 10, "project_id": 1}]]
assert result.saved_states == [HoneybadgerResumeConfig(next_url=next_url, project_id=1)]
- assert result.committed_states == []
- assert result.final_state is None
+ assert result.committed_states == [HoneybadgerResumeConfig(next_url=next_url, project_id=1)]
+ assert result.final_state == HoneybadgerResumeConfig(next_url=next_url, project_id=1)
assert result.params("page")[0:2] == [None, None]
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/sources/hyperspell/hyperspell.py b/products/warehouse_sources/sources/hyperspell/hyperspell.py
index 2431a64adac8..4bfa02d48cd9 100644
--- a/products/warehouse_sources/sources/hyperspell/hyperspell.py
+++ b/products/warehouse_sources/sources/hyperspell/hyperspell.py
@@ -196,6 +196,7 @@ def _get_list_rows(
# Save state AFTER yielding so a crash re-yields the last page (deduped on the
# primary key) rather than skipping it.
resumable_source_manager.save_state(HyperspellResumeConfig(cursor=next_cursor, user_id=principal))
+ resumable_source_manager.safe_point()
cursor = next_cursor
@@ -399,6 +400,7 @@ def get_rows(
# Advance the bookmark to the next user so a crash between users resumes correctly.
if index + 1 < len(remaining):
resumable_source_manager.save_state(HyperspellResumeConfig(cursor=None, user_id=remaining[index + 1]))
+ resumable_source_manager.safe_point()
def hyperspell_source(
diff --git a/products/warehouse_sources/sources/instana/instana.py b/products/warehouse_sources/sources/instana/instana.py
index 6c3dc3473eac..7ec8fbe10bbf 100644
--- a/products/warehouse_sources/sources/instana/instana.py
+++ b/products/warehouse_sources/sources/instana/instana.py
@@ -294,6 +294,7 @@ def _get_event_rows(
# Save AFTER yielding, and only while more windows remain, so a crash re-fetches the last
# chunk rather than skipping it — merge dedupes on the primary key.
resumable_source_manager.save_state(InstanaResumeConfig(events_window_from=window_to))
+ resumable_source_manager.safe_point()
window_from = window_to
@@ -498,6 +499,7 @@ def _get_metric_rows(
window_from = window_to
if window_from < end:
resumable_source_manager.save_state(InstanaResumeConfig(metrics_window_from=window_from, next_page=1))
+ resumable_source_manager.safe_point()
def _get_fan_out_rows(
diff --git a/products/warehouse_sources/sources/kapa_ai/kapa_ai.py b/products/warehouse_sources/sources/kapa_ai/kapa_ai.py
index 406669a7990a..65050313f8d2 100644
--- a/products/warehouse_sources/sources/kapa_ai/kapa_ai.py
+++ b/products/warehouse_sources/sources/kapa_ai/kapa_ai.py
@@ -134,11 +134,17 @@ def kapa_ai_source(
) -> SourceResponse:
definition = schema_for_resource(ENDPOINTS, endpoint)
- def get_rows() -> Iterator[list[dict[str, Any]]]:
+ def get_rows() -> Resource:
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
resource = build_resource(api_key, project_id, endpoint, team_id, job_id, resumable_source_manager, resume)
- yield from resource
- resumable_source_manager.clear_state()
+
+ def walk() -> Iterator[list[dict[str, Any]]]:
+ yield from resource
+ resumable_source_manager.clear_state()
+
+ # The walk hands each page on unchanged and holds no rows. As a `Resource` it keeps the
+ # framework's per-page safe points.
+ return Resource(walk, name=endpoint, hints=resource._hints)
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/sources/lacework/lacework.py b/products/warehouse_sources/sources/lacework/lacework.py
index 7a0bf032629d..db7349a3cbdc 100644
--- a/products/warehouse_sources/sources/lacework/lacework.py
+++ b/products/warehouse_sources/sources/lacework/lacework.py
@@ -381,12 +381,7 @@ def get_rows(
logger.warning("Lacework: stopping pagination, next URL host does not match the configured account")
next_page_url = None
- if rows:
- yield rows
-
- # Save AFTER yielding so a crash re-yields the current page rather than skipping it.
- # When the window is done, checkpoint the NEXT window instead so a resume doesn't
- # re-walk this one.
+ # When the window is done, stage the NEXT window so a resume doesn't re-walk this one.
if next_page_url is not None:
resumable_source_manager.save_state(
LaceworkResumeConfig(
@@ -395,21 +390,26 @@ def get_rows(
next_page_url=next_page_url,
)
)
- method, url, body = "GET", next_page_url, None
+ elif window_end < now:
+ resumable_source_manager.save_state(
+ LaceworkResumeConfig(
+ window_start=_format_datetime(window_end),
+ window_end=_format_datetime(min(window_end + timedelta(days=config.window_days), now)),
+ next_page_url=None,
+ )
+ )
+
+ if rows:
+ yield rows
else:
+ resumable_source_manager.safe_point()
+
+ if next_page_url is None:
break
+ method, url, body = "GET", next_page_url, None
window_start = window_end
window_end = None
- if window_start < now:
- next_window_end = min(window_start + timedelta(days=config.window_days), now)
- resumable_source_manager.save_state(
- LaceworkResumeConfig(
- window_start=_format_datetime(window_start),
- window_end=_format_datetime(next_window_end),
- next_page_url=None,
- )
- )
def lacework_source(
diff --git a/products/warehouse_sources/sources/llama_cloud/llama_cloud.py b/products/warehouse_sources/sources/llama_cloud/llama_cloud.py
index f3968635c4ac..fa46d53dc9b8 100644
--- a/products/warehouse_sources/sources/llama_cloud/llama_cloud.py
+++ b/products/warehouse_sources/sources/llama_cloud/llama_cloud.py
@@ -292,6 +292,7 @@ def get_rows(
# Save AFTER yielding so a crash re-yields the last page rather than skipping it —
# merge dedupes the re-yielded rows on the primary key.
resumable_source_manager.save_state(LlamaCloudResumeConfig(next_page_token=next_page_token))
+ resumable_source_manager.safe_point()
params["page_token"] = next_page_token
diff --git a/products/warehouse_sources/sources/lovable/lovable.py b/products/warehouse_sources/sources/lovable/lovable.py
index 40969cb01ec4..06f3ee7c3df9 100644
--- a/products/warehouse_sources/sources/lovable/lovable.py
+++ b/products/warehouse_sources/sources/lovable/lovable.py
@@ -1,4 +1,4 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from datetime import UTC, datetime, time, timedelta
from typing import Any, Optional
@@ -133,19 +133,25 @@ def _time_series_rows(time_series: dict[str, Any]) -> list[dict[str, Any]]:
return [rows[timestamp] for timestamp in sorted(rows)]
-def _iter_rows(client: RESTClient, path: str, params: dict[str, Any]) -> Iterator[dict[str, Any]]:
+def _iter_rows(
+ client: RESTClient, path: str, params: dict[str, Any], safe_point: Callable[[], None] | None = None
+) -> Iterator[dict[str, Any]]:
for page in client.paginate(path=path, params=params, paginator=_paginator(), data_selector="data"):
yield from page
+ if safe_point is not None:
+ safe_point()
-def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointConfig) -> Iterator[_ParentRef]:
+def _iter_parents(
+ client: RESTClient, api_version: str, config: LovableEndpointConfig, safe_point: Callable[[], None]
+) -> Iterator[_ParentRef]:
if config.scope == "account":
yield _ParentRef()
return
endpoints = version_config(api_version).endpoints
workspaces = endpoints["Workspaces"]
- for workspace in _iter_rows(client, _path(workspaces, _ParentRef()), _params(workspaces, _ParentRef())):
+ for workspace in _iter_rows(client, _path(workspaces, _ParentRef()), _params(workspaces, _ParentRef()), safe_point):
workspace_id = workspace.get("id")
if not workspace_id:
continue
@@ -155,7 +161,7 @@ def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointC
projects = endpoints["Projects"]
workspace_ref = _ParentRef(workspace_id=workspace_id)
- for project in _iter_rows(client, _path(projects, workspace_ref), _params(projects, workspace_ref)):
+ for project in _iter_rows(client, _path(projects, workspace_ref), _params(projects, workspace_ref), safe_point):
project_id = project.get("id")
if not project_id or (config.published_projects_only and not project.get("is_published")):
continue
@@ -165,7 +171,7 @@ def _iter_parents(client: RESTClient, api_version: str, config: LovableEndpointC
continue
scans = endpoints["ProjectSecurityScans"]
- for scan in _iter_rows(client, _path(scans, project_ref), _params(scans, project_ref)):
+ for scan in _iter_rows(client, _path(scans, project_ref), _params(scans, project_ref), safe_point):
scan_id = scan.get("id")
if scan_id:
yield _ParentRef(workspace_id=workspace_id, project_id=project_id, scan_id=scan_id)
@@ -210,7 +216,8 @@ def _iter_endpoint(
resumable_source_manager: ResumableSourceManager[LovableResumeConfig],
extra_params: dict[str, Any],
) -> Iterator[list[dict[str, Any]]]:
- parents = list(_iter_parents(client, api_version, config))
+ # Listing parents hands on no rows, so the staged cursor still covers every row handed on.
+ parents = list(_iter_parents(client, api_version, config, resumable_source_manager.safe_point))
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
start_index, cursor = resume_position(parents, resume)
@@ -240,6 +247,8 @@ def _iter_endpoint(
rows = [row for series in page for row in _time_series_rows(series)] if config.time_series else page
if rows:
yield [_with_parent_ids(row, parent) for row in rows]
+ # The staged cursor still fetches this page, so resuming from it loses no rows.
+ resumable_source_manager.safe_point()
cursor = None
diff --git a/products/warehouse_sources/sources/luma/luma.py b/products/warehouse_sources/sources/luma/luma.py
index 607513b459a2..ae670de43122 100644
--- a/products/warehouse_sources/sources/luma/luma.py
+++ b/products/warehouse_sources/sources/luma/luma.py
@@ -10,8 +10,10 @@ from products.warehouse_sources.sources.luma.settings import EVENTS_PATH, LUMA_E
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
+ interruptible_wait,
make_tracked_session,
max_retry_after_seconds,
+ reach_safe_point,
retry_with_deadline,
)
@@ -64,12 +66,23 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return _EXPONENTIAL_WAIT(retry_state)
+def _before_retry_wait(_state: RetryCallState) -> None:
+ # Callers send each request with no rows held back, so the run can stop before any retry wait.
+ reach_safe_point()
+
+
+def _retry_sleep(seconds: float) -> None:
+ interruptible_wait(seconds, safe_point=reach_safe_point)
+
+
# Luma holds a rate-limited key for one minute, so the waits must add up to more than that.
# Eight attempts wait about 1+2+4+8+16+30+30 = 91 seconds in total.
@retry_with_deadline(
retry=retry_if_exception_type((LumaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(8),
wait=_retry_wait,
+ sleep=_retry_sleep,
+ before_sleep=_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -177,6 +190,7 @@ def _get_fan_out_rows(
)
for event_api_id in event_api_ids:
yield from _get_child_rows_for_event(session, config, event_api_id, logger)
+ resumable_source_manager.safe_point()
if not next_events_cursor or not event_entries:
break
@@ -185,6 +199,7 @@ def _get_fan_out_rows(
# Save AFTER yielding every child row of this events page, so a crash re-pulls at most one page
# of parents; merge dedupes the re-pulled rows on the primary key.
resumable_source_manager.save_state(LumaResumeConfig(pagination_cursor=events_cursor))
+ resumable_source_manager.safe_point()
def _get_top_level_rows(
@@ -215,6 +230,7 @@ def _get_top_level_rows(
# Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
# persisted); merge dedupes the re-pulled page on the primary key.
resumable_source_manager.save_state(LumaResumeConfig(pagination_cursor=cursor))
+ resumable_source_manager.safe_point()
def get_rows(
diff --git a/products/warehouse_sources/sources/mailersend/mailersend.py b/products/warehouse_sources/sources/mailersend/mailersend.py
index 9a71b40b1703..1db1943ba1db 100644
--- a/products/warehouse_sources/sources/mailersend/mailersend.py
+++ b/products/warehouse_sources/sources/mailersend/mailersend.py
@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (
ClientConfig,
EndpointResource,
JSONResponsePaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
@@ -345,18 +346,33 @@ def mailersend_source(
items: Callable[[], Any]
if config.fan_out_over_domains:
- items = partial(
- _activity_pages,
- api_token=api_token,
- endpoint=endpoint,
- config=config,
- team_id=team_id,
- job_id=job_id,
- resume_hook=save_checkpoint,
- initial_state=initial_state,
- should_use_incremental_field=should_use_incremental_field,
- db_incremental_field_last_value=db_incremental_field_last_value,
+ widest_window = _date_window(
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ (config.window_tiers_days or (MAILERSEND_DEFAULT_ACTIVITY_DAYS,))[0],
+ )
+ hints_resource = _activity_resource(
+ api_token, endpoint, config, team_id, job_id, widest_window, save_checkpoint, initial_state
+ )
+ # The tier loop hands each page on unchanged and holds no rows. As a `Resource` it keeps the
+ # framework's per-page safe points.
+ activity = Resource(
+ partial(
+ _activity_pages,
+ api_token=api_token,
+ endpoint=endpoint,
+ config=config,
+ team_id=team_id,
+ job_id=job_id,
+ resume_hook=save_checkpoint,
+ initial_state=initial_state,
+ should_use_incremental_field=should_use_incremental_field,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ ),
+ name=endpoint,
+ hints=hints_resource._hints,
)
+ items = lambda: activity
else:
endpoint_resource: EndpointResource
if config.analytics_by_date:
diff --git a/products/warehouse_sources/sources/mention/mention.py b/products/warehouse_sources/sources/mention/mention.py
index e3bc3afc1be2..27fc6ad0762a 100644
--- a/products/warehouse_sources/sources/mention/mention.py
+++ b/products/warehouse_sources/sources/mention/mention.py
@@ -6,7 +6,7 @@ from urllib.parse import urlencode, urljoin
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.mention.settings import MENTION_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (
SourceResponse,
frozen,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -104,10 +105,17 @@ def _more_url(payload: dict[str, Any]) -> Optional[str]:
return resolved
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows
+ # while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((MentionRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -195,19 +203,16 @@ def _alert_rows(
while url:
payload = _fetch_page(session, url, logger)
rows = [_unwrap_alert(row) for row in _rows_from_payload(payload, "alerts", url)]
- if rows:
- yield rows
# A missing ``_links.more`` marks the end of the collection. An empty page also terminates
# defensively so a lingering cursor can never produce an infinite loop.
- next_url = _more_url(payload)
- if not next_url or not rows:
- break
+ next_url = _more_url(payload) if rows else None
+ if next_url:
+ resumable_source_manager.save_state(MentionResumeConfig(next_url=next_url))
+ if rows:
+ yield rows
url = next_url
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted); merge dedupes the re-pulled page on the primary key.
- resumable_source_manager.save_state(MentionResumeConfig(next_url=next_url))
def _fan_out_alert_ids(
@@ -224,40 +229,6 @@ def _fan_out_alert_ids(
return _list_all_alert_ids(session, account_id, logger), None
-def _mention_rows(
- session: requests.Session,
- logger: FilteringBoundLogger,
- resume: MentionResumeConfig | None,
- resumable_source_manager: ResumableSourceManager[MentionResumeConfig],
-) -> Iterator[list[dict[str, Any]]]:
- account_id = str(_get_account(session, logger)["id"])
- alert_ids, next_url = _fan_out_alert_ids(session, account_id, logger, resume)
-
- while alert_ids:
- alert_id = alert_ids[0]
- url: Optional[str] = next_url or _list_url(f"/accounts/{account_id}/alerts/{alert_id}/mentions")
-
- while url:
- payload = _fetch_page(session, url, logger)
- rows = _rows_from_payload(payload, "mentions", url)
- for row in rows:
- # Normalized to a string so the merge key type is consistent across alerts.
- row["alert_id"] = alert_id
- if rows:
- yield rows
-
- more = _more_url(payload)
- if not more or not rows:
- break
-
- url = more
- resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids, next_url=url))
-
- alert_ids = alert_ids[1:]
- next_url = None
- resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids))
-
-
def _alert_tag_rows(
session: requests.Session,
logger: FilteringBoundLogger,
@@ -275,11 +246,13 @@ def _alert_tag_rows(
rows = _rows_from_payload(payload, "tags", url)
for row in rows:
row["alert_id"] = alert_id
- if rows:
- yield rows
alert_ids = alert_ids[1:]
resumable_source_manager.save_state(MentionResumeConfig(alert_ids=alert_ids))
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
def _paginated_alert_fan_out_rows(
@@ -423,7 +396,9 @@ def get_rows(
elif endpoint == "alert_tags":
yield from _alert_tag_rows(session, logger, resume, resumable_source_manager)
elif endpoint == "mentions":
- yield from _mention_rows(session, logger, resume, resumable_source_manager)
+ yield from _paginated_alert_fan_out_rows(
+ session, logger, resume, resumable_source_manager, path="mentions", key="mentions"
+ )
elif endpoint == "alert_authors":
yield from _paginated_alert_fan_out_rows(
session, logger, resume, resumable_source_manager, path="authors", key="authors"
diff --git a/products/warehouse_sources/sources/meteostat/meteostat.py b/products/warehouse_sources/sources/meteostat/meteostat.py
index b05e44ab3a98..95c3d376daee 100644
--- a/products/warehouse_sources/sources/meteostat/meteostat.py
+++ b/products/warehouse_sources/sources/meteostat/meteostat.py
@@ -278,6 +278,10 @@ def _get_rows(
window_end = min(cursor + timedelta(days=endpoint.window_days - 1), end_boundary)
data = _fetch_window(session, headers, endpoint, target, cursor, window_end, units)
+ cursor = window_end + timedelta(days=1)
+ resumable_source_manager.save_state(
+ MeteostatResumeConfig(station_index=index, next_start=cursor.isoformat())
+ )
if data:
rows = []
for row in data:
@@ -286,13 +290,8 @@ def _get_rows(
row[endpoint.date_field] = _parse_timestamp(row.get(endpoint.date_field))
rows.append(row)
yield rows
-
- cursor = window_end + timedelta(days=1)
- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window
- # instead of skipping it.
- resumable_source_manager.save_state(
- MeteostatResumeConfig(station_index=index, next_start=cursor.isoformat())
- )
+ else:
+ resumable_source_manager.safe_point()
def _get_snapshot_rows(
diff --git a/products/warehouse_sources/sources/moengage/moengage.py b/products/warehouse_sources/sources/moengage/moengage.py
index db920c613ee5..f5370f827894 100644
--- a/products/warehouse_sources/sources/moengage/moengage.py
+++ b/products/warehouse_sources/sources/moengage/moengage.py
@@ -325,6 +325,7 @@ def _iter_report_days(
days: list[date],
save_checkpoint: Callable[[date], None],
clear_checkpoint: Callable[[], None],
+ safe_point: Callable[[], None],
) -> Iterator[list[dict[str, Any]]]:
"""Yield every day's pages in ascending day order, checkpointing once a day is fully yielded.
@@ -335,6 +336,8 @@ def _iter_report_days(
yield from build_resource(day)
if index + 1 < len(days):
save_checkpoint(days[index + 1])
+ # Most days hold no stats, so a backfill can make many requests without a yield.
+ safe_point()
# The walk finished: leaving the last checkpoint would make a later attempt resume mid-stream.
clear_checkpoint()
@@ -565,7 +568,12 @@ def moengage_source(
resumable_source_manager.save_state(MoEngageResumeConfig(report_day_state={"start": next_day.isoformat()}))
items = partial(
- _iter_report_days, build_day_resource, days, save_day_checkpoint, resumable_source_manager.clear_state
+ _iter_report_days,
+ build_day_resource,
+ days,
+ save_day_checkpoint,
+ resumable_source_manager.clear_state,
+ resumable_source_manager.safe_point,
)
return SourceResponse(
diff --git a/products/warehouse_sources/sources/monte_carlo/monte_carlo.py b/products/warehouse_sources/sources/monte_carlo/monte_carlo.py
index 695c81916cd0..50f44fbd703f 100644
--- a/products/warehouse_sources/sources/monte_carlo/monte_carlo.py
+++ b/products/warehouse_sources/sources/monte_carlo/monte_carlo.py
@@ -336,6 +336,7 @@ def _get_alert_rows(
# Bookmark the next window so a crash between windows doesn't restart the backfill.
if index + 1 < len(windows):
resumable_source_manager.save_state(MonteCarloResumeConfig(window_after=time_range["before"]))
+ resumable_source_manager.safe_point()
def _get_dbt_test_result_rows(
diff --git a/products/warehouse_sources/sources/new_relic/new_relic.py b/products/warehouse_sources/sources/new_relic/new_relic.py
index e0a89e625f7d..f8cadb9d52a1 100644
--- a/products/warehouse_sources/sources/new_relic/new_relic.py
+++ b/products/warehouse_sources/sources/new_relic/new_relic.py
@@ -373,6 +373,7 @@ def _get_event_rows(
# rows a re-yield can append.
if window_end_ms < until_ms:
resumable_source_manager.save_state(NewRelicResumeConfig(window_start_ms=window_end_ms))
+ resumable_source_manager.safe_point()
window_start_ms = window_end_ms
diff --git a/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py b/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py
index 86b90b6293ca..c9bca333a743 100644
--- a/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py
+++ b/products/warehouse_sources/sources/new_relic/tests/test_new_relic.py
@@ -47,6 +47,9 @@ class FakeResumableSourceManager:
self.saved.append(state)
self.state = state
+ def safe_point(self) -> None:
+ pass
+
def _nrql_data(rows: list[dict[str, Any]]) -> dict[str, Any]:
return {"actor": {"account": {"nrql": {"results": rows}}}}
diff --git a/products/warehouse_sources/sources/notion/notion.py b/products/warehouse_sources/sources/notion/notion.py
index 32746dba0305..d6ff38e7b4ad 100644
--- a/products/warehouse_sources/sources/notion/notion.py
+++ b/products/warehouse_sources/sources/notion/notion.py
@@ -365,6 +365,10 @@ def _search_stream(
if not has_more or not next_cursor:
break
+ if not batcher.should_yield(include_incomplete_chunk=True):
+ # Nothing is buffered, so every row before `next_cursor` is already yielded.
+ resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))
+ resumable_source_manager.safe_point()
cursor = next_cursor
if batcher.should_yield(include_incomplete_chunk=True):
@@ -413,6 +417,9 @@ def _cursor_list_stream(
if batcher.should_yield():
resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))
yield batcher.get_table()
+ elif not batcher.should_yield(include_incomplete_chunk=True):
+ resumable_source_manager.save_state(NotionResumeConfig(next_cursor=next_cursor))
+ resumable_source_manager.safe_point()
cursor = next_cursor
if batcher.should_yield(include_incomplete_chunk=True):
@@ -458,7 +465,10 @@ def _permission_groups_stream(
def _iter_page_ids(
- session: requests.Session, logger: FilteringBoundLogger, throttle: Optional["_RateLimiter"] = None
+ session: requests.Session,
+ logger: FilteringBoundLogger,
+ throttle: Optional["_RateLimiter"] = None,
+ resumable_source_manager: ResumableSourceManager[NotionResumeConfig] | None = None,
) -> Iterator[str]:
cursor: str | None = None
while True:
@@ -482,6 +492,9 @@ def _iter_page_ids(
if not data.get("has_more") or not data.get("next_cursor"):
break
cursor = data["next_cursor"]
+ if resumable_source_manager is not None:
+ # The enumeration buffers page ids, not rows, and stages no state, so a resume restarts it.
+ resumable_source_manager.safe_point()
def _iter_block_children(
@@ -564,7 +577,7 @@ def _page_fan_out(
else:
# Materialize the full set of page IDs up front so the fan-out position can be persisted as a
# shrinking queue (a crash during this initial enumeration restarts it, like other sources).
- page_ids = list(_iter_page_ids(session, logger, throttle))
+ page_ids = list(_iter_page_ids(session, logger, throttle, resumable_source_manager))
last_flush_at = time.monotonic()
last_empty_stage_at = last_flush_at
diff --git a/products/warehouse_sources/sources/npm_registry/npm_registry.py b/products/warehouse_sources/sources/npm_registry/npm_registry.py
index d489f7972b0f..a03f1cb197e4 100644
--- a/products/warehouse_sources/sources/npm_registry/npm_registry.py
+++ b/products/warehouse_sources/sources/npm_registry/npm_registry.py
@@ -187,16 +187,23 @@ def _iter_downloads(
for entry in document.get("downloads") or []
if isinstance(entry, dict) and entry.get("day")
]
+ last_window = window_end >= today
+ if last_window:
+ manager.save_state(NpmRegistryResumeConfig(package_index=package_index + 1, window_start=None))
+ else:
+ next_start = window_end + timedelta(days=1)
+ manager.save_state(
+ NpmRegistryResumeConfig(package_index=package_index, window_start=next_start.isoformat())
+ )
+
if rows:
yield rows
+ else:
+ manager.safe_point()
- if window_end >= today:
+ if last_window:
break
-
window_start = window_end + timedelta(days=1)
- # Save AFTER yielding the window so a crash re-fetches it rather than skipping it; merge
- # dedupes the re-pulled days on the [package, day] primary key.
- manager.save_state(NpmRegistryResumeConfig(package_index=package_index, window_start=window_start.isoformat()))
def _iter_version_downloads(
@@ -303,6 +310,7 @@ def get_rows(
# Advance past this package before moving on, so a crash between packages resumes at the
# next one rather than re-walking one we already finished.
resumable_source_manager.save_state(NpmRegistryResumeConfig(package_index=index + 1, window_start=None))
+ resumable_source_manager.safe_point()
def validate_packages(packages_raw: str | None) -> tuple[bool, str | None]:
diff --git a/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py b/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py
index 1ce8781711a6..b4afa7b30784 100644
--- a/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py
+++ b/products/warehouse_sources/sources/npm_registry/tests/test_npm_registry.py
@@ -96,16 +96,16 @@ class TestGetRowsDownloads:
assert len(result.requests) == 1
@time_machine.travel("2024-02-15", tick=False)
- def test_saves_state_after_each_window_and_advances_package_on_completion(self) -> None:
+ def test_stages_state_before_each_window_and_advances_package_on_completion(self) -> None:
result = _driver().run(
"Downloads", always(ScriptedResponse(json={"downloads": [{"day": "2015-01-10", "downloads": 1}]}))
)
- # Final call always advances to the next package index with no in-progress window.
+ # The last window stages the next package, and the package loop stages it again.
assert result.raised is None
assert len(result.requests) > 1
- assert len(result.saved_states) == len(result.requests)
- assert result.saved_states[-1] == NpmRegistryResumeConfig(package_index=1)
+ assert len(result.saved_states) == len(result.requests) + 1
+ assert result.saved_states[-2:] == [NpmRegistryResumeConfig(package_index=1)] * 2
assert result.committed_states == result.saved_states[:-1]
diff --git a/products/warehouse_sources/sources/ortto/ortto.py b/products/warehouse_sources/sources/ortto/ortto.py
index 2cd82f70f268..da73315dbccb 100644
--- a/products/warehouse_sources/sources/ortto/ortto.py
+++ b/products/warehouse_sources/sources/ortto/ortto.py
@@ -140,8 +140,13 @@ def get_rows(
if config.pagination == "cursor":
if not isinstance(data, dict) or not data.get("has_more"):
break
- offset = data.get("next_offset", offset + len(items))
- cursor_id = data.get("cursor_id") or cursor_id
+ next_offset = data.get("next_offset", offset + len(items))
+ next_cursor_id = data.get("cursor_id") or cursor_id
+ if next_offset == offset and next_cursor_id == cursor_id:
+ # The next request would repeat this one and get the same page forever.
+ logger.warning(f"Ortto: {endpoint} reported has_more without advancing past offset {offset}")
+ break
+ offset, cursor_id = next_offset, next_cursor_id
else:
if len(items) < config.page_size:
break
@@ -150,6 +155,7 @@ def get_rows(
# Save state AFTER yielding so a crash re-yields the in-flight page
# (merge dedupes on primary key) rather than skipping it.
resumable_source_manager.save_state(OrttoResumeConfig(next_offset=offset, cursor_id=cursor_id))
+ resumable_source_manager.safe_point()
def ortto_source(
diff --git a/products/warehouse_sources/sources/ortto/tests/test_ortto.py b/products/warehouse_sources/sources/ortto/tests/test_ortto.py
index c7ec393156e2..7206637392e4 100644
--- a/products/warehouse_sources/sources/ortto/tests/test_ortto.py
+++ b/products/warehouse_sources/sources/ortto/tests/test_ortto.py
@@ -48,6 +48,22 @@ class TestGetRowsCursorPagination:
# State saved once, after the first page yielded.
assert result.saved_states == [OrttoResumeConfig(next_offset=500, cursor_id="cur-1")]
+ @pytest.mark.parametrize(
+ "stuck_page",
+ [
+ {"contacts": [], "has_more": True},
+ _contacts_page([], has_more=True, next_offset=0, cursor_id=None),
+ ],
+ ids=["no_next_offset", "next_offset_unchanged"],
+ )
+ def test_stops_when_has_more_does_not_advance(self, stuck_page: dict[str, Any]) -> None:
+ result = _driver().run(
+ "people", [ScriptedResponse(json={"fields": []}), *[ScriptedResponse(json=stuck_page)] * 3]
+ )
+
+ assert result.raised is None
+ assert len(result.requests) == 2
+
def test_resumes_from_saved_offset_and_cursor(self) -> None:
result = _driver().run(
"people",
diff --git a/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py b/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
index 779e423f3687..36431395deac 100644
--- a/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
+++ b/products/warehouse_sources/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
@@ -4,7 +4,7 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.pabbly_subscriptions_billing.settings import (
PABBLY_ENDPOINTS,
@@ -14,6 +14,7 @@ from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -59,10 +60,17 @@ def _is_no_data_message(message: str) -> bool:
return lowered.startswith("no ") and "found" in lowered
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows
+ # while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((PabblyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -127,17 +135,16 @@ def _iter_top_level(
while True:
rows = _fetch_page(session, config.path, page, PAGE_SIZE, logger, config.ignore_no_data_errors)
+ # A short page means we've reached the end of the list.
+ has_next_page = len(rows) >= PAGE_SIZE
+ if has_next_page:
+ resumable_source_manager.save_state(PabblyResumeConfig(page=page + 1))
if rows:
yield rows
- # A short page means we've reached the end of the list.
- if len(rows) < PAGE_SIZE:
+ if not has_next_page:
break
-
page += 1
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted); merge dedupes any re-pulled rows on the primary key.
- resumable_source_manager.save_state(PabblyResumeConfig(page=page))
def _iter_children_of_parent(
@@ -181,6 +188,9 @@ def _iter_fan_out(
for parent in parents:
parent_id = str(parent["id"])
yield from _iter_children_of_parent(session, config, parent_id, logger)
+ # The staged cursor is this parent page, and every child row read so far is yielded.
+ # A parent with no children yields nothing, so the worker needs this point to stop.
+ resumable_source_manager.safe_point()
if len(parents) < PAGE_SIZE:
break
diff --git a/products/warehouse_sources/sources/pipeliner/pipeliner.py b/products/warehouse_sources/sources/pipeliner/pipeliner.py
index b23fab863c44..e83a2b851a7f 100644
--- a/products/warehouse_sources/sources/pipeliner/pipeliner.py
+++ b/products/warehouse_sources/sources/pipeliner/pipeliner.py
@@ -1,6 +1,5 @@
import re
import dataclasses
-from collections.abc import Iterator
from datetime import UTC, date, datetime
from typing import Any, Optional
@@ -10,6 +9,7 @@ from requests import Request, Response
from products.warehouse_sources.sources.pipeliner.settings import PIPELINER_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
BasePaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SourceResponse,
@@ -158,7 +158,7 @@ def pipeliner_source(
) -> SourceResponse:
config = PIPELINER_ENDPOINTS[endpoint]
- def items() -> Iterator[list[dict[str, Any]]]:
+ def items() -> Resource:
# Re-check at run time (not just at source-create) in case the service URL was edited or now
# resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.
host_ok, host_err = _is_host_safe(normalize_service_url(service_url), team_id)
@@ -190,8 +190,7 @@ def pipeliner_source(
params[f"filter-op[{cursor_field}]"] = "gte"
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist AFTER a page is yielded and only while a next page remains, so a crash
- # re-fetches from the next page (already-yielded pages are persisted); merge dedupes.
+ # Persist only while a next page remains. The framework stages it with the page it covers.
if state and state.get("cursor") is not None:
resumable_source_manager.save_state(
PipelinerResumeConfig(cursor=state["cursor"], filter_value=filter_value)
@@ -224,7 +223,7 @@ def pipeliner_source(
],
}
- resource = rest_api_resource(
+ return rest_api_resource(
rest_config,
team_id,
job_id,
@@ -232,7 +231,6 @@ def pipeliner_source(
resume_hook=save_checkpoint,
initial_paginator_state=initial_paginator_state,
)
- yield from resource
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py b/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py
index ee8915e7329f..2f44c83a5052 100644
--- a/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py
+++ b/products/warehouse_sources/sources/power_bi_admin/power_bi_admin.py
@@ -181,40 +181,42 @@ def _activity_event_pages(
body = client.get("/activityevents", params=params)
rows = body.get("activityEventEntities") or []
- if rows:
- yield [_normalize_activity_event(row) for row in rows]
-
continuation_token = body.get("continuationToken")
pages += 1
# A page can come back empty and still carry a token, so termination is
# "token gone", never "no rows".
- if not continuation_token:
- break
-
- if continuation_token in seen_tokens:
+ if continuation_token and continuation_token in seen_tokens:
logger.warning(f"Power BI admin: repeated continuation token for {day.isoformat()}, stopping the day")
continuation_token = None
- break
- seen_tokens.add(continuation_token)
-
- if pages >= MAX_CONTINUATION_PAGES:
- logger.warning(
- f"Power BI admin: hit the {MAX_CONTINUATION_PAGES}-page cap for {day.isoformat()}, "
- "the rest of the day is skipped"
+ elif continuation_token:
+ seen_tokens.add(continuation_token)
+ if pages >= MAX_CONTINUATION_PAGES:
+ logger.warning(
+ f"Power BI admin: hit the {MAX_CONTINUATION_PAGES}-page cap for {day.isoformat()}, "
+ "the rest of the day is skipped"
+ )
+ continuation_token = None
+
+ # Stage mid-day so a heartbeat timeout resumes inside the day rather than refetching it.
+ if continuation_token:
+ resumable_source_manager.save_state(
+ PowerBiAdminResumeConfig(next_day=day.isoformat(), continuation_token=continuation_token)
+ )
+ else:
+ resumable_source_manager.save_state(
+ PowerBiAdminResumeConfig(next_day=(day + timedelta(days=1)).isoformat())
)
- continuation_token = None
- break
- # Save mid-day so a heartbeat timeout resumes inside the day rather than
- # refetching it. Saved after yielding, so a crash re-yields the last page and
- # merge dedupes on `Id`.
- resumable_source_manager.save_state(
- PowerBiAdminResumeConfig(next_day=day.isoformat(), continuation_token=continuation_token)
- )
+ if rows:
+ yield [_normalize_activity_event(row) for row in rows]
+ else:
+ resumable_source_manager.safe_point()
+
+ if not continuation_token:
+ break
day += timedelta(days=1)
- resumable_source_manager.save_state(PowerBiAdminResumeConfig(next_day=day.isoformat()))
def _odata_pages(
diff --git a/products/warehouse_sources/sources/pylon/pylon.py b/products/warehouse_sources/sources/pylon/pylon.py
index a95cae7554b8..761575a9b497 100644
--- a/products/warehouse_sources/sources/pylon/pylon.py
+++ b/products/warehouse_sources/sources/pylon/pylon.py
@@ -188,12 +188,12 @@ def _get_simple_rows(
start_cursor = resume.cursor if resume else None
for items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):
- if items:
- yield items
- # Save AFTER yielding so a crash re-yields the last page rather than skipping it; merge dedupes
- # on the primary key.
if next_cursor:
manager.save_state(PylonResumeConfig(cursor=next_cursor))
+ if items:
+ yield items
+ else:
+ manager.safe_point()
def _get_fan_out_rows(
@@ -220,19 +220,21 @@ def _get_fan_out_rows(
start_cursor = resume_cursor if index == 0 else None
resume_cursor = None
+ next_object_type = remaining[index + 1] if index + 1 < len(remaining) else None
for items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):
# Stamp the queried object_type so a single table covers every type and the composite
# [object_type, id] primary key stays unique even if two types reuse an id.
stamped = [{**item, "object_type": item.get("object_type") or object_type} for item in items]
- if stamped:
- yield stamped
if next_cursor:
manager.save_state(PylonResumeConfig(cursor=next_cursor, object_type=object_type))
-
- # Advance the bookmark to the next object_type so a crash between types resumes correctly.
- if index + 1 < len(remaining):
- manager.save_state(PylonResumeConfig(cursor=None, object_type=remaining[index + 1]))
+ elif next_object_type is not None:
+ # The last page of this type: advance the bookmark so a crash between types resumes correctly.
+ manager.save_state(PylonResumeConfig(cursor=None, object_type=next_object_type))
+ if stamped:
+ yield stamped
+ else:
+ manager.safe_point()
def _get_windowed_rows(
@@ -286,14 +288,17 @@ def _get_windowed_rows(
resume_cursor = None
for items, next_cursor in _iter_pages(session, headers, base_url, config.path, params, logger, start_cursor):
- if items:
- yield items
if next_cursor:
manager.save_state(PylonResumeConfig(cursor=next_cursor, window_start=params["start_time"]))
+ elif window_end < now:
+ # The last page of this window: point the bookmark at the next window.
+ manager.save_state(PylonResumeConfig(cursor=None, window_start=_format_rfc3339(window_end)))
+ if items:
+ yield items
+ else:
+ manager.safe_point()
window_start = window_end
- if window_start < now:
- manager.save_state(PylonResumeConfig(cursor=None, window_start=_format_rfc3339(window_start)))
def _iter_parent_pages(
diff --git a/products/warehouse_sources/sources/replicate/replicate.py b/products/warehouse_sources/sources/replicate/replicate.py
index 01e33bf46a2a..b20be88be2d7 100644
--- a/products/warehouse_sources/sources/replicate/replicate.py
+++ b/products/warehouse_sources/sources/replicate/replicate.py
@@ -391,6 +391,7 @@ def get_rows(
# dedupes on the primary key). Tag it with the watermark so a later run with an advanced
# watermark rebuilds from the first page instead of trusting this cursor.
resumable_source_manager.save_state(ReplicateResumeConfig(next_url=next_url, created_after=current_watermark))
+ resumable_source_manager.safe_point()
url = next_url
diff --git a/products/warehouse_sources/sources/runpod/runpod.py b/products/warehouse_sources/sources/runpod/runpod.py
index 89dd5420e337..bad943b91abc 100644
--- a/products/warehouse_sources/sources/runpod/runpod.py
+++ b/products/warehouse_sources/sources/runpod/runpod.py
@@ -212,14 +212,15 @@ def _iter_billing_rows(
# Response ordering is undocumented; each window arrives whole, so sort client-side to keep
# the ascending guarantee sort_mode="asc" promises the pipeline.
rows = sorted((_normalize_billing_record(record) for record in records), key=lambda r: r.get("time") or "")
+ if window_end is not None:
+ resumable_source_manager.save_state(RunPodResumeConfig(window_start=_format_rfc3339(window_end)))
if rows:
yield rows
+ else:
+ resumable_source_manager.safe_point()
if window_end is None:
break
- # Save AFTER yielding so a crash re-yields the last window rather than skipping it — merge
- # dedupes the re-pulled rows on the surrogate id.
- resumable_source_manager.save_state(RunPodResumeConfig(window_start=_format_rfc3339(window_end)))
window_start = window_end
diff --git a/products/warehouse_sources/sources/sage_hr/sage_hr.py b/products/warehouse_sources/sources/sage_hr/sage_hr.py
index d3719339d3c8..eb9674b90198 100644
--- a/products/warehouse_sources/sources/sage_hr/sage_hr.py
+++ b/products/warehouse_sources/sources/sage_hr/sage_hr.py
@@ -7,7 +7,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.sage_hr.settings import SAGE_HR_ENDPOINTS, SageHREndpointConfig
from products.warehouse_sources.sources.sdk import (
@@ -15,6 +15,7 @@ from products.warehouse_sources.sources.sdk import (
SourceResponse,
SyncWindow,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
@@ -93,10 +94,16 @@ def _extract_next_page(payload: dict[str, Any]) -> int | None:
return next_page if isinstance(next_page, int) else None
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Each page is fetched only after the rows before it are yielded, and nothing is buffered.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((SageHRRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -158,6 +165,7 @@ def _get_windowed_rows(
page = 1
if resume is not None and resume.next_page is not None and resume.window_from == window_start.isoformat():
page = resume.next_page
+ next_window_start = window_end + timedelta(days=1)
while True:
params = {"page": page, "from": window_start.isoformat(), "to": window_end.isoformat()}
@@ -172,28 +180,28 @@ def _get_windowed_rows(
continue
seen_ids.add(row_id)
fresh.append(row)
- if fresh:
- yield fresh
# `next_page` is null on the last page; an empty or non-advancing page also terminates
# defensively so a bad meta block can never produce an infinite loop.
- if not next_page or not items or next_page <= page:
- break
+ has_next_page = bool(next_page and items and next_page > page)
+ if has_next_page:
+ resumable_source_manager.save_state(
+ SageHRResumeConfig(next_page=next_page, window_from=window_start.isoformat())
+ )
+ elif next_window_start <= window.end:
+ # Window exhausted: a resume can skip straight to the next window.
+ resumable_source_manager.save_state(
+ SageHRResumeConfig(next_page=1, window_from=next_window_start.isoformat())
+ )
+ if fresh:
+ yield fresh
+ else:
+ # Most windows are empty, so the walk needs a stop point that does not depend on rows.
+ resumable_source_manager.safe_point()
+ if not has_next_page or next_page is None:
+ break
page = next_page
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages
- # are persisted).
- resumable_source_manager.save_state(
- SageHRResumeConfig(next_page=page, window_from=window_start.isoformat())
- )
-
- # Window exhausted — everything up to it has been yielded, so a resume can skip straight to
- # the next window.
- next_window_start = window_end + timedelta(days=1)
- if next_window_start <= window.end:
- resumable_source_manager.save_state(
- SageHRResumeConfig(next_page=1, window_from=next_window_start.isoformat())
- )
def get_rows(
@@ -224,18 +232,17 @@ def get_rows(
page = resume.next_page if (resume is not None and resume.next_page is not None) else 1
while True:
items, next_page = _fetch_page(session, _build_url(base_url, config.path, {"page": page}), logger)
+ # `next_page` is null on the last page; an empty or non-advancing page also terminates
+ # defensively so a bad meta block can never produce an infinite loop.
+ has_next_page = bool(next_page and items and next_page > page)
+ if has_next_page:
+ resumable_source_manager.save_state(SageHRResumeConfig(next_page=next_page))
if items:
yield items
- # `next_page` is null on the last page; an empty or non-advancing page also terminates
- # defensively so a bad meta block can never produce an infinite loop.
- if not next_page or not items or next_page <= page:
+ if not has_next_page or next_page is None:
break
-
page = next_page
- # Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages are
- # persisted).
- resumable_source_manager.save_state(SageHRResumeConfig(next_page=page))
def sage_hr_source(
diff --git a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
index 780e06b94c55..17013c74c8b0 100644
--- a/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
+++ b/products/warehouse_sources/sources/sage_hr/tests/test_sage_hr.py
@@ -40,6 +40,9 @@ class _FakeResumableManager:
def save_state(self, data: SageHRResumeConfig) -> None:
self.saved.append(data)
+ def safe_point(self) -> None:
+ pass
+
class TestNormalizeSubdomain:
@parameterized.expand(
diff --git a/products/warehouse_sources/sources/sevdesk/sevdesk.py b/products/warehouse_sources/sources/sevdesk/sevdesk.py
index 0bbf542800b1..7d5e897cd94a 100644
--- a/products/warehouse_sources/sources/sevdesk/sevdesk.py
+++ b/products/warehouse_sources/sources/sevdesk/sevdesk.py
@@ -1,10 +1,10 @@
-from collections.abc import Iterator
from typing import Any, cast
from products.warehouse_sources.sources.sdk import (
APIKeyAuth,
Endpoint,
EndpointResource,
+ Resource,
RESTAPIConfig,
RESTClient,
ResumableSourceManager,
@@ -84,11 +84,11 @@ def sevdesk_source(
SevdeskResumeConfig(offset=state["offset"]) if state is not None else SevdeskResumeConfig(completed=True)
)
- def get_rows() -> Iterator[list[dict[str, Any]]]:
+ def get_rows() -> Resource | list[list[dict[str, Any]]]:
state = resumable_source_manager.load_state()
if state is not None and state.completed:
- return
- resource = rest_api_resource(
+ return []
+ return rest_api_resource(
config,
team_id=team_id,
job_id=job_id,
@@ -96,7 +96,6 @@ def sevdesk_source(
resume_hook=save_page_state,
initial_paginator_state={"offset": state.offset} if state is not None else None,
)
- yield from resource
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/sources/shippo/shippo.py b/products/warehouse_sources/sources/shippo/shippo.py
index 3452a36415f9..833612fb7ce9 100644
--- a/products/warehouse_sources/sources/shippo/shippo.py
+++ b/products/warehouse_sources/sources/shippo/shippo.py
@@ -6,12 +6,13 @@ from urllib.parse import parse_qs, urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
from products.warehouse_sources.sources.shippo.settings import SHIPPO_ENDPOINTS
@@ -94,10 +95,17 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{SHIPPO_BASE_URL}{path}/?{urlencode(clean_params)}"
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows
+ # while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((ShippoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=5, max=90),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(session: requests.Session, url: str, logger: FilteringBoundLogger) -> dict[str, Any]:
@@ -177,6 +185,7 @@ def _paginate(
# Save AFTER yielding so a crash re-fetches from the next page (already-yielded pages
# are persisted); merge dedupes the re-pulled page on the primary key.
resumable_source_manager.save_state(ShippoResumeConfig(next_url=url, window_start=window_start))
+ resumable_source_manager.safe_point()
def get_rows(
diff --git a/products/warehouse_sources/sources/smaily/smaily.py b/products/warehouse_sources/sources/smaily/smaily.py
index f39022cb8568..f7cc71c0663c 100644
--- a/products/warehouse_sources/sources/smaily/smaily.py
+++ b/products/warehouse_sources/sources/smaily/smaily.py
@@ -5,12 +5,13 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
from products.warehouse_sources.sources.smaily.settings import (
@@ -21,9 +22,6 @@ from products.warehouse_sources.sources.smaily.settings import (
)
REQUEST_TIMEOUT_SECONDS = 60
-# Fan-out endpoints issue one request per parent; yield + checkpoint every N parents so a resumed
-# job re-fetches at most one chunk.
-STATS_CHUNK_SIZE = 100
# Cheap endpoint used to confirm credentials are genuine: `list.php` takes no params and every API
# user can read it, so one probe validates access to the account.
DEFAULT_PROBE_PATH = "list.php"
@@ -69,10 +67,17 @@ def _make_session(username: str, password: str) -> requests.Session:
return session
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages: the rows before it are yielded, and no caller holds rows
+ # while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((SmailyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch(
@@ -129,19 +134,18 @@ def _list_rows(
resume = manager.load_state() if manager.can_resume() else None
page = resume.page if resume else 0
+ assert config.page_size is not None
while True:
rows = _fetch(session, url, _paged_params(config, page), logger, expect_list=True)
+ has_next_page = len(rows) >= config.page_size
+ if has_next_page:
+ manager.save_state(SmailyResumeConfig(page=page + 1))
if rows:
yield rows
- assert config.page_size is not None
- if len(rows) < config.page_size:
+ if not has_next_page:
break
-
page += 1
- # Save AFTER yielding so a crash re-fetches the last page (already-yielded pages are
- # persisted); merge dedupes the re-pulled page on the primary key.
- manager.save_state(SmailyResumeConfig(page=page))
def _fetch_all_pages(
@@ -189,6 +193,7 @@ def _segment_subscriber_rows(
pending = sorted(str(segment["id"]) for segment in segments)
page = 0
+ assert config.page_size is not None
while pending:
segment_id = pending[0]
@@ -196,19 +201,22 @@ def _segment_subscriber_rows(
params = {"list": segment_id, **_paged_params(config, page)}
rows = _fetch(session, url, params, logger, expect_list=True)
normalized = [_normalize_subscriber(row, segment_id) for row in rows]
+ has_next_page = len(rows) >= config.page_size
+ if has_next_page:
+ manager.save_state(SmailyResumeConfig(page=page + 1, pending_parent_ids=pending))
+ else:
+ manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending[1:]))
if normalized:
yield normalized
+ else:
+ manager.safe_point()
- assert config.page_size is not None
- if len(rows) < config.page_size:
+ if not has_next_page:
break
-
page += 1
- manager.save_state(SmailyResumeConfig(page=page, pending_parent_ids=pending))
pending = pending[1:]
page = 0
- manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))
def _campaign_statistics_rows(
@@ -227,24 +235,23 @@ def _campaign_statistics_rows(
campaigns = _fetch_all_pages(session, base_url, SMAILY_ENDPOINTS["campaigns"], logger)
pending = [str(campaign["id"]) for campaign in campaigns]
- batch: list[dict[str, Any]] = []
+ # One row per campaign is yielded at once: a local batch would hold rows across many requests,
+ # where the run can neither stop nor commit its cursor.
while pending:
campaign_id = pending[0]
stats = _fetch(session, url, {"id": campaign_id}, logger)
if isinstance(stats, dict) and stats:
- batch.append(stats)
+ row = stats
else:
# Keep one row per campaign even when the stats payload is unusable (e.g. an empty
# list for a never-sent campaign) so the table doesn't silently omit campaigns;
# the stats columns stay null. Match the numeric id type of regular stats rows.
logger.warning(f"Smaily: campaign {campaign_id} statistics had an unexpected payload shape")
- batch.append({"id": int(campaign_id) if campaign_id.isdigit() else campaign_id})
+ row = {"id": int(campaign_id) if campaign_id.isdigit() else campaign_id}
pending = pending[1:]
- if batch and (len(batch) >= STATS_CHUNK_SIZE or not pending):
- yield batch
- batch = []
- manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))
+ manager.save_state(SmailyResumeConfig(page=0, pending_parent_ids=pending))
+ yield [row]
def get_rows(
diff --git a/products/warehouse_sources/sources/square/square.py b/products/warehouse_sources/sources/square/square.py
index 24a74d62dfcf..98baf649fdb3 100644
--- a/products/warehouse_sources/sources/square/square.py
+++ b/products/warehouse_sources/sources/square/square.py
@@ -268,10 +268,13 @@ def get_rows(
# resume into.
if config.paginated and next_cursor:
resumable_source_manager.save_state(SquareResumeConfig(cursor=next_cursor))
+ elif config.paginated and next_cursor:
+ resumable_source_manager.save_state(SquareResumeConfig(cursor=next_cursor))
if not config.paginated or not next_cursor:
break
+ resumable_source_manager.safe_point()
cursor = next_cursor
diff --git a/products/warehouse_sources/sources/sumo_logic/sumo_logic.py b/products/warehouse_sources/sources/sumo_logic/sumo_logic.py
index e1f8fd3e7b4b..0e1fadee2cb8 100644
--- a/products/warehouse_sources/sources/sumo_logic/sumo_logic.py
+++ b/products/warehouse_sources/sources/sumo_logic/sumo_logic.py
@@ -212,6 +212,7 @@ def _get_token_paginated_rows(
# Save AFTER yielding the batch — a crash re-yields the last page (merge dedupes on the
# primary key) instead of skipping it.
resumable_source_manager.save_state(SumoLogicResumeConfig(token=next_token))
+ resumable_source_manager.safe_point()
token = next_token
diff --git a/products/warehouse_sources/sources/swarmia/swarmia.py b/products/warehouse_sources/sources/swarmia/swarmia.py
index 785c56f28494..b08c72b27349 100644
--- a/products/warehouse_sources/sources/swarmia/swarmia.py
+++ b/products/warehouse_sources/sources/swarmia/swarmia.py
@@ -28,10 +28,10 @@ class SwarmiaRetryableError(Exception):
pass
[email protected](frozen=False)
[email protected](frozen=True)
class SwarmiaResumeConfig:
- # ISO date (YYYY-MM-DD) of the start of the next window to fetch. Saved after each window's rows
- # are yielded, so a crash resumes at the first window whose rows may not have been committed.
+ # ISO date (YYYY-MM-DD) of the start of the next window to fetch. Staged before each window's rows
+ # are yielded, so the pipeline commits it together with those rows.
next_window_start: str
@@ -377,14 +377,14 @@ def get_rows(
params = _window_params(config, window_start, window_end)
raw_rows = _fetch_csv(session, config.path, params, headers, logger)
rows = _rows_from_csv(config, raw_rows)
- if rows:
- yield rows
- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window rather than
- # skipping it.
if index + 1 < len(windows):
resumable_source_manager.save_state(
SwarmiaResumeConfig(next_window_start=windows[index + 1][0].isoformat())
)
+ if rows:
+ yield rows
+ else:
+ resumable_source_manager.safe_point()
def swarmia_source(
diff --git a/products/warehouse_sources/sources/tailscale/tailscale.py b/products/warehouse_sources/sources/tailscale/tailscale.py
index 45122e272478..3159f0cebb4d 100644
--- a/products/warehouse_sources/sources/tailscale/tailscale.py
+++ b/products/warehouse_sources/sources/tailscale/tailscale.py
@@ -204,7 +204,8 @@ def _key_rows(
response.raise_for_status()
listed = _extract_rows(response.json(), config.data_key)
- rows: list[dict[str, Any]] = []
+ logger.debug(f"Tailscale: listed {len(listed)} rows from {config.name}")
+ # Each detail is yielded as it arrives: a local batch would hold rows across one request per key.
for key in listed:
key_id = key.get("id") if isinstance(key, dict) else None
if not key_id:
@@ -216,11 +217,7 @@ def _key_rows(
continue
detail_response.raise_for_status()
detail = detail_response.json()
- rows.append(detail if isinstance(detail, dict) else key)
-
- logger.debug(f"Tailscale: fetched {len(rows)} rows from {config.name}")
- if rows:
- yield rows
+ yield [detail if isinstance(detail, dict) else key]
def _audit_log_rows(
@@ -280,15 +277,15 @@ def _audit_log_rows(
# must be yielded in ascending eventTime order.
fresh.sort(key=lambda item: item[0] or datetime.min.replace(tzinfo=UTC))
+ resumable_source_manager.save_state(TailscaleResumeConfig(window_start=_format_rfc3339(window_end)))
if fresh:
- yield [row for _, row in fresh]
parsed_times = [event_time for event_time, _ in fresh if event_time is not None]
if parsed_times:
cursor = max(parsed_times)
-
- # Checkpoint AFTER yielding so a crash re-fetches (and dedupes) the last window
- # instead of skipping it.
- resumable_source_manager.save_state(TailscaleResumeConfig(window_start=_format_rfc3339(window_end)))
+ yield [row for _, row in fresh]
+ else:
+ # Most windows are empty, so the walk needs a stop point that does not depend on rows.
+ resumable_source_manager.safe_point()
start = window_end
diff --git a/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py b/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py
index 37ec055b9d6e..85f47eae4083 100644
--- a/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py
+++ b/products/warehouse_sources/sources/terraform_cloud/terraform_cloud.py
@@ -185,13 +185,18 @@ def _page_predates_cutoff(rows: list[dict[str, Any]], cutoff: datetime | None) -
def _iter_workspaces(
- session: requests.Session, organization: str, logger: FilteringBoundLogger
+ session: requests.Session,
+ organization: str,
+ logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[TerraformCloudResumeConfig],
) -> Iterator[dict[str, Any]]:
"""Page through the organization's workspaces, yielding raw JSON:API workspace objects."""
url: str | None = _build_url(f"/organizations/{quote(organization, safe='')}/workspaces", {"page[size]": PAGE_SIZE})
while url:
data = _fetch_json(session, url, logger)
yield from _data_items(data)
+ # The workspace list holds parents, not rows, so the walk can stop here and restart.
+ resumable_source_manager.safe_point()
url = _next_url(data)
@@ -220,6 +225,7 @@ def _get_top_level_rows(
# page rather than skipping it — merge dedupes on the primary key.
if next_url:
resumable_source_manager.save_state(TerraformCloudResumeConfig(next_url=next_url))
+ resumable_source_manager.safe_point()
url = next_url
@@ -249,7 +255,7 @@ def _get_fan_out_rows(
parent workspace without unpacking relationships. Run and state-version ids are globally
unique, so `id` alone stays a valid table-wide primary key.
"""
- workspaces = list(_iter_workspaces(session, organization, logger))
+ workspaces = list(_iter_workspaces(session, organization, logger, resumable_source_manager))
# Resolve the saved workspace bookmark to the slice still to process. If the bookmarked
# workspace no longer exists (deleted between attempts), start over from the first one —
@@ -297,6 +303,7 @@ def _get_fan_out_rows(
# sync re-walks each workspace's full history.
if _page_predates_cutoff(rows, cutoff):
break
+ resumable_source_manager.safe_point()
url = next_url
except requests.HTTPError as exc:
# A workspace deleted between enumeration and this fetch 404s. Skip it rather than
@@ -311,6 +318,7 @@ def _get_fan_out_rows(
resumable_source_manager.save_state(
TerraformCloudResumeConfig(next_url=None, workspace_id=remaining[index + 1]["id"])
)
+ resumable_source_manager.safe_point()
def get_rows(
diff --git a/products/warehouse_sources/sources/ubidots/ubidots.py b/products/warehouse_sources/sources/ubidots/ubidots.py
index f9ba81a61159..599bbdb9db48 100644
--- a/products/warehouse_sources/sources/ubidots/ubidots.py
+++ b/products/warehouse_sources/sources/ubidots/ubidots.py
@@ -8,13 +8,14 @@ from urllib.parse import urlencode, urlparse, urlunparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.sdk import (
UNVERSIONED_API_VERSION,
ResumableSourceManager,
SourceResponse,
make_tracked_session,
+ reach_safe_point,
retry_with_deadline,
)
from products.warehouse_sources.sources.ubidots.settings import (
@@ -147,10 +148,17 @@ def _start_timestamp_ms(value: Any) -> Optional[int]:
return None
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # Every request runs between pages or windows: the rows before it are yielded, and no caller
+ # holds rows while it fetches.
+ reach_safe_point()
+
+
@retry_with_deadline(
retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_page(
@@ -210,12 +218,14 @@ def get_rows(
# Save AFTER yielding so a crash re-fetches from the next cursor (already-yielded pages are
# persisted); merge dedupes the re-pulled page on the primary key.
resumable_source_manager.save_state(UbidotsResumeConfig(next_url=url))
+ resumable_source_manager.safe_point()
def _iter_variable_ids(
session: requests.Session,
base_url: str,
logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[UbidotsResumeConfig],
) -> Iterator[str]:
"""List every variable id via the v2.0 API — the parent set the values stream fans out over."""
url: Optional[str] = f"{base_url}{VARIABLES_LIST_PATH}?{urlencode({'page_size': PAGE_SIZE})}"
@@ -226,6 +236,8 @@ def _iter_variable_ids(
# Direct access on purpose: a variable without an id would otherwise silently drop its
# whole time series from the sync — better to fail loudly on a malformed page.
yield str(item["id"])
+ # The caller has finished every variable on this page and holds no rows.
+ resumable_source_manager.safe_point()
def _initial_values_url(base_url: str, variable_id: str, start: Optional[int]) -> str:
@@ -255,7 +267,7 @@ def get_values_rows(
start = _start_timestamp_ms(db_incremental_field_last_value) if should_use_incremental_field else None
- for variable_id in _iter_variable_ids(session, base_url, logger):
+ for variable_id in _iter_variable_ids(session, base_url, logger, resumable_source_manager):
if variable_id in completed:
continue
@@ -293,9 +305,11 @@ def get_values_rows(
completed_variable_ids=sorted(completed),
)
)
+ resumable_source_manager.safe_point()
completed.add(variable_id)
resumable_source_manager.save_state(UbidotsResumeConfig(completed_variable_ids=sorted(completed)))
+ resumable_source_manager.safe_point()
def _read_capped_body(response: requests.Response) -> bytes:
@@ -328,6 +342,7 @@ def _read_capped_body(response: requests.Response) -> bytes:
retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_data_series(
@@ -428,7 +443,7 @@ def get_values_rows_v2(
floor = min(watermark, now_ms) if watermark is not None else None
url = f"{base_url}{DATA_SERIES_PATH}?{urlencode({'results_format': 'object'})}"
- for variable_id in _iter_variable_ids(session, base_url, logger):
+ for variable_id in _iter_variable_ids(session, base_url, logger, resumable_source_manager):
if variable_id in completed:
continue
@@ -480,9 +495,11 @@ def get_values_rows_v2(
completed_variable_ids=sorted(completed),
)
)
+ resumable_source_manager.safe_point()
completed.add(variable_id)
resumable_source_manager.save_state(UbidotsResumeConfig(completed_variable_ids=sorted(completed)))
+ resumable_source_manager.safe_point()
def ubidots_source(
diff --git a/products/warehouse_sources/sources/unleash/unleash.py b/products/warehouse_sources/sources/unleash/unleash.py
index dd0d8bef1b8c..0273c6bb8af4 100644
--- a/products/warehouse_sources/sources/unleash/unleash.py
+++ b/products/warehouse_sources/sources/unleash/unleash.py
@@ -1,6 +1,5 @@
import re
import dataclasses
-from collections.abc import Iterator
from typing import Any, Optional
from urllib.parse import urlparse
@@ -9,6 +8,7 @@ import requests
from products.warehouse_sources.sources.sdk import (
Endpoint,
OffsetPaginator,
+ Resource,
RESTAPIConfig,
ResumableSourceManager,
SinglePagePaginator,
@@ -170,16 +170,15 @@ def unleash_source(
initial_paginator_state = {"offset": resume.offset}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
- # Persist only when a next page remains; save AFTER a page is yielded so a crash re-yields
- # the last page (merge dedupes) rather than skipping it.
+ # Persist only when a next page remains. The framework stages it with the page it covers.
if state and state.get("offset") is not None:
resumable_source_manager.save_state(UnleashResumeConfig(offset=int(state["offset"])))
- def items() -> Iterator[list[dict[str, Any]]]:
+ def items() -> Resource:
# Re-check at run time (not just at source-create) in case the instance URL was edited or
# now resolves to an internal address (SSRF / DNS rebinding). Only enforced on cloud.
_check_host(instance_url, team_id)
- resource = rest_api_resource(
+ return rest_api_resource(
rest_config,
team_id,
job_id,
@@ -187,10 +186,6 @@ def unleash_source(
resume_hook=save_checkpoint if config.paginated else None,
initial_paginator_state=initial_paginator_state,
)
- for batch in resource:
- # An empty collection/page must not push an empty batch into the pipeline.
- if batch:
- yield batch
return SourceResponse(
name=endpoint,
diff --git a/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py b/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py
index 15b504f61777..3154c7b6b4a4 100644
--- a/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py
+++ b/products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py
@@ -1,6 +1,6 @@
import re
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from datetime import UTC, date, datetime, timedelta
from typing import Any, Optional
from urllib.parse import quote
@@ -161,6 +161,7 @@ def _article_rows(
window_start: date,
window_end: date,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for article in articles:
@@ -169,6 +170,8 @@ def _article_rows(
f"/{quote(article, safe='')}/daily/{window_start:%Y%m%d}/{window_end:%Y%m%d}"
)
rows.extend(_with_date(item) for item in _get_items(session, url, logger))
+ if not rows:
+ safe_point()
return rows
@@ -210,6 +213,7 @@ def _country_pageview_rows(
window_start: date,
window_end: date,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
# One request per month whose first day falls in the window, so consecutive windows never
@@ -234,6 +238,8 @@ def _country_pageview_rows(
"rank": country.get("rank"),
}
)
+ if not rows:
+ safe_point()
month = _next_month(month)
return rows
@@ -272,6 +278,7 @@ def _unique_device_rows(
window_start: date,
window_end: date,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for access_site in UNIQUE_DEVICES_ACCESS_SITES:
@@ -286,6 +293,8 @@ def _unique_device_rows(
if item_date is not None:
item["date"] = datetime(item_date.year, item_date.month, item_date.day, tzinfo=UTC)
rows.append(item)
+ if not rows:
+ safe_point()
return rows
@@ -296,6 +305,7 @@ def _media_request_rows(
window_start: date,
window_end: date,
logger: FilteringBoundLogger,
+ safe_point: Callable[[], None],
) -> list[dict[str, Any]]:
media_agent = agent if agent in MEDIA_REQUEST_AGENTS else "all-agents"
rows: list[dict[str, Any]] = []
@@ -305,6 +315,8 @@ def _media_request_rows(
f"/daily/{window_start:%Y%m%d}/{window_end:%Y%m%d}"
)
rows.extend(_with_date(item) for item in _get_items(session, url, logger))
+ if not rows:
+ safe_point()
return rows
@@ -381,41 +393,50 @@ def _get_rows(
window_days = TOP_WINDOW_DAYS if endpoint in DAILY_REQUEST_ENDPOINTS else WINDOW_DAYS
+ # The staged cursor points at the window in progress until the window is yielded. While the
+ # window holds no rows, a hand-off therefore loses nothing, so the per-request helpers reach a
+ # safe point only before the window has any rows.
+ safe_point = resumable_source_manager.safe_point
+
cursor = start
while cursor <= end_boundary:
window_end = min(cursor + timedelta(days=window_days - 1), end_boundary)
+ rows: list[dict[str, Any]] = []
if endpoint == TOP_ARTICLES_ENDPOINT:
- rows = []
day = cursor
while day <= window_end:
rows.extend(_top_rows(session, normalized_project, access, day, logger))
+ if not rows:
+ safe_point()
day += timedelta(days=1)
elif endpoint == COUNTRY_TOP_ARTICLES_ENDPOINT:
- rows = []
day = cursor
while day <= window_end:
for country in country_codes:
rows.extend(_country_top_rows(session, country, access, day, logger))
+ if not rows:
+ safe_point()
day += timedelta(days=1)
elif endpoint == COUNTRY_PAGEVIEWS_ENDPOINT:
- rows = _country_pageview_rows(session, normalized_project, access, cursor, window_end, logger)
+ rows = _country_pageview_rows(session, normalized_project, access, cursor, window_end, logger, safe_point)
elif endpoint == UNIQUE_DEVICES_ENDPOINT:
- rows = _unique_device_rows(session, normalized_project, cursor, window_end, logger)
+ rows = _unique_device_rows(session, normalized_project, cursor, window_end, logger, safe_point)
elif endpoint == MEDIA_REQUESTS_ENDPOINT:
- rows = _media_request_rows(session, normalized_project, agent, cursor, window_end, logger)
+ rows = _media_request_rows(session, normalized_project, agent, cursor, window_end, logger, safe_point)
elif endpoint == ARTICLE_PAGEVIEWS_ENDPOINT:
- rows = _article_rows(session, normalized_project, access, agent, articles, cursor, window_end, logger)
+ rows = _article_rows(
+ session, normalized_project, access, agent, articles, cursor, window_end, logger, safe_point
+ )
else:
rows = _aggregate_rows(session, normalized_project, access, agent, cursor, window_end, logger)
- if rows:
- yield rows
-
cursor = window_end + timedelta(days=1)
- # Save AFTER yielding so a crash re-fetches (and merge dedupes) the last window
- # instead of skipping it.
resumable_source_manager.save_state(WikipediaPageviewsResumeConfig(next_start=cursor.isoformat()))
+ if rows:
+ yield rows
+ else:
+ safe_point()
def wikipedia_pageviews_source(
diff --git a/products/warehouse_sources/sources/writesonic/writesonic.py b/products/warehouse_sources/sources/writesonic/writesonic.py
index fe71109dd4e4..6a85537aa28b 100644
--- a/products/warehouse_sources/sources/writesonic/writesonic.py
+++ b/products/warehouse_sources/sources/writesonic/writesonic.py
@@ -245,8 +245,8 @@ def _iter_daily(
"""Sync a daily export endpoint one UTC day at a time.
The required `date` param is both the incremental granularity and the resume granularity.
- State is saved *after* each page's rows are yielded, so a crash re-fetches the in-flight
- page (merge dedupes on the primary key) rather than skipping it."""
+ The next page is staged before each page's rows are yielded, so the pipeline commits it
+ together with those rows."""
resume = manager.load_state() if manager.can_resume() else None
resumed_date = _to_date(resume.date) if resume and resume.date else None
current = resumed_date or start_date
@@ -264,11 +264,13 @@ def _iter_daily(
# Content export rows don't carry the export date; stamp it in since it's part
# of the primary key, the partition key, and the incremental cursor.
items = [{**item, "date": day} for item in items]
- yield items
manager.save_state(WritesonicResumeConfig(date=day, page=page + 1))
+ yield items
next_day = current + timedelta(days=1)
manager.save_state(WritesonicResumeConfig(date=next_day.isoformat(), page=1))
+ # Every row of the day is yielded, so the staged day loses nothing.
+ manager.safe_point()
current = next_day
start_page = 1
@@ -292,8 +294,8 @@ def _iter_config(
for page, items in _iter_pages(
endpoint_config, api_key=api_key, params=params, start_page=start_page, logger=logger
):
- yield items
manager.save_state(WritesonicResumeConfig(page=page + 1))
+ yield items
def get_rows(
diff --git a/products/warehouse_sources/sources/zenduty/zenduty.py b/products/warehouse_sources/sources/zenduty/zenduty.py
index 0ffeb2e462fc..730bf1b514ed 100644
--- a/products/warehouse_sources/sources/zenduty/zenduty.py
+++ b/products/warehouse_sources/sources/zenduty/zenduty.py
@@ -119,9 +119,16 @@ def _with_team_id(row: dict[str, Any], team_id: str) -> dict[str, Any]:
def _list_team_ids(
- session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger, page_size: int
+ session: requests.Session,
+ headers: dict[str, str],
+ logger: FilteringBoundLogger,
+ page_size: int,
+ resumable_source_manager: ResumableSourceManager[ZendutyResumeConfig],
) -> list[str]:
- """Enumerate every team's `unique_id` — the parents fan-out endpoints iterate over."""
+ """Enumerate every team's `unique_id` — the parents fan-out endpoints iterate over.
+
+ No rows are held while it runs, so each page is a safe point.
+ """
url: str | None = _initial_url(ZENDUTY_ENDPOINTS["teams"].path, page_size)
team_ids: list[str] = []
while url:
@@ -131,6 +138,7 @@ def _list_team_ids(
unique_id = row.get("unique_id")
if isinstance(unique_id, str) and unique_id:
team_ids.append(unique_id)
+ resumable_source_manager.safe_point()
url = _ensure_zenduty_url(next_url) if next_url else None
return team_ids
@@ -156,6 +164,7 @@ def _paginate_top_level(
# dedupes on the primary key. Advance the URL before the next fetch to avoid re-looping it.
# Validate before saving so an off-origin URL never lands in the resume checkpoint.
resumable_source_manager.save_state(ZendutyResumeConfig(next_url=_ensure_zenduty_url(next_url)))
+ resumable_source_manager.safe_point()
url = next_url
@@ -168,7 +177,7 @@ def _paginate_fan_out(
resume: ZendutyResumeConfig | None,
page_size: int,
) -> Iterator[list[dict[str, Any]]]:
- team_ids = _list_team_ids(session, headers, logger, page_size)
+ team_ids = _list_team_ids(session, headers, logger, page_size, resumable_source_manager)
if not team_ids:
return
@@ -194,12 +203,14 @@ def _paginate_fan_out(
resumable_source_manager.save_state(
ZendutyResumeConfig(next_url=_ensure_zenduty_url(next_url), team_id=team_id)
)
+ resumable_source_manager.safe_point()
url = next_url
continue
# Team done — checkpoint at the next team's start so a resume skips completed teams
# instead of re-walking them.
if idx + 1 < len(team_ids):
resumable_source_manager.save_state(ZendutyResumeConfig(next_url=None, team_id=team_ids[idx + 1]))
+ resumable_source_manager.safe_point()
break