:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): resume 9 simple sources from a saved cursor (#127477)
Tom Owers2235415b7fa9
diff --git a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txtindex ee2aeff8e6e2..e7b32bb4968b 100644--- a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt+++ b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt@@ -46,7 +46,6 @@ shutterstock/shutterstock.py::get_rowstelli/telli.py::telli_source.get_rowstwenty/twenty.py::twenty_source.itemstwitter/twitter.py::_iter_pages-vitally/vitally.py::vitally_sourcewho_gho/who_gho.py::_catalog_pageswho_gho/who_gho.py::_dimension_values_pageswho_gho/who_gho.py::_indicator_data_pagesdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex eba4f8224e68..247149d422dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -1,7 +1,6 @@AmazonS3 not-checkable:schemasAppStoreConnect not-checkable:no-requestAppleSearchAds not-checkable:no-request-AppsFlyer fail:safe-pointAttentive not-checkable:databaseAwsAthena not-checkable:schemasAwsDocumentDB not-checkable:driver:pymongo@@ -17,7 +16,6 @@ CiscoDuo not-checkable:no-requestClickHouse not-checkable:schemasCloudflareD1 not-checkable:schemasCloudflareR2 not-checkable:schemas-Coda fail:safe-pointConvex not-checkable:schemasCosmosDB not-checkable:schemasCoveralls not-checkable:no-request@@ -42,15 +40,12 @@ GooglePlayConsole not-checkable:no-requestGoogleSearchConsole not-checkable:databaseGoogleSheets not-checkable:schemasGrafanaIRM not-checkable:no-request-Groq fail:safe-pointHatchet not-checkable:no-requestHelpScout not-checkable:databaseHubspot not-checkable:databaseInstagram not-checkable:databaseIntercom not-checkable:databaseKafka not-checkable:driver:confluent_kafka-Kandji fail:safe-point-Kernel fail:safe-pointKestra not-checkable:no-requestLinear not-checkable:databaseLinkedinAds not-checkable:database@@ -60,13 +55,11 @@ MetaAds not-checkable:databaseModal not-checkable:driver:grpclibMongoDB not-checkable:driver:pymongoMotherduck not-checkable:driver:duckdb-MyHours fail:safe-pointMySQL not-checkable:driver:psycopgNeo4j not-checkable:schemasNeon not-checkable:driver:psycopgOpenMeteo not-checkable:no-requestOpenWeather not-checkable:no-request-Optimizely fail:safe-pointPardot not-checkable:databasePinterestAds not-checkable:databasePipedrive not-checkable:database@@ -87,9 +80,7 @@ TemporalIO not-checkable:driver:temporalio.clientTikTokAds not-checkable:databaseTrino not-checkable:schemasTwitterAds not-checkable:database-Vitally fail:safe-pointWrike not-checkable:no-requestYouTubeAnalytics not-checkable:database-ZapierSupportedStorage fail:safe-pointshared-path:rest-client-paginate fail:safe-pointshared-path:rest-framework-wrapped-resource fail:safe-pointdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.pyindex 693f0b4c1be8..9160273b608d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py@@ -12,7 +12,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldSelectConfig,SourceFieldSelectConfigOption,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (CanonicalDescriptions,)@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.erris_transient_egress_proxy_error,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchemafrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.vitally import (@@ -32,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.seUPDATED_AT_INCREMENTAL_FIELD,)from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally import (+ VitallyResumeConfig,list_custom_object_definitions,validate_credentials as validate_vitally_credentials,vitally_source,@@ -40,7 +42,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType@SourceRegistry.register-class VitallySource(SimpleSource[VitallySourceConfig]):+class VitallySource(ResumableSource[VitallySourceConfig, VitallyResumeConfig]):codebase_detection = detect(env=("^VITALLY_",), script_hosts=("cdn.vitally.io",))api_docs_url = "https://docs.vitally.io"@@ -151,24 +153,32 @@ class VitallySource(SimpleSource[VitallySourceConfig]):"403 Client Error: Forbidden for url": "Your Vitally secret token does not have permission to access this data. Please check the token's permissions and reconnect.",}- def source_for_pipeline(self, config: VitallySourceConfig, inputs: SourceInputs) -> SourceResponse:- items = vitally_source(- secret_token=config.secret_token,- region=config.region.selection,- subdomain=config.region.subdomain,- endpoint=inputs.schema_name,- team_id=inputs.team_id,- job_id=inputs.job_id,- should_use_incremental_field=inputs.should_use_incremental_field,- db_incremental_field_last_value=inputs.db_incremental_field_last_value- if inputs.should_use_incremental_field- else None,- logger=inputs.logger,- )+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[VitallyResumeConfig]:+ return ResumableSourceManager[VitallyResumeConfig](inputs, VitallyResumeConfig)+ def source_for_pipeline(+ self,+ config: VitallySourceConfig,+ resumable_source_manager: ResumableSourceManager[VitallyResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return SourceResponse(name=inputs.schema_name,- items=lambda: items,+ on_complete=resumable_source_manager.clear_state,+ items=lambda: vitally_source(+ secret_token=config.secret_token,+ region=config.region.selection,+ subdomain=config.region.subdomain,+ endpoint=inputs.schema_name,+ team_id=inputs.team_id,+ job_id=inputs.job_id,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value+ if inputs.should_use_incremental_field+ else None,+ logger=inputs.logger,+ resumable_source_manager=resumable_source_manager,+ ),primary_keys=["id"],partition_count=1, # this enables partitioningpartition_size=1, # this enables partitioningdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.pyindex 77ce0b155a29..e3aa9d6ff3ca 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py@@ -7,15 +7,21 @@ from requests import HTTPErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,+ SourceDriver,always,scripted_network,)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.vitally import (+ VitallyRegionConfig,+ VitallySourceConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.settings import (CUSTOM_OBJECT_SCHEMA_PREFIX,)from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.source import VitallySourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally import (VitallyPaginator,+ VitallyResumeConfig,get_custom_object_records_resource,get_messages,list_custom_object_definitions,@@ -23,6 +29,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vi)+def _no_resume() -> Mock:+ return Mock(can_resume=Mock(return_value=False))++def _make_response(json_data: dict[str, Any], status_code: int = 200) -> Mock:response = Mock()response.status_code = status_code@@ -120,12 +130,108 @@ class TestGetMessages:# KeyError('results'); now raise_for_status lets the real HTTP error through.with scripted_network(always(ScriptedResponse(status=401, json={"message": "Unauthorized"}))):with pytest.raises(HTTPError, match="401"):- list(get_messages("secret", "EU", None, None, False, MagicMock()))+ list(get_messages("secret", "EU", None, None, False, MagicMock(), _no_resume()))def test_handles_missing_results_key_without_keyerror(self):# Defensive: a 200 response that omits "results" must end pagination cleanly, not KeyError.with scripted_network([ScriptedResponse(json={"next": None})]):- assert list(get_messages("secret", "EU", None, None, False, MagicMock())) == []+ assert list(get_messages("secret", "EU", None, None, False, MagicMock(), _no_resume())) == []+++class TestResume:+ @pytest.mark.parametrize(+ "endpoint, saved, script, expected_paths, expected_from, expected_ids",+ [+ pytest.param(+ "Organizations",+ VitallyResumeConfig(cursor="cursor-2"),+ [ScriptedResponse(json={"results": [{"id": "org-2"}], "next": None})],+ ["/resources/organizations"],+ ["cursor-2"],+ ["org-2"],+ id="list_endpoint_starts_at_saved_cursor",+ ),+ pytest.param(+ "Messages",+ VitallyResumeConfig(cursor="cursor-2", conversations_done=["c1"]),+ [+ ScriptedResponse(json={"results": [{"id": "c1"}, {"id": "c2"}], "next": None}),+ ScriptedResponse(json={"messages": [{"id": "m2"}]}),+ ],+ ["/resources/conversations", "/resources/conversations/c2"],+ ["cursor-2", None],+ ["m2"],+ id="messages_skip_conversations_already_handed_on",+ ),+ pytest.param(+ "Messages",+ VitallyResumeConfig(cursor=None, conversations_done=["c1"]),+ [+ ScriptedResponse(json={"results": [{"id": "c3"}, {"id": "c1"}, {"id": "c2"}], "next": None}),+ ScriptedResponse(json={"messages": [{"id": "m3"}]}),+ ScriptedResponse(json={"messages": [{"id": "m2"}]}),+ ],+ ["/resources/conversations", "/resources/conversations/c3", "/resources/conversations/c2"],+ [None, None, None],+ ["m3", "m2"],+ id="messages_read_a_conversation_that_moved_ahead_of_finished_ones",+ ),+ pytest.param(+ "Organizations",+ VitallyResumeConfig(completed=True),+ [],+ [],+ [],+ [],+ id="list_endpoint_reads_nothing_after_its_last_page",+ ),+ ],+ )+ def test_resumes_from_saved_state(+ self,+ endpoint: str,+ saved: VitallyResumeConfig,+ script: list[ScriptedResponse],+ expected_paths: list[str],+ expected_from: list[str | None],+ expected_ids: list[str],+ ) -> None:+ driver = SourceDriver(+ VitallySource(),+ VitallySourceConfig(secret_token="secret", region=VitallyRegionConfig(subdomain="", selection="EU")),+ )++ result = driver.run(endpoint, script, resume_state=saved)++ assert result.raised is None+ assert result.paths == expected_paths+ assert result.params("from") == expected_from+ assert [row["id"] for row in result.rows] == expected_ids++ def test_messages_stage_each_conversation_before_its_messages(self) -> None:+ driver = SourceDriver(+ VitallySource(),+ VitallySourceConfig(secret_token="secret", region=VitallyRegionConfig(subdomain="", selection="EU")),+ )++ result = driver.run(+ "Messages",+ [+ ScriptedResponse(json={"results": [{"id": "c1"}, {"id": "c2"}], "next": "cursor-2"}),+ ScriptedResponse(json={"messages": [{"id": "m1"}]}),+ ScriptedResponse(json={"messages": []}),+ ScriptedResponse(json={"results": [], "next": None}),+ ],+ )++ assert result.raised is None+ assert result.items == [[{"id": "m1", "conversation_updated_at": None}]]+ assert result.saved_states == [+ VitallyResumeConfig(cursor=None, conversations_done=[]),+ VitallyResumeConfig(cursor=None, conversations_done=["c1"]),+ VitallyResumeConfig(cursor=None, conversations_done=["c1", "c2"]),+ VitallyResumeConfig(cursor="cursor-2"),+ ]class TestGetCustomObjectRecordsResource:@@ -172,6 +278,7 @@ class TestVitallySourceCustomObjectRouting:job_id="job-1",logger=MagicMock(),db_incremental_field_last_value=None,+ resumable_source_manager=_no_resume(),should_use_incremental_field=False,))@@ -202,6 +309,7 @@ class TestVitallySourceCustomObjectRouting:job_id="job-1",logger=MagicMock(),db_incremental_field_last_value=None,+ resumable_source_manager=_no_resume(),should_use_incremental_field=False,))diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.pyindex 75bfc5e8d975..f880b64611d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py@@ -1,11 +1,15 @@import base64+from collections.abc import Iterable, Iterator+from dataclasses import fieldfrom datetime import datetimefrom typing import Any, Optionalfrom dateutil import parser-from requests import HTTPError, JSONDecodeError, Request, Response+from requests import HTTPError, JSONDecodeError, Request, Response, Sessionfrom structlog.types import FilteringBoundLogger+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,@@ -13,11 +17,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginatorfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.vitally.settings import (CUSTOM_OBJECT_SCHEMA_PREFIX,)+@frozen+class VitallyResumeConfig:+ # The `from` cursor of the page to read. None is the first page.+ cursor: str | None = None+ # Messages only: the ids of that page's conversations already handed on. Ids, not a count, because+ # a conversation updated between attempts moves within the `updatedAt` order.+ conversations_done: list[str] = field(default_factory=list)+ # List endpoints only: the last page is handed on, so a resume reads nothing again.+ completed: bool = False++def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResource:resources: dict[str, EndpointResource] = {"Organizations": {@@ -381,12 +397,26 @@ class VitallyPaginator(BasePaginator):else:self._has_next_page = False+ def init_request(self, request: Request) -> None:+ # Apply a resumed cursor to the first request.+ if self._cursor is not None:+ self.update_request(request)+def update_request(self, request: Request) -> None:if request.params is None:request.params = {}request.params["from"] = self._cursor+ def get_resume_state(self) -> Optional[dict[str, Any]]:+ return {"cursor": self._cursor} if self._has_next_page and self._cursor else None++ def set_resume_state(self, state: dict[str, Any]) -> None:+ cursor = state.get("cursor")+ if cursor:+ self._cursor = str(cursor)+ self._has_next_page = True+def get_messages(secret_token: str,@@ -395,7 +425,8 @@ def get_messages(db_incremental_field_last_value: Optional[Any],should_use_incremental_field: bool,logger: FilteringBoundLogger,-):+ manager: ResumableSourceManager[VitallyResumeConfig],+) -> Iterator[list[dict[str, Any]]]:"""Messages are a field on conversations which only get returnedwhen you request each conversation individually. This queriesfor conversations and then gets and yields the messages for each@@ -429,10 +460,19 @@ def get_messages(headers={"Authorization": f"Basic {basic_token}:"},)+ resume = manager.load_state() if manager.can_resume() else None+ page_cursor = resume.cursor if resume is not None else None+ done = list(resume.conversations_done) if resume is not None else []+ if page_cursor is not None:+ paginator.set_resume_state({"cursor": page_cursor})+logger.debug("Requesting first page")with make_tracked_session() as session:while paginator.has_next_page:+ # Every conversation before this point is handed on, so an empty page can end the run here.+ manager.save_state(VitallyResumeConfig(cursor=page_cursor, conversations_done=list(done)))+ manager.safe_point()paginator.update_request(request)prepared_request = session.prepare_request(request)response = session.send(prepared_request)@@ -442,33 +482,60 @@ def get_messages(results = response.json().get("results") or []for conversation in results:- id = conversation.get("id")- conversation_updated_at = conversation.get("updatedAt")- logger.debug(f"Requesting messages for conversation {id}")-- conversation_response = session.get(- f"{get_base_url(region, subdomain)}resources/conversations/{id}",- headers={"Authorization": f"Basic {basic_token}:"},- )-- try:- conversation_response.raise_for_status()-- messages = conversation_response.json().get("messages") or []- logger.debug(f"Yielding {len(messages)} messages")- for message in messages:- message["conversation_updated_at"] = conversation_updated_at- yield message- except HTTPError as e:- logger.debug(- f"Failed to fetch messages for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"- )- except JSONDecodeError as e:- logger.debug(- f"Failed to decode JSON response for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"- )+ # A resumed page is read again from its start, so skip the conversations already handed on.+ conversation_id = str(conversation.get("id"))+ if conversation_id in done:+ continue+ messages = _conversation_messages(session, conversation, region, subdomain, basic_token, logger)+ done.append(conversation_id)+ manager.save_state(VitallyResumeConfig(cursor=page_cursor, conversations_done=list(done)))+ if messages:+ # One list per conversation, so its messages and the cursor past it commit together.+ yield messages+ else:+ manager.safe_point()+ done = []paginator.update_state(response)+ next_state = paginator.get_resume_state()+ page_cursor = next_state["cursor"] if next_state else None+++def _conversation_messages(+ session: Session,+ conversation: dict[str, Any],+ region: str,+ subdomain: Optional[str],+ basic_token: str,+ logger: FilteringBoundLogger,+) -> list[dict[str, Any]]:+ id = conversation.get("id")+ conversation_updated_at = conversation.get("updatedAt")+ logger.debug(f"Requesting messages for conversation {id}")++ conversation_response = session.get(+ f"{get_base_url(region, subdomain)}resources/conversations/{id}",+ headers={"Authorization": f"Basic {basic_token}:"},+ timeout=60,+ )++ try:+ conversation_response.raise_for_status()++ messages = conversation_response.json().get("messages") or []+ logger.debug(f"Yielding {len(messages)} messages")+ for message in messages:+ message["conversation_updated_at"] = conversation_updated_at+ return messages+ except HTTPError as e:+ logger.debug(+ f"Failed to fetch messages for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"+ )+ except JSONDecodeError as e:+ logger.debug(+ f"Failed to decode JSON response for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"+ )+ return []def get_base_url(region: str, subdomain: Optional[str]) -> str:@@ -487,13 +554,19 @@ def vitally_source(job_id: str,logger: FilteringBoundLogger,db_incremental_field_last_value: Optional[Any],+ resumable_source_manager: ResumableSourceManager[VitallyResumeConfig],should_use_incremental_field: bool = False,-):+) -> Iterable[Any]:if endpoint == "Messages":- yield from get_messages(- secret_token, region, subdomain, db_incremental_field_last_value, should_use_incremental_field, logger+ return get_messages(+ secret_token,+ region,+ subdomain,+ db_incremental_field_last_value,+ should_use_incremental_field,+ logger,+ resumable_source_manager,)- returnif endpoint.startswith(CUSTOM_OBJECT_SCHEMA_PREFIX):machine_name = endpoint[len(CUSTOM_OBJECT_SCHEMA_PREFIX) :]@@ -532,7 +605,26 @@ def vitally_source("resources": [endpoint_resource],}- yield from rest_api_resource(config, team_id, job_id, db_incremental_field_last_value)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state and state.get("cursor"):+ resumable_source_manager.save_state(VitallyResumeConfig(cursor=str(state["cursor"])))+ elif state is None:+ resumable_source_manager.save_state(VitallyResumeConfig(completed=True))++ if resume is not None and resume.completed:+ return []++ # Returned, not wrapped, so the framework reaches a safe point after each page.+ return rest_api_resource(+ config,+ team_id,+ job_id,+ db_incremental_field_last_value,+ resume_hook=save_checkpoint,+ initial_paginator_state={"cursor": resume.cursor} if resume is not None and resume.cursor else None,+ )def validate_credentials(secret_token: str, region: str, subdomain: Optional[str]) -> bool:diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.pyindex 61b8db733a74..6c8f1d0d6798 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py@@ -3060,12 +3060,8 @@ async def test_worker_shutdown_desc_sort_order(team):def mock_raise_if_is_worker_shutdown(self):raise WorkerShuttingDownError("test_id", "test_type", "test_queue", 1, "test_workflow", "test_workflow_type")- def mock_get_messages(*args, **kwargs):- yield {- "id": "test-message-id",- "conversation_updated_at": datetime.now().isoformat(),- "created_at": datetime.now().isoformat(),- }+ def mock_get_rows(*args, **kwargs):+ yield [{"_board_id": "1", "id": "test-log-id", "created_at": datetime.now().isoformat()}]with (mock.patch.object(ShutdownMonitor, "raise_if_is_worker_shutdown", mock_raise_if_is_worker_shutdown),@@ -3076,22 +3072,19 @@ async def test_worker_shutdown_desc_sort_order(team):"products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher.DEFAULT_CHUNK_SIZE", 1),mock.patch(- "products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally.get_messages",- mock_get_messages,+ "products.warehouse_sources.sources.monday.monday.get_rows",+ mock_get_rows,),):_, inputs = await _run(team=team,- schema_name="Messages",- table_name="vitally_messages",- source_type="Vitally",- job_inputs={- "secret_token": "test_token",- "region": {"selection": "EU", "subdomain": ""},- },+ schema_name="board_activity_logs",+ table_name="monday_board_activity_logs",+ source_type="Monday",+ job_inputs={"api_token": "test_token"},mock_data_response=[],sync_type=ExternalDataSchemaSyncType.INCREMENTAL,- sync_type_config={"incremental_field": "conversation_updated_at", "incremental_field_type": "datetime"},+ sync_type_config={"incremental_field": "created_at", "incremental_field_type": "datetime"},ignore_assertions=True,)diff --git a/products/warehouse_sources/sources/appsflyer/appsflyer.py b/products/warehouse_sources/sources/appsflyer/appsflyer.pyindex 0649785daa1c..9b39512b61a1 100644--- a/products/warehouse_sources/sources/appsflyer/appsflyer.py+++ b/products/warehouse_sources/sources/appsflyer/appsflyer.py@@ -15,7 +15,13 @@ from products.warehouse_sources.sources.appsflyer.settings import (AppsFlyerEndpointConfig,AppsFlyerReportKind,)-from products.warehouse_sources.sources.sdk import SourceResponse, frozen, make_tracked_session, retry_with_deadline+from products.warehouse_sources.sources.sdk import (+ ResumableSourceManager,+ SourceResponse,+ frozen,+ make_tracked_session,+ retry_with_deadline,+)APPSFLYER_BASE_URL = "https://hq1.appsflyer.com"# Aggregate pull requests cap the date range at ~1000 days.@@ -54,6 +60,12 @@ MAX_REJECTION_REASON_CHARS = 300_HTML_MARKUP = re.compile(r"<\s*(!doctype|html|head|body)\b", re.IGNORECASE)+@frozen+class AppsFlyerResumeConfig:+ # The first day of the next request window, as an ISO date.+ next_window_start: str++@frozenclass _ReportWindow:start: date@@ -351,6 +363,7 @@ def get_rows(app_id: str,endpoint: str,logger: FilteringBoundLogger,+ manager: ResumableSourceManager[AppsFlyerResumeConfig],should_use_incremental_field: bool = False,db_incremental_field_last_value: Any = None,) -> Iterator[list[dict[str, Any]]]:@@ -360,6 +373,13 @@ def get_rows(today = datetime.now(UTC).date()start = _window_start(config, today, should_use_incremental_field, db_incremental_field_last_value)+ # A full refresh appends, so the part of a window read before a hand-off would load twice.+ resume = manager.load_state() if should_use_incremental_field and manager.can_resume() else None+ if resume is not None:+ # Never earlier than the computed start, which keeps a raw pull inside its lookback limit.+ start = max(start, date.fromisoformat(resume.next_window_start))+ if start > today:+ returnchunk: list[dict[str, Any]] = []for window in _request_windows(config.kind, start, today):@@ -379,8 +399,14 @@ def get_rows(window_end=window.end.isoformat(),row_cap=RAW_MAX_ROWS,)- if chunk:- yield chunk+ # A window is the resume unit: rows have no documented order inside one, so a resume reads+ # a part-way window again. The rest of the window goes out with the cursor past it.+ manager.save_state(AppsFlyerResumeConfig(next_window_start=(window.end + timedelta(days=1)).isoformat()))+ if chunk:+ yield chunk+ chunk = []+ else:+ manager.safe_point()def appsflyer_source(@@ -388,6 +414,7 @@ def appsflyer_source(app_id: str,endpoint: str,logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[AppsFlyerResumeConfig],should_use_incremental_field: bool = False,db_incremental_field_last_value: Any = None,) -> SourceResponse:@@ -395,11 +422,13 @@ def appsflyer_source(return SourceResponse(name=endpoint,+ on_complete=resumable_source_manager.clear_state,items=lambda: get_rows(api_token=api_token,app_id=app_id,endpoint=endpoint,logger=logger,+ manager=resumable_source_manager,should_use_incremental_field=should_use_incremental_field,db_incremental_field_last_value=db_incremental_field_last_value,),@@ -409,6 +438,7 @@ def appsflyer_source(partition_mode="datetime",partition_format=config.partition_format,partition_keys=[config.partition_key],+ supports_resume=should_use_incremental_field,# Dimension keys can collide (e.g. blank campaign values) — an expected trait of report# data, not something the user can fix. Don't set has_duplicate_primary_keys: that flag# tells validate_incremental_sync to block incremental syncing altogether. The merge's owndiff --git a/products/warehouse_sources/sources/appsflyer/source.py b/products/warehouse_sources/sources/appsflyer/source.pyindex 31791af47822..9139341ac707 100644--- a/products/warehouse_sources/sources/appsflyer/source.py+++ b/products/warehouse_sources/sources/appsflyer/source.py@@ -5,6 +5,7 @@ import requestsfrom products.warehouse_sources.sources.appsflyer._config import AppsFlyerSourceConfigfrom products.warehouse_sources.sources.appsflyer.appsflyer import (AppsFlyerCredentialsError,+ AppsFlyerResumeConfig,AppsFlyerRetryableError,appsflyer_source,validate_credentials as validate_appsflyer_credentials,@@ -16,7 +17,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -30,7 +32,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class AppsFlyerSource(SimpleSource[AppsFlyerSourceConfig]):+class AppsFlyerSource(ResumableSource[AppsFlyerSourceConfig, AppsFlyerResumeConfig]):codebase_detection = detect(env=("^APPSFLYER_",),script_hosts=("websdk.appsflyersdk.com",),@@ -190,12 +192,24 @@ Raw data tables (installs, in-app events, uninstalls, retargeting conversions, a"Could not reach AppsFlyer to validate credentials. This may be a temporary rate-limit or network issue — please try again.",)- def source_for_pipeline(self, config: AppsFlyerSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def resume_covers_run(self, *, incremental_or_append: bool, schema_name: str | None = None) -> bool:+ return incremental_or_append++ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[AppsFlyerResumeConfig]:+ return ResumableSourceManager[AppsFlyerResumeConfig](inputs, AppsFlyerResumeConfig)++ def source_for_pipeline(+ self,+ config: AppsFlyerSourceConfig,+ resumable_source_manager: ResumableSourceManager[AppsFlyerResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return appsflyer_source(api_token=config.api_token,app_id=config.app_id,endpoint=inputs.schema_name,logger=inputs.logger,+ resumable_source_manager=resumable_source_manager,should_use_incremental_field=inputs.should_use_incremental_field,db_incremental_field_last_value=inputs.db_incremental_field_last_valueif inputs.should_use_incremental_fielddiff --git a/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py b/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.pyindex 197741b72628..b9347113e8ab 100644--- a/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py+++ b/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py@@ -13,6 +13,7 @@ from products.warehouse_sources.sources.appsflyer.appsflyer import (RAW_MAX_LOOKBACK_DAYS,RAW_MAX_REQUEST_DAYS,AppsFlyerCredentialsError,+ AppsFlyerResumeConfig,AppsFlyerRetryableError,_normalize_header,_parse_csv_rows,@@ -283,6 +284,45 @@ class TestRawAndMasterWindows:for window_start, window_end in windows:assert 0 <= (window_end - window_start).days < MASTER_MAX_REQUEST_DAYS+ @pytest.mark.parametrize(+ "endpoint, incremental, saved_days_ago, expected_first_from_days_ago, expected_requests",+ [+ # A raw pull resumes at the saved window, and each window stages the start of the next.+ ("installs", True, 20, 20, 3),+ # The saved day is clamped to the lookback limit, which moves forward between attempts.+ ("installs", True, RAW_MAX_LOOKBACK_DAYS + 5, RAW_MAX_LOOKBACK_DAYS, 13),+ ("master_report", True, 40, 40, 2),+ # A single-window report whose window is already written reads nothing.+ ("daily_report", True, -1, None, 0),+ # A full refresh appends a replayed window part twice, so it reads from the start.+ ("installs", False, 20, RAW_MAX_LOOKBACK_DAYS, 13),+ ],+ )+ def test_resumes_at_the_saved_window(+ self,+ endpoint: str,+ incremental: bool,+ saved_days_ago: int,+ expected_first_from_days_ago: int | None,+ expected_requests: int,+ ) -> None:+ today = datetime.now(UTC).date()+ saved = AppsFlyerResumeConfig(next_window_start=(today - timedelta(days=saved_days_ago)).isoformat())++ result = _driver().run(+ endpoint, always(_csv_response()), resume_state=saved, should_use_incremental_field=incremental+ )++ assert result.raised is None+ assert result.response is not None+ assert result.response.supports_resume is incremental+ assert len(result.requests) == expected_requests+ if expected_first_from_days_ago is not None:+ assert result.params("from")[0] == (today - timedelta(days=expected_first_from_days_ago)).isoformat()+ assert result.final_state == AppsFlyerResumeConfig(+ next_window_start=(today + timedelta(days=1)).isoformat()+ )+@pytest.mark.parametrize("endpoint", ["installs", "in_app_events", "ad_revenue"])def test_raw_incremental_repulls_a_whole_window_before_the_watermark(self, endpoint: str) -> None:# A worker shutdown can interrupt a window part-way through, and AppsFlyer documents no row@@ -371,5 +411,7 @@ class TestRawAndMasterWindows:class TestAppsFlyerSourceResponse:def test_geo_report_key_includes_country(self):- response = appsflyer_source("token", "id123", "geo_report", mock.MagicMock())+ response = appsflyer_source(+ "token", "id123", "geo_report", mock.MagicMock(), resumable_source_manager=mock.MagicMock()+ )assert "country" in (response.primary_keys or [])diff --git a/products/warehouse_sources/sources/coda/coda.py b/products/warehouse_sources/sources/coda/coda.pyindex a69fb783b801..d2f1d339e1ac 100644--- a/products/warehouse_sources/sources/coda/coda.py+++ b/products/warehouse_sources/sources/coda/coda.py@@ -1,4 +1,4 @@-from collections.abc import Callable+from collections.abc import Callable, Iterablefrom typing import Any, castfrom products.warehouse_sources.sources.coda.settings import CODA_ENDPOINTS@@ -7,7 +7,9 @@ from products.warehouse_sources.sources.sdk import (JSONResponseCursorPaginator,ResponseAction,RESTAPIConfig,+ ResumableSourceManager,SourceResponse,+ frozen,make_tracked_session,rest_api_resources,validate_via_probe,@@ -17,6 +19,16 @@ CODA_BASE_URL = "https://coda.io/apis/v1"PAGE_SIZE = 100ROWS_PAGE_SIZE = 200++@frozen+class CodaResumeConfig:+ # What the framework handed the resume hook: the next page token of a list endpoint, or the+ # finished and in-progress parents of a fan-out. Stored unchanged.+ paginator_state: dict[str, Any] | None = None+ # The last page of a list endpoint is handed on, so a resume reads nothing again.+ completed: bool = False++# The docs list also returns docs the token user only opened (shared or published docs from other# workspaces). Their sharing settings and their workspace's members are not readable, so skip those# parents instead of failing the whole fan-out.@@ -305,6 +317,7 @@ def coda_source(endpoint: str,team_id: int,job_id: str,+ resumable_source_manager: ResumableSourceManager[CodaResumeConfig],) -> SourceResponse:resource_chain = _resource_chain(endpoint)config = CODA_ENDPOINTS[endpoint]@@ -318,12 +331,34 @@ def coda_source("resources": cast("list[str | EndpointResource]", resource_chain),}- resources = rest_api_resources(rest_config, team_id, job_id, None)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state:+ resumable_source_manager.save_state(CodaResumeConfig(paginator_state=state))+ elif state is None:+ resumable_source_manager.save_state(CodaResumeConfig(completed=True))++ # The framework resumes a fan-out with one dependent level only. Rows and columns chain two+ # levels, so they read every doc and table again after a resume.+ resources = rest_api_resources(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=resume.paginator_state if resume is not None else None,+ )resource = next(r for r in resources if r.name == endpoint)+ completed = resume is not None and resume.completed++ def items() -> Iterable[Any]:+ return [] if completed else resourcereturn SourceResponse(name=endpoint,- items=lambda: resource,+ on_complete=resumable_source_manager.clear_state,+ items=items,primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/sources/coda/source.py b/products/warehouse_sources/sources/coda/source.pyindex 08dc6f374a70..3db04440d0ba 100644--- a/products/warehouse_sources/sources/coda/source.py+++ b/products/warehouse_sources/sources/coda/source.py@@ -2,6 +2,7 @@ from typing import Optional, castfrom products.warehouse_sources.sources.coda._config import CodaSourceConfigfrom products.warehouse_sources.sources.coda.coda import (+ CodaResumeConfig,coda_source,validate_credentials as validate_coda_credentials,)@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -26,7 +28,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class CodaSource(SimpleSource[CodaSourceConfig]):+class CodaSource(ResumableSource[CodaSourceConfig, CodaResumeConfig]):codebase_detection = detect(env=("^CODA_API_",), npm=("@codahq/packs-sdk",), pypi=("codaio",))lists_tables_without_credentials = True # static endpoint catalog — safe for public docs@@ -97,10 +99,19 @@ You can generate an API token in [Coda account settings](https://coda.io/accountreturn False, "Invalid Coda API token"- def source_for_pipeline(self, config: CodaSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[CodaResumeConfig]:+ return ResumableSourceManager[CodaResumeConfig](inputs, CodaResumeConfig)++ def source_for_pipeline(+ self,+ config: CodaSourceConfig,+ resumable_source_manager: ResumableSourceManager[CodaResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return coda_source(api_token=config.api_token,endpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,)diff --git a/products/warehouse_sources/sources/coda/tests/test_coda.py b/products/warehouse_sources/sources/coda/tests/test_coda.pyindex 44138777b012..6b2c91855683 100644--- a/products/warehouse_sources/sources/coda/tests/test_coda.py+++ b/products/warehouse_sources/sources/coda/tests/test_coda.py@@ -1,9 +1,10 @@from typing import Anyimport pytest+from unittest.mock import MagicMockfrom products.warehouse_sources.sources.coda._config import CodaSourceConfig-from products.warehouse_sources.sources.coda.coda import coda_source, validate_credentials+from products.warehouse_sources.sources.coda.coda import CodaResumeConfig, coda_source, validate_credentialsfrom products.warehouse_sources.sources.coda.settings import CODA_ENDPOINTS, ENDPOINTSfrom products.warehouse_sources.sources.coda.source import CodaSourcefrom products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network@@ -161,6 +162,55 @@ class TestGetRows:key_column, parent_id = keyassert [row[key_column] for row in result.rows] == [parent_id]+ @pytest.mark.parametrize(+ "endpoint, saved, responses, expected_paths, expected_tokens, expected_ids",+ [+ (+ "docs",+ {"cursor": "token-2"},+ [_response([{"id": "doc2"}])],+ ["/apis/v1/docs"],+ ["token-2"],+ ["doc2"],+ ),+ (+ "pages",+ {"completed": ["/docs/doc1/pages"], "current": None, "child_state": None},+ [_response([{"id": "doc1"}, {"id": "doc2"}]), _response([{"id": "canvas-2"}])],+ ["/apis/v1/docs", "/apis/v1/docs/doc2/pages"],+ [None, None],+ ["canvas-2"],+ ),+ ],+ )+ def test_resumes_from_saved_state(+ self,+ endpoint: str,+ saved: dict[str, Any],+ responses: list[ScriptedResponse],+ expected_paths: list[str],+ expected_tokens: list[str | None],+ expected_ids: list[str],+ ) -> None:+ result = _driver().run(endpoint, responses, resume_state=CodaResumeConfig(paginator_state=saved))++ assert result.raised is None+ assert result.paths == expected_paths+ assert result.params("pageToken") == expected_tokens+ assert [row["id"] for row in result.rows] == expected_ids++ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:+ result = _driver().run("docs", [_response([{"id": "doc1"}])])++ assert result.raised is None+ assert result.saved_states[-1] == CodaResumeConfig(completed=True)++ resumed = _driver().run("docs", [], resume_state=CodaResumeConfig(completed=True))++ assert resumed.raised is None+ assert resumed.paths == []+ assert resumed.rows == []+def test_table_without_id_fails_fast_in_rows(self) -> None:result = _driver().run("rows", [_response([{"id": "doc1"}]), _response([{"name": "broken"}])])@@ -174,7 +224,9 @@ class TestCodaSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = CODA_ENDPOINTS[endpoint]- response = coda_source("token", endpoint, team_id=1, job_id="j")+ manager = MagicMock()+ manager.can_resume.return_value = False+ response = coda_source("token", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)assert response.name == endpointassert response.primary_keys == config.primary_keysdiff --git a/products/warehouse_sources/sources/groq/groq.py b/products/warehouse_sources/sources/groq/groq.pyindex 9d1fcf24745f..85ec29ed7e76 100644--- a/products/warehouse_sources/sources/groq/groq.py+++ b/products/warehouse_sources/sources/groq/groq.py@@ -1,10 +1,15 @@+from collections.abc import Iterable+from typing import Any+from products.warehouse_sources.sources.groq.settings import GROQ_ENDPOINTS, GroqEndpointConfigfrom products.warehouse_sources.sources.sdk import (BasePaginator,JSONResponseCursorPaginator,RESTAPIConfig,+ ResumableSourceManager,SinglePagePaginator,SourceResponse,+ frozen,make_tracked_session,rest_api_resource,validate_via_probe,@@ -13,6 +18,14 @@ from products.warehouse_sources.sources.sdk import (GROQ_BASE_URL = "https://api.groq.com/openai/v1"+@frozen+class GroqResumeConfig:+ # The `paging.next_cursor` of the last page handed on. Only `batches` paginates.+ cursor: str | None = None+ # The last page is handed on, so a resume reads nothing again.+ completed: bool = False++def _get_headers(api_key: str) -> dict[str, str]:return {"Authorization": f"Bearer {api_key}",@@ -34,6 +47,7 @@ def groq_source(endpoint: str,team_id: int,job_id: str,+ resumable_source_manager: ResumableSourceManager[GroqResumeConfig],) -> SourceResponse:config: GroqEndpointConfig = GROQ_ENDPOINTS[endpoint]@@ -63,11 +77,31 @@ def groq_source(],}- resource = rest_api_resource(rest_config, team_id, job_id, None)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state and state.get("cursor"):+ resumable_source_manager.save_state(GroqResumeConfig(cursor=str(state["cursor"])))+ elif state is None:+ resumable_source_manager.save_state(GroqResumeConfig(completed=True))++ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state={"cursor": resume.cursor} if resume is not None and resume.cursor else None,+ )+ completed = resume is not None and resume.completed++ def items() -> Iterable[Any]:+ return [] if completed else resourcereturn SourceResponse(name=endpoint,- items=lambda: resource,+ on_complete=resumable_source_manager.clear_state,+ items=items,primary_keys=config.primary_keys,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/sources/groq/source.py b/products/warehouse_sources/sources/groq/source.pyindex bea39185cdff..99007d6577bb 100644--- a/products/warehouse_sources/sources/groq/source.py+++ b/products/warehouse_sources/sources/groq/source.py@@ -2,6 +2,7 @@ from typing import Optional, castfrom products.warehouse_sources.sources.groq._config import GroqSourceConfigfrom products.warehouse_sources.sources.groq.groq import (+ GroqResumeConfig,groq_source,validate_credentials as validate_groq_credentials,)@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -25,7 +27,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class GroqSource(SimpleSource[GroqSourceConfig]):+class GroqSource(ResumableSource[GroqSourceConfig, GroqResumeConfig]):codebase_detection = detect(env=("^GROQ_",), npm=("groq-sdk",), pypi=("groq",))lists_tables_without_credentials = True # static endpoint catalog — safe for public docs@@ -120,10 +122,19 @@ Create an API key in the [Groq console](https://console.groq.com/keys). Groq expreturn False, "Your Groq API key is missing the permissions needed to sync this data"return False, "Could not connect to Groq with the provided API key"- def source_for_pipeline(self, config: GroqSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[GroqResumeConfig]:+ return ResumableSourceManager[GroqResumeConfig](inputs, GroqResumeConfig)++ def source_for_pipeline(+ self,+ config: GroqSourceConfig,+ resumable_source_manager: ResumableSourceManager[GroqResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return groq_source(api_key=config.api_key,endpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,)diff --git a/products/warehouse_sources/sources/groq/tests/test_groq.py b/products/warehouse_sources/sources/groq/tests/test_groq.pyindex 715c8d52bfcb..3ce5c394b842 100644--- a/products/warehouse_sources/sources/groq/tests/test_groq.py+++ b/products/warehouse_sources/sources/groq/tests/test_groq.py@@ -1,8 +1,10 @@+from unittest.mock import MagicMock+import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.groq._config import GroqSourceConfig-from products.warehouse_sources.sources.groq.groq import groq_source, validate_credentials+from products.warehouse_sources.sources.groq.groq import GroqResumeConfig, groq_source, validate_credentialsfrom products.warehouse_sources.sources.groq.source import GroqSourcefrom products.warehouse_sources.sources.sdk import RESTClientRetryableErrorfrom products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network@@ -26,6 +28,41 @@ class TestGroq:assert [row["id"] for row in result.rows] == ["a", "b"]assert result.paths == [f"/openai/v1/{endpoint}"]+ @parameterized.expand(+ [+ (+ "fresh_run",+ None,+ [None, "c2"],+ ["a", "b"],+ [GroqResumeConfig(cursor="c2"), GroqResumeConfig(completed=True)],+ ),+ ("resumed_run", GroqResumeConfig(cursor="c2"), ["c2"], ["b"], [GroqResumeConfig(completed=True)]),+ # A hand-off after the last page must not read that page again.+ ("completed_run", GroqResumeConfig(completed=True), [], [], []),+ ]+ )+ def test_batches_resume_from_saved_cursor(+ self,+ _name: str,+ resume_state: GroqResumeConfig | None,+ expected_cursors: list[str | None],+ expected_ids: list[str],+ expected_saved: list[GroqResumeConfig],+ ) -> None:+ pages = {+ None: ScriptedResponse(json={"data": [{"id": "a"}], "paging": {"next_cursor": "c2"}}),+ "c2": ScriptedResponse(json={"data": [{"id": "b"}], "paging": {}}),+ }+ result = self._driver().run(+ "batches", lambda request: pages[request.param("cursor")], resume_state=resume_state+ )++ assert result.raised is None+ assert result.params("cursor") == expected_cursors+ assert [row["id"] for row in result.rows] == expected_ids+ assert result.saved_states == expected_saved+def test_non_dict_body_is_retried_then_reraises(self) -> None:# A non-object body (bare array) is an unexpected shape; the request is reissued and, if it# never recovers, surfaces as a retryable error after the attempt cap.@@ -72,7 +109,9 @@ class TestGroq:)def test_groq_source_maps_primary_keys_and_partitioning(self, endpoint: str, partition_key: str) -> None:# No network: SourceResponse metadata is built eagerly, rows only on iteration.- response = groq_source("gsk_k", endpoint, team_id=1, job_id="j")+ manager = MagicMock()+ manager.can_resume.return_value = False+ response = groq_source("gsk_k", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)assert response.name == endpointassert response.primary_keys == ["id"]assert response.partition_mode == "datetime"diff --git a/products/warehouse_sources/sources/kandji/kandji.py b/products/warehouse_sources/sources/kandji/kandji.pyindex b57dbb4dec6d..2733f5a05996 100644--- a/products/warehouse_sources/sources/kandji/kandji.py+++ b/products/warehouse_sources/sources/kandji/kandji.py@@ -19,13 +19,25 @@ from products.warehouse_sources.sources.sdk import (EndpointResource,OffsetPaginator,RESTAPIConfig,+ ResumableSourceManager,SinglePagePaginator,SourceResponse,build_dependent_resource,+ frozen,make_tracked_session,rest_api_resource,)++@frozen+class KandjiResumeConfig:+ # What the framework handed the resume hook: the next offset or `next` query of a list endpoint,+ # or the device listing position and finished devices of a fan-out. Stored unchanged.+ paginator_state: dict[str, Any] | None = None+ # The last page of a list endpoint is handed on, so a resume reads nothing again.+ completed: bool = False++REGION_TEMPLATES = {"us": US_API_HOST_TEMPLATE,"eu": EU_API_HOST_TEMPLATE,@@ -87,10 +99,23 @@ class KandjiNextLinkPaginator(BasePaginator):self._next_params = next_paramsself._has_next_page = True+ def init_request(self, request: Request) -> None:+ # Apply a resumed `next` query to the first request.+ self.update_request(request)+def update_request(self, request: Request) -> None:if self._next_params is not None:request.params = {**(request.params or {}), **self._next_params}+ def get_resume_state(self) -> Optional[dict[str, Any]]:+ return {"next_params": self._next_params} if self._has_next_page and self._next_params else None++ def set_resume_state(self, state: dict[str, Any]) -> None:+ next_params = state.get("next_params")+ if isinstance(next_params, dict) and next_params:+ self._next_params = {str(key): str(value) for key, value in next_params.items()}+ self._has_next_page = True+def _list_paginator(config: KandjiEndpointConfig) -> BasePaginator:if config.pagination == "single":@@ -157,9 +182,12 @@ def get_resource(config: KandjiEndpointConfig) -> EndpointResource:}-def _make_source_response(config: KandjiEndpointConfig, items_fn) -> SourceResponse:+def _make_source_response(+ config: KandjiEndpointConfig, items_fn, manager: ResumableSourceManager[KandjiResumeConfig]+) -> SourceResponse:return SourceResponse(name=config.name,+ on_complete=manager.clear_state,items=items_fn,primary_keys=config.primary_key if isinstance(config.primary_key, list) else [config.primary_key],# Full refresh only — Kandji's list endpoints expose no stable updated-since cursor.@@ -174,11 +202,21 @@ def kandji_source(endpoint: str,team_id: int,job_id: str,+ resumable_source_manager: ResumableSourceManager[KandjiResumeConfig],) -> SourceResponse:config = KANDJI_ENDPOINTS[endpoint]base_url = build_base_url(subdomain, region)client_config = _rest_api_client_config(base_url, api_token)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ initial_paginator_state = resume.paginator_state if resume is not None else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state:+ resumable_source_manager.save_state(KandjiResumeConfig(paginator_state=state))+ elif state is None:+ resumable_source_manager.save_state(KandjiResumeConfig(completed=True))+if config.fanout is not None:parent_config = KANDJI_ENDPOINTS[config.fanout.parent_name]dependent_resource = cast(@@ -202,14 +240,25 @@ def kandji_source("paginator": _list_paginator(config),"data_selector": config.data_selector,},+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ resume_parent_pagination=True,),)- return _make_source_response(config, lambda: dependent_resource)+ return _make_source_response(config, lambda: dependent_resource, resumable_source_manager)rest_config: RESTAPIConfig = {"client": client_config,"resource_defaults": {"write_disposition": "replace"},"resources": [get_resource(config)],}- resource = rest_api_resource(rest_config, team_id, job_id, None)- return _make_source_response(config, lambda: resource)+ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state=initial_paginator_state,+ )+ completed = resume is not None and resume.completed+ return _make_source_response(config, lambda: [] if completed else resource, resumable_source_manager)diff --git a/products/warehouse_sources/sources/kandji/source.py b/products/warehouse_sources/sources/kandji/source.pyindex 334ee40e9480..1edf9ee4f0c0 100644--- a/products/warehouse_sources/sources/kandji/source.py+++ b/products/warehouse_sources/sources/kandji/source.py@@ -2,6 +2,7 @@ from typing import Optional, castfrom products.warehouse_sources.sources.kandji._config import KandjiSourceConfigfrom products.warehouse_sources.sources.kandji.kandji import (+ KandjiResumeConfig,kandji_source,validate_credentials as validate_kandji_credentials,)@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -27,7 +29,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class KandjiSource(SimpleSource[KandjiSourceConfig]):+class KandjiSource(ResumableSource[KandjiSourceConfig, KandjiResumeConfig]):codebase_detection = detect(env=("^KANDJI_",))# `get_schemas` iterates a static endpoint catalog with no I/O, so the table list is safe to@@ -95,7 +97,15 @@ class KandjiSource(SimpleSource[KandjiSourceConfig]):schema_name=schema_name,)- def source_for_pipeline(self, config: KandjiSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KandjiResumeConfig]:+ return ResumableSourceManager[KandjiResumeConfig](inputs, KandjiResumeConfig)++ def source_for_pipeline(+ self,+ config: KandjiSourceConfig,+ resumable_source_manager: ResumableSourceManager[KandjiResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return kandji_source(api_token=config.api_token,subdomain=config.subdomain,@@ -103,6 +113,7 @@ class KandjiSource(SimpleSource[KandjiSourceConfig]):endpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,)@propertydiff --git a/products/warehouse_sources/sources/kandji/tests/test_kandji.py b/products/warehouse_sources/sources/kandji/tests/test_kandji.pyindex 0a40d9e340c3..8d2eeb9be8fc 100644--- a/products/warehouse_sources/sources/kandji/tests/test_kandji.py+++ b/products/warehouse_sources/sources/kandji/tests/test_kandji.py@@ -1,3 +1,5 @@+from typing import Any+import pytestfrom unittest.mock import Mock, patch@@ -7,6 +9,7 @@ from parameterized import parameterizedfrom products.warehouse_sources.sources.kandji._config import KandjiSourceConfigfrom products.warehouse_sources.sources.kandji.kandji import (KandjiNextLinkPaginator,+ KandjiResumeConfig,build_base_url,kandji_source,validate_credentials,@@ -93,6 +96,7 @@ class TestKandjiTransport:endpoint="devices",team_id=1,job_id="job-1",+ resumable_source_manager=Mock(can_resume=Mock(return_value=False)),)assert response.name == "devices"@@ -118,6 +122,67 @@ class TestKandjiTransport:assert result.response is not Noneassert result.response.primary_keys == ["device_id", "bundle_id"]+ @parameterized.expand(+ [+ (+ "offset_list",+ "devices",+ {"offset": 300},+ [ScriptedResponse(json=[{"device_id": "dev_301"}])],+ ["https://accuhive.api.kandji.io/api/v1/devices?offset=300&limit=300"],+ ),+ (+ "next_link_list",+ "users",+ {"next_params": {"cursor": "abc"}},+ [ScriptedResponse(json={"results": [{"id": "u2"}], "next": None})],+ ["https://accuhive.api.kandji.io/api/v1/users?sizePerPage=300&cursor=abc"],+ ),+ (+ "fan_out",+ "device_apps",+ {"completed": ["/devices/dev_1/apps"], "current": None, "child_state": None, "parent": None},+ [+ ScriptedResponse(json=[{"device_id": "dev_1"}, {"device_id": "dev_2"}]),+ ScriptedResponse(json={"apps": [{"bundle_id": "com.apple.Safari"}]}),+ ],+ [+ "https://accuhive.api.kandji.io/api/v1/devices?limit=300&offset=0",+ "https://accuhive.api.kandji.io/api/v1/devices/dev_2/apps?limit=300",+ ],+ ),+ ]+ )+ def test_resumes_from_saved_state(+ self,+ _name: str,+ endpoint: str,+ saved: dict[str, Any],+ script: list[ScriptedResponse],+ expected_urls: list[str],+ ) -> None:+ result = SourceDriver(+ KandjiSource(), KandjiSourceConfig(api_token="tok", subdomain="accuhive", region="us")+ ).run(endpoint, script, resume_state=KandjiResumeConfig(paginator_state=saved))++ assert result.raised is None+ assert result.urls == expected_urls+ assert len(result.rows) == 1++ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:+ driver = SourceDriver(KandjiSource(), KandjiSourceConfig(api_token="tok", subdomain="accuhive", region="us"))++ result = driver.run("users", [ScriptedResponse(json={"results": [{"id": "u1"}], "next": None})])++ assert result.raised is None+ assert result.saved_states[-1] == KandjiResumeConfig(completed=True)++ resumed = driver.run("users", [], resume_state=KandjiResumeConfig(completed=True))++ assert resumed.raised is None+ assert resumed.urls == []+ assert resumed.rows == []+class TestKandjiNextLinkPaginator:def test_copies_next_query_onto_original_request(self) -> None:diff --git a/products/warehouse_sources/sources/kernel/kernel.py b/products/warehouse_sources/sources/kernel/kernel.pyindex 527df5ee08e0..83b089ef84a0 100644--- a/products/warehouse_sources/sources/kernel/kernel.py+++ b/products/warehouse_sources/sources/kernel/kernel.py@@ -19,8 +19,13 @@ from products.warehouse_sources.sources.kernel.settings import (TELEMETRY_RETENTION_DAYS,KernelEndpointConfig,)-from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline-from products.warehouse_sources.sources.sdk.internals import Batcher+from products.warehouse_sources.sources.sdk import (+ ResumableSourceManager,+ SourceResponse,+ frozen,+ make_tracked_session,+ retry_with_deadline,+)KERNEL_BASE_URL = "https://api.onkernel.com"@@ -33,6 +38,20 @@ AUDIT_LOG_MIN_WINDOW = timedelta(hours=1)AUDIT_LOG_RETENTION = timedelta(days=365)+@frozen+class KernelResumeConfig:+ # List endpoints: the offset of the next page.+ offset: int | None = None+ # audit_logs: the start of the next time window, as an ISO timestamp.+ audit_log_start: str | None = None+ # browser_telemetry_events: the browser session to read next, and the offset cursor of its next+ # page. Sessions are read in id order, and a None cursor starts the session at its first event.+ telemetry_session_id: str | None = None+ telemetry_cursor: int | None = None+ # List and telemetry endpoints: the last page is handed on, so a resume reads nothing again.+ completed: bool = False++class KernelRetryableError(Exception):pass@@ -228,8 +247,14 @@ def _fetch_audit_log_page(def _iter_audit_log_windows(- session: requests.Session, headers: dict[str, str], start: datetime, end: datetime, logger: FilteringBoundLogger-) -> Iterator[list[dict[str, Any]]]:+ session: requests.Session,+ headers: dict[str, str],+ start: datetime,+ end: datetime,+ logger: FilteringBoundLogger,+ manager: ResumableSourceManager[KernelResumeConfig],+) -> Iterator[tuple[datetime, list[dict[str, Any]]]]:+ """Yield the end of each window with its rows, oldest window first. A window with no rows is yielded too."""windows: list[tuple[datetime, datetime]] = []window_start = startwhile window_start < end:@@ -243,52 +268,57 @@ def _iter_audit_log_windows(midpoint = window_start + (window_end - window_start) / 2pending.append((midpoint, window_end))pending.append((window_start, midpoint))+ # The rows of the window are dropped and read again in the halves, so none is held.+ manager.safe_point()continuewhile next_token is not None:page, next_token = _fetch_audit_log_page(session, headers, window_start, window_end, next_token, logger)rows.extend(page)- if rows:- rows.sort(key=lambda row: _parse_datetime(row.get("timestamp")) or datetime.min.replace(tzinfo=UTC))- yield _add_audit_log_ids(rows)+ rows.sort(key=lambda row: _parse_datetime(row.get("timestamp")) or datetime.min.replace(tzinfo=UTC))+ yield window_end, _add_audit_log_ids(rows)def get_audit_log_rows(- api_key: str, logger: FilteringBoundLogger, db_incremental_field_last_value: Optional[Any] = None+ api_key: str,+ logger: FilteringBoundLogger,+ manager: ResumableSourceManager[KernelResumeConfig],+ db_incremental_field_last_value: Optional[Any] = None,) -> Iterator[Any]:headers = _get_headers(api_key)- batcher = Batcher(logger=logger, chunk_size=2000, chunk_size_bytes=100 * 1024 * 1024)session = make_tracked_session(capture=False)end = datetime.now(UTC)start = end - AUDIT_LOG_RETENTIONlast_value = _parse_datetime(db_incremental_field_last_value)if last_value is not None and last_value > start:start = last_value- for rows in _iter_audit_log_windows(session, headers, start, end, logger):- for row in rows:- batcher.batch(row)- if batcher.should_yield():- yield batcher.get_table()- if batcher.should_yield(include_incomplete_chunk=True):- yield batcher.get_table()+ resume = manager.load_state() if manager.can_resume() else None+ resume_start = _parse_datetime(resume.audit_log_start) if resume is not None else None+ if resume_start is not None and resume_start > start:+ start = resume_start+ for window_end, rows in _iter_audit_log_windows(session, headers, start, end, logger, manager):+ manager.save_state(KernelResumeConfig(audit_log_start=_format_datetime(window_end)))+ if rows:+ # One item per window, so a window and the cursor past it commit together.+ yield rows+ else:+ manager.safe_point()-def _iter_list_items(+def _iter_list_pages(session: requests.Session,headers: dict[str, str],config: KernelEndpointConfig,logger: FilteringBoundLogger,-) -> Iterator[Any]:- # Full refresh only: no resumable offset state. A crashed sync restarts from offset 0 and- # the pipeline overwrites the table on the first chunk, so re-fetched pages never duplicate- # rows (full-refresh appends have no primary-key dedupe, so resuming mid-table would).- offset = 0+ offset: int = 0,+) -> Iterator[tuple[list[Any], int | None]]:+ """Yield each page with the offset of the page after it, or None after the last page."""while True:params: dict[str, Any] = {"limit": PAGE_SIZE, "offset": offset, **config.extra_params}url = _build_url(config.path, params)try:response = _fetch_page(session, url, headers, logger, empty_on_error_code=config.empty_on_error_code)except KernelResourceDisabledError:- break+ returnitems = _extract_items(response.json())has_more, next_offset = _next_page(response.headers, offset, len(items))@@ -298,14 +328,15 @@ def _iter_list_items(# via X-Has-More. Keep going, but stop if the offset can't advance (an empty page# with no X-Next-Offset would otherwise loop forever on the same request).if not has_more or next_offset <= offset:- break+ return+ yield [], next_offsetoffset = next_offsetcontinue- yield from items+ yield items, next_offset if has_more else Noneif not has_more:- break+ returnoffset = next_offset@@ -325,24 +356,29 @@ def _telemetry_session_ids(headers: dict[str, str],retention_start: datetime,logger: FilteringBoundLogger,+ manager: ResumableSourceManager[KernelResumeConfig],) -> list[str]:- """Session ids of every browser that can still have retained telemetry events.+ """Session ids of every browser that can still have retained telemetry events, in id order.Ids are collected before fanning out so the offset-paginated /browsers walk isn't stretched- across thousands of child requests while new sessions shift its pages.+ across thousands of child requests while new sessions shift its pages. The id order is the+ order a resume relies on."""session_ids: list[str] = []- for browser in _iter_list_items(session, headers, KERNEL_ENDPOINTS["browsers"], logger):- if not isinstance(browser, dict):- continue- session_id = browser.get("session_id")- if not isinstance(session_id, str) or not session_id:- continue- deleted_at = _parse_timestamp(browser.get("deleted_at"))- if deleted_at is not None and deleted_at < retention_start:- continue- session_ids.append(session_id)- return session_ids+ for browsers, _next_offset in _iter_list_pages(session, headers, KERNEL_ENDPOINTS["browsers"], logger):+ for browser in browsers:+ if not isinstance(browser, dict):+ continue+ session_id = browser.get("session_id")+ if not isinstance(session_id, str) or not session_id:+ continue+ deleted_at = _parse_timestamp(browser.get("deleted_at"))+ if deleted_at is not None and deleted_at < retention_start:+ continue+ session_ids.append(session_id)+ # No row is handed on while the ids are collected, so the run can stop here.+ manager.safe_point()+ return sorted(set(session_ids))def _next_telemetry_cursor(headers: Any) -> int | None:@@ -371,15 +407,16 @@ def _telemetry_row(session_id: str, envelope: dict[str, Any]) -> dict[str, Any]:return row-def _iter_telemetry_events(+def _iter_telemetry_pages(session: requests.Session,headers: dict[str, str],session_id: str,since: str,logger: FilteringBoundLogger,-) -> Iterator[dict[str, Any]]:+ cursor: int | None,+) -> Iterator[tuple[list[dict[str, Any]], int | None]]:+ """Yield each page of a session's events with the cursor of the page after it, or None after the last."""path = KERNEL_ENDPOINTS[BROWSER_TELEMETRY_EVENTS].path.format(session_id=quote(session_id, safe=""))- cursor: int | None = Nonewhile True:# `since` defaults to the last 5 minutes, so it must be set to read the archive.# Kernel ignores it once an offset cursor is passed.@@ -390,15 +427,19 @@ def _iter_telemetry_events(if response.status_code == 404:# The session can be purged between listing browsers and reading its events.logger.debug(f"Kernel browser session {session_id} not found, skipping its telemetry events")+ yield [], Nonereturn- for envelope in _extract_items(response.json()):- if isinstance(envelope, dict):- yield _telemetry_row(session_id, envelope)-+ rows = [+ _telemetry_row(session_id, envelope)+ for envelope in _extract_items(response.json())+ if isinstance(envelope, dict)+ ]next_cursor = _next_telemetry_cursor(response.headers)if next_cursor is None or next_cursor == cursor:+ yield rows, Nonereturn+ yield rows, next_cursorcursor = next_cursor@@ -406,47 +447,70 @@ def _iter_browser_telemetry_events(session: requests.Session,headers: dict[str, str],logger: FilteringBoundLogger,-) -> Iterator[dict[str, Any]]:+ manager: ResumableSourceManager[KernelResumeConfig],+) -> Iterator[list[dict[str, Any]]]:retention_start = datetime.now(UTC) - timedelta(days=TELEMETRY_RETENTION_DAYS)since = retention_start.strftime("%Y-%m-%dT%H:%M:%SZ")- for session_id in _telemetry_session_ids(session, headers, retention_start, logger):- yield from _iter_telemetry_events(session, headers, session_id, since, logger)+ resume = manager.load_state() if manager.can_resume() else None+ if resume is not None and resume.completed:+ return+ resume_session_id = resume.telemetry_session_id if resume is not None else None+ session_ids = _telemetry_session_ids(session, headers, retention_start, logger, manager)+ for index, session_id in enumerate(session_ids):+ if resume_session_id is not None and session_id < resume_session_id:+ continue+ cursor = resume.telemetry_cursor if resume is not None and session_id == resume_session_id else None+ following = session_ids[index + 1] if index + 1 < len(session_ids) else None+ for rows, next_cursor in _iter_telemetry_pages(session, headers, session_id, since, logger, cursor):+ if next_cursor is not None:+ manager.save_state(KernelResumeConfig(telemetry_session_id=session_id, telemetry_cursor=next_cursor))+ elif following is not None:+ manager.save_state(KernelResumeConfig(telemetry_session_id=following))+ else:+ manager.save_state(KernelResumeConfig(completed=True))+ if rows:+ yield rows+ else:+ manager.safe_point()def get_rows(api_key: str,endpoint: str,logger: FilteringBoundLogger,+ manager: ResumableSourceManager[KernelResumeConfig],) -> Iterator[Any]:config = KERNEL_ENDPOINTS[endpoint]headers = _get_headers(api_key)- batcher = Batcher(logger=logger, chunk_size=2000, chunk_size_bytes=100 * 1024 * 1024)# One session reused across every page so urllib3 keeps the connection alive.# capture=False: Kernel responses carry secret-bearing fields (see SENSITIVE_FIELDS) that the# generic HTTP-sample scrubber does not know to redact, and sampling happens before redaction.session = make_tracked_session(capture=False)+ # Each page goes out as its own item, so the cursor staged for a page covers every row handed on.if endpoint == BROWSER_TELEMETRY_EVENTS:- items: Iterator[Any] = _iter_browser_telemetry_events(session, headers, logger)- else:- items = (- _redact_sensitive_fields(item, config.nested_sensitive_fields)- for item in _iter_list_items(session, headers, config, logger)+ yield from _iter_browser_telemetry_events(session, headers, logger, manager)+ return++ resume = manager.load_state() if manager.can_resume() else None+ if resume is not None and resume.completed:+ return+ offset = resume.offset if resume is not None and resume.offset is not None else 0+ for items, next_offset in _iter_list_pages(session, headers, config, logger, offset):+ manager.save_state(+ KernelResumeConfig(offset=next_offset) if next_offset is not None else KernelResumeConfig(completed=True))-- for item in items:- batcher.batch(item)- if batcher.should_yield():- yield batcher.get_table()-- if batcher.should_yield(include_incomplete_chunk=True):- yield batcher.get_table()+ if items:+ yield [_redact_sensitive_fields(item, config.nested_sensitive_fields) for item in items]+ else:+ manager.safe_point()def kernel_source(api_key: str,endpoint: str,logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[KernelResumeConfig],db_incremental_field_last_value: Optional[Any] = None,) -> SourceResponse:endpoint_config: KernelEndpointConfig = KERNEL_ENDPOINTS[endpoint]@@ -455,13 +519,15 @@ def kernel_source(get_audit_log_rows,api_key=api_key,logger=logger,+ manager=resumable_source_manager,db_incremental_field_last_value=db_incremental_field_last_value,)if endpoint == AUDIT_LOGS_ENDPOINT- else partial(get_rows, api_key=api_key, endpoint=endpoint, logger=logger)+ else partial(get_rows, api_key=api_key, endpoint=endpoint, logger=logger, manager=resumable_source_manager))return SourceResponse(name=endpoint,+ on_complete=resumable_source_manager.clear_state,items=items,primary_keys=endpoint_config.primary_keys,sort_mode="asc",diff --git a/products/warehouse_sources/sources/kernel/source.py b/products/warehouse_sources/sources/kernel/source.pyindex 853b1e8fe9eb..2ca17259d446 100644--- a/products/warehouse_sources/sources/kernel/source.py+++ b/products/warehouse_sources/sources/kernel/source.py@@ -2,6 +2,7 @@ from typing import Optional, castfrom products.warehouse_sources.sources.kernel._config import KernelSourceConfigfrom products.warehouse_sources.sources.kernel.kernel import (+ KernelResumeConfig,kernel_source,validate_credentials as validate_kernel_credentials,)@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -25,7 +27,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class KernelSource(SimpleSource[KernelSourceConfig]):+class KernelSource(ResumableSource[KernelSourceConfig, KernelResumeConfig]):codebase_detection = detect(env=("^KERNEL_API_KEY$", "^KERNEL_ORG_API_KEY$"),npm=("@onkernel/sdk",),@@ -128,11 +130,20 @@ You can create an API key in your [Kernel dashboard](https://dashboard.onkernel.return False, "Invalid Kernel API key"- def source_for_pipeline(self, config: KernelSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KernelResumeConfig]:+ return ResumableSourceManager[KernelResumeConfig](inputs, KernelResumeConfig)++ def source_for_pipeline(+ self,+ config: KernelSourceConfig,+ resumable_source_manager: ResumableSourceManager[KernelResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return kernel_source(api_key=config.api_key,endpoint=inputs.schema_name,logger=inputs.logger,+ resumable_source_manager=resumable_source_manager,db_incremental_field_last_value=(inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None),diff --git a/products/warehouse_sources/sources/kernel/tests/test_kernel.py b/products/warehouse_sources/sources/kernel/tests/test_kernel.pyindex 47048a0330c3..db5c6521e7c1 100644--- a/products/warehouse_sources/sources/kernel/tests/test_kernel.py+++ b/products/warehouse_sources/sources/kernel/tests/test_kernel.py@@ -1,4 +1,3 @@-import jsonfrom collections.abc import Callablefrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -13,6 +12,7 @@ from products.warehouse_sources.sources.kernel._config import KernelSourceConfigfrom products.warehouse_sources.sources.kernel.kernel import (AUDIT_LOG_MAX_WINDOW,PAGE_SIZE,+ KernelResumeConfig,KernelRetryableError,KernelUnexpectedResponseError,_extract_items,@@ -37,10 +37,6 @@ def _driver() -> SourceDriver:return SourceDriver(KernelSource(), KernelSourceConfig(api_key="sk_test"))-def _rows(result: DriveResult) -> list[dict[str, Any]]:- return [row for table in result.rows for row in table.to_pylist()]--def _response(items: Any, *, status_code: int = 200, has_more: bool | None = None, next_offset: int | None = None) -> ScriptedResponse:@@ -118,13 +114,16 @@ class TestGetRows:)assert result.raised is None- assert _rows(result) == [{"id": "a1"}, {"id": "a2"}]+ # Row lists, not tables, so the pipeline batches pages together.+ assert result.items == [[{"id": "a1"}], [{"id": "a2"}]]assert result.params("offset") == ["0", "100"]+ # The last page carries a terminal cursor, so a hand-off after it reads nothing again.+ assert result.committed_states == [KernelResumeConfig(offset=100), KernelResumeConfig(completed=True)]def test_empty_first_page_yields_nothing(self) -> None:result = _driver().run("profiles", [_response([], has_more=False)])assert result.raised is None- assert _rows(result) == []+ assert result.rows == []def test_empty_page_with_more_pages_keeps_paging(self) -> None:# An empty page that still reports X-Has-More must not end the sync early.@@ -136,7 +135,7 @@ class TestGetRows:],)assert result.raised is None- assert _rows(result) == [{"id": "a1"}]+ assert result.rows == [{"id": "a1"}]assert result.params("offset") == ["0", "100"]@pytest.mark.parametrize(@@ -150,13 +149,79 @@ class TestGetRows:assert isinstance(result.raised, requests.HTTPError)else:assert result.raised is None- assert _rows(result) == []+ assert result.rows == []def test_unexpected_response_shape_raises(self) -> None:result = _driver().run("apps", [_response({"unexpected": [{"id": "a1"}]}, has_more=False)])assert isinstance(result.raised, KernelUnexpectedResponseError)+ @pytest.mark.parametrize(+ "endpoint, saved, responses, expected_paths, expected_offsets, row_key, expected_keys",+ [+ pytest.param(+ "deployments",+ KernelResumeConfig(offset=100),+ {"/deployments": [_response([{"id": "a2"}], has_more=False)]},+ ["/deployments"],+ ["100"],+ "id",+ ["a2"],+ id="list_starts_at_saved_offset",+ ),+ pytest.param(+ "browser_telemetry_events",+ KernelResumeConfig(telemetry_session_id="b2", telemetry_cursor=987),+ {+ "/browsers": [_response([{"session_id": "b3"}, {"session_id": "b1"}, {"session_id": "b2"}])],+ "/browsers/b2/telemetry/events": [_response([{"seq": 2, "event": {}}], has_more=False)],+ "/browsers/b3/telemetry/events": [_response([{"seq": 3, "event": {}}], has_more=False)],+ },+ ["/browsers", "/browsers/b2/telemetry/events", "/browsers/b3/telemetry/events"],+ ["0", "987", None],+ "seq",+ [2, 3],+ id="telemetry_skips_finished_sessions_and_resumes_the_saved_one",+ ),+ pytest.param(+ "deployments",+ KernelResumeConfig(completed=True),+ {},+ [],+ [],+ "id",+ [],+ id="list_after_its_last_page_reads_nothing",+ ),+ pytest.param(+ "browser_telemetry_events",+ KernelResumeConfig(completed=True),+ {},+ [],+ [],+ "seq",+ [],+ id="telemetry_after_its_last_page_reads_nothing",+ ),+ ],+ )+ def test_resumes_from_saved_state(+ self,+ endpoint: str,+ saved: KernelResumeConfig,+ responses: dict[str, list[ScriptedResponse]],+ expected_paths: list[str],+ expected_offsets: list[str | None],+ row_key: str,+ expected_keys: list[Any],+ ) -> None:+ result = _driver().run(endpoint, route(responses), resume_state=saved)++ assert result.raised is None+ assert result.paths == expected_paths+ assert result.params("offset") == expected_offsets+ assert [row[row_key] for row in result.rows] == expected_keys+def test_retries_exhausted_raises(self) -> None:result = _driver().run("apps", always(_response(None, status_code=503)))@@ -205,7 +270,7 @@ class TestBrowserTelemetryEvents:)assert result.raised is None- assert [{**row, "data": json.loads(row["data"])} for row in _rows(result)] == [+ assert result.rows == [{"browser_session_id": "active","seq": 1,@@ -253,7 +318,7 @@ class TestBrowserTelemetryEvents:)assert result.raised is None- assert [row["seq"] for row in _rows(result)] == [1]+ assert [row["seq"] for row in result.rows] == [1]assert len(result.requests) == 2def test_skips_browser_purged_after_listing(self) -> None:@@ -267,14 +332,14 @@ class TestBrowserTelemetryEvents:)assert result.raised is None- assert [(row["browser_session_id"], row["seq"]) for row in _rows(result)] == [("b2", 7)]+ assert [(row["browser_session_id"], row["seq"]) for row in result.rows] == [("b2", 7)]class TestKernelSourceResponse:@pytest.mark.parametrize("endpoint", list(ENDPOINTS))def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = KERNEL_ENDPOINTS[endpoint]- response = kernel_source("sk_test", endpoint, mock.MagicMock())+ response = kernel_source("sk_test", endpoint, mock.MagicMock(), resumable_source_manager=mock.MagicMock())assert response.name == endpointassert response.primary_keys == config.primary_keys@@ -341,7 +406,7 @@ class TestAuditLogRows:_audit_record(_NOW - timedelta(hours=2), path="/apps"),]result = _driver().run("audit_logs", _fake_audit_log_api(records))- rows = _rows(result)+ rows = result.rowsassert result.raised is Noneassert len(rows) == len(records)@@ -366,23 +431,29 @@ class TestAuditLogRows:db_incremental_field_last_value=_NOW - timedelta(hours=2),)assert rerun.raised is None- assert {r["id"] for r in _rows(rerun)} == {r["id"] for r in rows[-2:]}+ assert {r["id"] for r in rerun.rows} == {r["id"] for r in rows[-2:]}@pytest.mark.parametrize(- "last_value, expected_start",+ "last_value, resume_start, expected_start",[- (None, _NOW - timedelta(days=365)),- (_NOW - timedelta(days=2), _NOW - timedelta(days=2)),- ("2026-05-30T00:00:00Z", datetime(2026, 5, 30, tzinfo=UTC)),+ (None, None, _NOW - timedelta(days=365)),+ (_NOW - timedelta(days=2), None, _NOW - timedelta(days=2)),+ ("2026-05-30T00:00:00Z", None, datetime(2026, 5, 30, tzinfo=UTC)),# Kernel keeps one year of audit logs, so an older watermark starts at the retention edge.- (_NOW - timedelta(days=500), _NOW - timedelta(days=365)),+ (_NOW - timedelta(days=500), None, _NOW - timedelta(days=365)),+ # A resumed run starts after the last window it handed on.+ (None, "2026-05-10T00:00:00.000000Z", datetime(2026, 5, 10, tzinfo=UTC)),+ (_NOW - timedelta(days=2), "2026-05-10T00:00:00.000000Z", _NOW - timedelta(days=2)),],)@time_machine.travel(_NOW, tick=False)- def test_search_starts_at_watermark_or_retention_edge(self, last_value: Any, expected_start: datetime) -> None:+ def test_search_starts_at_watermark_or_retention_edge(+ self, last_value: Any, resume_start: str | None, expected_start: datetime+ ) -> None:result = _driver().run("audit_logs",always(_response([], has_more=False)),+ resume_state=KernelResumeConfig(audit_log_start=resume_start) if resume_start else None,incremental_field="timestamp" if last_value is not None else None,db_incremental_field_last_value=last_value,)diff --git a/products/warehouse_sources/sources/my_hours/my_hours.py b/products/warehouse_sources/sources/my_hours/my_hours.pyindex eadbe73a3b53..e9b25083788e 100644--- a/products/warehouse_sources/sources/my_hours/my_hours.py+++ b/products/warehouse_sources/sources/my_hours/my_hours.py@@ -17,12 +17,12 @@ from products.warehouse_sources.sources.sdk import (Resource,RESTAPIConfig,RESTClient,+ ResumableSourceManager,SinglePagePaginator,SourceResponse,build_dependent_resource,frozen,make_tracked_session,- reach_safe_point,rest_api_resource,)@@ -63,6 +63,14 @@ def _client_config(api_key: str) -> ClientConfig:}+@frozen+class MyHoursResumeConfig:+ # time_logs: the first day of the next date window, as an ISO date.+ next_window_start: str | None = None+ # Fan-out endpoints: the finished and in-progress parents, as the framework hands them over.+ paginator_state: dict[str, Any] | None = None++@frozenclass _TimeLogWindow:date_from: date@@ -88,9 +96,17 @@ def _time_log_windows(start: date, today: date) -> Iterator[_TimeLogWindow]:start = end + timedelta(days=1)-def _time_log_pages(client: RESTClient, start: date) -> Iterator[list[dict[str, Any]]]:+def _time_log_pages(+ client: RESTClient, start: date, manager: ResumableSourceManager[MyHoursResumeConfig]+) -> Iterator[list[dict[str, Any]]]:+ resume = manager.load_state() if manager.can_resume() else None# One day past today in UTC covers accounts in timezones already on the next day.today = datetime.now(UTC).date() + timedelta(days=1)+ if resume is not None and resume.next_window_start:+ start = date.fromisoformat(resume.next_window_start)+ if start > today:+ # The last window is handed on. The window walk would clamp to today and read it again.+ returnfor window in _time_log_windows(start, today):rows: list[dict[str, Any]] = []for page in client.paginate(@@ -113,8 +129,11 @@ def _time_log_pages(client: RESTClient, start: date) -> Iterator[list[dict[str,data_selector_malformed_retryable=True,):rows.extend(page)+ # Each window is one request, so the next window is where a resume starts.+ next_start = window.date_to + timedelta(days=1)+ manager.save_state(MyHoursResumeConfig(next_window_start=next_start.isoformat()))if not rows:- reach_safe_point()+ manager.safe_point()continue# The report's row order is not documented. Sorting each window keeps the incremental# watermark ascending across batches.@@ -141,6 +160,7 @@ def my_hours_source(endpoint: str,team_id: int,job_id: str,+ resumable_source_manager: ResumableSourceManager[MyHoursResumeConfig],should_use_incremental_field: bool = False,db_incremental_field_last_value: Any = None,) -> SourceResponse:@@ -156,13 +176,20 @@ def my_hours_source(start = _start_date(db_incremental_field_last_value if should_use_incremental_field else None)return SourceResponse(name=endpoint,- items=lambda: _time_log_pages(client, start),+ on_complete=resumable_source_manager.clear_state,+ items=lambda: _time_log_pages(client, start, resumable_source_manager),primary_keys=config.primary_keys,partition_count=1,partition_size=1,)if config.fanout is not None:+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state:+ resumable_source_manager.save_state(MyHoursResumeConfig(paginator_state=state))+resource = cast(Resource,build_dependent_resource(@@ -175,12 +202,15 @@ def my_hours_source(job_id=job_id,db_incremental_field_last_value=None,page_size_param=None,+ resume_hook=save_checkpoint,+ initial_paginator_state=resume.paginator_state if resume is not None else None,),)if endpoint == PROJECT_TASKS_ENDPOINT:resource = resource.add_map(_flatten_task_lists)return SourceResponse(name=endpoint,+ on_complete=resumable_source_manager.clear_state,items=lambda: resource,primary_keys=config.primary_keys,partition_count=1,@@ -208,6 +238,7 @@ def my_hours_source(return SourceResponse(name=endpoint,+ on_complete=resumable_source_manager.clear_state,items=lambda: resource,primary_keys=config.primary_keys,partition_count=1,diff --git a/products/warehouse_sources/sources/my_hours/source.py b/products/warehouse_sources/sources/my_hours/source.pyindex 0b3c2b1ab011..35ff0102c133 100644--- a/products/warehouse_sources/sources/my_hours/source.py+++ b/products/warehouse_sources/sources/my_hours/source.py@@ -1,7 +1,7 @@from typing import Optional, castfrom products.warehouse_sources.sources.my_hours._config import MyHoursSourceConfig-from products.warehouse_sources.sources.my_hours.my_hours import check_access, my_hours_source+from products.warehouse_sources.sources.my_hours.my_hours import MyHoursResumeConfig, check_access, my_hours_sourcefrom products.warehouse_sources.sources.my_hours.settings import (ENDPOINTS,INCREMENTAL_FIELDS,@@ -18,7 +18,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -32,7 +33,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class MyHoursSource(SimpleSource[MyHoursSourceConfig]):+class MyHoursSource(ResumableSource[MyHoursSourceConfig, MyHoursResumeConfig]):codebase_detection = detect(env=("^(MY_HOURS|MYHOURS)_",))api_docs_url = "https://documenter.getpostman.com/view/8879268/TVmV4YYU"@@ -129,7 +130,15 @@ API keys are available on paid plans. Create one under **Settings → Integratioreturn False, "Invalid My Hours API key"return False, message or "Could not validate My Hours API key"- def source_for_pipeline(self, config: MyHoursSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[MyHoursResumeConfig]:+ return ResumableSourceManager[MyHoursResumeConfig](inputs, MyHoursResumeConfig)++ def source_for_pipeline(+ self,+ config: MyHoursSourceConfig,+ resumable_source_manager: ResumableSourceManager[MyHoursResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:if inputs.schema_name not in MY_HOURS_ENDPOINTS:raise ValueError(f"Unknown My Hours schema '{inputs.schema_name}'")@@ -138,6 +147,7 @@ API keys are available on paid plans. Create one under **Settings → Integratioendpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,should_use_incremental_field=inputs.should_use_incremental_field,db_incremental_field_last_value=inputs.db_incremental_field_last_value,)diff --git a/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py b/products/warehouse_sources/sources/my_hours/tests/test_my_hours.pyindex a72614f0b262..7214972b137d 100644--- a/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py+++ b/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py@@ -6,7 +6,12 @@ import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.my_hours._config import MyHoursSourceConfig-from products.warehouse_sources.sources.my_hours.my_hours import MY_HOURS_BASE_URL, MyHoursApiKeyAuth, check_access+from products.warehouse_sources.sources.my_hours.my_hours import (+ MY_HOURS_BASE_URL,+ MyHoursApiKeyAuth,+ MyHoursResumeConfig,+ check_access,+)from products.warehouse_sources.sources.my_hours.settings import ENDPOINTS, MY_HOURS_ENDPOINTS, TEAMS_UNAVAILABLE_ERRORfrom products.warehouse_sources.sources.my_hours.source import MyHoursSourcefrom products.warehouse_sources.sources.sdk import RESTClientRetryableError@@ -143,9 +148,12 @@ class TestMyHoursSourceResponse:class TestTimeLogs:@parameterized.expand([- ("full_refresh", False, None, [("2010-01-01", "2010-01-31"), ("2010-02-01", "2010-02-21")]),- ("incremental_from_watermark", True, datetime(2010, 2, 5), [("2010-02-05", "2010-02-21")]),- ("incremental_watermark_after_today", True, datetime(2010, 3, 10), [("2010-02-21", "2010-02-21")]),+ ("full_refresh", False, None, None, [("2010-01-01", "2010-01-31"), ("2010-02-01", "2010-02-21")]),+ ("incremental_from_watermark", True, datetime(2010, 2, 5), None, [("2010-02-05", "2010-02-21")]),+ ("incremental_watermark_after_today", True, datetime(2010, 3, 10), None, [("2010-02-21", "2010-02-21")]),+ ("resumed_full_refresh", False, None, "2010-02-01", [("2010-02-01", "2010-02-21")]),+ ("resumed_incremental", True, datetime(2010, 1, 5), "2010-02-04", [("2010-02-04", "2010-02-21")]),+ ("resumed_after_last_window", False, None, "2010-02-22", []),])@time_machine.travel("2010-02-20 12:00:00", tick=False)@@ -154,16 +162,26 @@ class TestTimeLogs:_name: str,incremental: bool,last_value: datetime | None,+ resume_from: str | None,expected_windows: list[tuple[str, str]],) -> None:inputs = {"incremental_field": "date", "db_incremental_field_last_value": last_value} if incremental else {}+ resume_state = MyHoursResumeConfig(next_window_start=resume_from) if resume_from else None- result = _driver().run("time_logs", [ScriptedResponse(json={"data": []}) for _ in expected_windows], **inputs)+ result = _driver().run(+ "time_logs",+ [ScriptedResponse(json={"data": []}) for _ in expected_windows],+ resume_state=resume_state,+ **inputs,+ )assert result.raised is Noneassert all(r.method == "PUT" and r.url == f"{MY_HOURS_BASE_URL}/Reports/activitydx" for r in result.requests)options = [r.json()["options"] for r in result.requests]assert [(o["dateFrom"], o["dateTo"]) for o in options] == expected_windows+ # Each window stages the start of the next one, so a resume never reads a finished window again.+ assert result.final_state == MyHoursResumeConfig(next_window_start="2010-02-22")+ assert result.completed_state is None@time_machine.travel("2010-02-20 12:00:00", tick=False)def test_yields_each_non_empty_window_sorted_by_date(self) -> None:diff --git a/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py b/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.pyindex aad273de2123..78dd6be90012 100644--- a/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py+++ b/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py@@ -34,9 +34,11 @@ class TestMyHoursSource:def test_source_for_pipeline_plumbs_arguments(self, mock_source: mock.MagicMock) -> None:inputs = _make_inputs(schema_name="clients")- self.source.source_for_pipeline(self.config, inputs)+ manager = mock.MagicMock()+ self.source.source_for_pipeline(self.config, manager, inputs)mock_source.assert_called_once()kwargs = mock_source.call_args.kwargsassert kwargs["api_key"] == "mh-key"assert kwargs["endpoint"] == "clients"+ assert kwargs["resumable_source_manager"] is managerdiff --git a/products/warehouse_sources/sources/optimizely/optimizely.py b/products/warehouse_sources/sources/optimizely/optimizely.pyindex 7145f3eb757a..d1d711569f30 100644--- a/products/warehouse_sources/sources/optimizely/optimizely.py+++ b/products/warehouse_sources/sources/optimizely/optimizely.py@@ -1,3 +1,6 @@+from collections.abc import Callable+from typing import Any+from products.warehouse_sources.sources.optimizely.settings import OPTIMIZELY_ENDPOINTS, OptimizelyEndpointConfigfrom products.warehouse_sources.sources.sdk import (ClientConfig,@@ -6,8 +9,10 @@ from products.warehouse_sources.sources.sdk import (Resource,ResponseAction,RESTAPIConfig,+ ResumableSourceManager,SinglePagePaginator,SourceResponse,+ frozen,make_tracked_session,rest_api_resource,rest_api_resources,@@ -19,6 +24,16 @@ OPTIMIZELY_BASE_URL = f"https://{OPTIMIZELY_API_HOST}/v2"# Optimizely list pages cap at 100 items.PAGE_SIZE = 100++@frozen+class OptimizelyResumeConfig:+ # What the framework handed the resume hook: the next `Link` URL of a list endpoint, or the+ # finished and in-progress projects of a project fan-out. Stored unchanged.+ paginator_state: dict[str, Any] | None = None+ # The last page of a list endpoint is handed on, so a resume reads nothing again.+ completed: bool = False++_SKIP_INACCESSIBLE: list[ResponseAction] = [{"status_code": 400, "action": "ignore"},{"status_code": 403, "action": "ignore"},@@ -42,7 +57,12 @@ def _client_config(api_token: str) -> ClientConfig:def _simple_source(- api_token: str, endpoint: str, config: OptimizelyEndpointConfig, team_id: int, job_id: str+ api_token: str,+ endpoint: str,+ config: OptimizelyEndpointConfig,+ team_id: int,+ job_id: str,+ manager: ResumableSourceManager[OptimizelyResumeConfig],) -> Resource:rest_config: RESTAPIConfig = {"client": _client_config(api_token),@@ -59,7 +79,14 @@ def _simple_source(}],}- return rest_api_resource(rest_config, team_id, job_id, None)+ return rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=_resume_hook(manager),+ initial_paginator_state=_initial_state(manager),+ )def _project_scoped_resource(name: str, path: str) -> EndpointResource:@@ -107,7 +134,12 @@ def _per_parent_resource(endpoint: str, path: str, parent: str, parent_id_param:def _fan_out_source(- api_token: str, endpoint: str, config: OptimizelyEndpointConfig, team_id: int, job_id: str+ api_token: str,+ endpoint: str,+ config: OptimizelyEndpointConfig,+ team_id: int,+ job_id: str,+ manager: ResumableSourceManager[OptimizelyResumeConfig],) -> Resource:resources: list[str | EndpointResource] = [{@@ -129,22 +161,55 @@ def _fan_out_source("resource_defaults": {},"resources": resources,}- built = rest_api_resources(rest_config, team_id, job_id, None)+ # The framework resumes a fan-out with one dependent level only. The per-parent results tables+ # chain two levels, so they read every project again after a resume (the merge dedupes).+ built = rest_api_resources(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=_resume_hook(manager),+ initial_paginator_state=_initial_state(manager),+ )# Only the requested endpoint's rows are emitted; its ancestors are iterated internally to drive the fan-out.return next(resource for resource in built if resource.name == endpoint)-def optimizely_source(api_token: str, endpoint: str, team_id: int, job_id: str) -> SourceResponse:+def _initial_state(manager: ResumableSourceManager[OptimizelyResumeConfig]) -> dict[str, Any] | None:+ resume = manager.load_state() if manager.can_resume() else None+ return resume.paginator_state if resume is not None else None+++def _resume_hook(manager: ResumableSourceManager[OptimizelyResumeConfig]) -> Callable[[dict[str, Any] | None], None]:+ def save_checkpoint(state: dict[str, Any] | None) -> None:+ if state:+ manager.save_state(OptimizelyResumeConfig(paginator_state=state))+ elif state is None:+ manager.save_state(OptimizelyResumeConfig(completed=True))++ return save_checkpoint+++def optimizely_source(+ api_token: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[OptimizelyResumeConfig],+) -> SourceResponse:config = OPTIMIZELY_ENDPOINTS[endpoint]+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ completed = resume is not None and resume.completedif config.project_scoped:- resource = _fan_out_source(api_token, endpoint, config, team_id, job_id)+ resource = _fan_out_source(api_token, endpoint, config, team_id, job_id, resumable_source_manager)else:- resource = _simple_source(api_token, endpoint, config, team_id, job_id)+ resource = _simple_source(api_token, endpoint, config, team_id, job_id, resumable_source_manager)return SourceResponse(name=endpoint,- items=lambda: resource,+ on_complete=resumable_source_manager.clear_state,+ items=lambda: [] if completed else resource,primary_keys=[config.primary_key],partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/sources/optimizely/source.py b/products/warehouse_sources/sources/optimizely/source.pyindex 5647609f7c78..d444dbc49ee8 100644--- a/products/warehouse_sources/sources/optimizely/source.py+++ b/products/warehouse_sources/sources/optimizely/source.py@@ -2,6 +2,7 @@ from typing import Optional, castfrom products.warehouse_sources.sources.optimizely._config import OptimizelySourceConfigfrom products.warehouse_sources.sources.optimizely.optimizely import (+ OptimizelyResumeConfig,optimizely_source,validate_credentials as validate_optimizely_credentials,)@@ -13,7 +14,8 @@ from products.warehouse_sources.sources.sdk import (FieldType,ProductKey,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -26,7 +28,7 @@ from products.warehouse_sources.sources.sdk import (@SourceRegistry.register-class OptimizelySource(SimpleSource[OptimizelySourceConfig]):+class OptimizelySource(ResumableSource[OptimizelySourceConfig, OptimizelyResumeConfig]):codebase_detection = detect(env=("^OPTIMIZELY_", "^NEXT_PUBLIC_OPTIMIZELY_SDK_KEY$", "^EXPO_PUBLIC_OPTIMIZELY_SDK_KEY$"),script_hosts=("cdn.optimizely.com", "cdn-pci.optimizely.com", "optimizely-edge.com"),@@ -165,10 +167,19 @@ An admin can generate a personal access token in Optimizely under Account Settinreturn False, "Invalid Optimizely personal access token"- def source_for_pipeline(self, config: OptimizelySourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[OptimizelyResumeConfig]:+ return ResumableSourceManager[OptimizelyResumeConfig](inputs, OptimizelyResumeConfig)++ def source_for_pipeline(+ self,+ config: OptimizelySourceConfig,+ resumable_source_manager: ResumableSourceManager[OptimizelyResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return optimizely_source(api_token=config.api_token,endpoint=inputs.schema_name,team_id=inputs.team_id,job_id=inputs.job_id,+ resumable_source_manager=resumable_source_manager,)diff --git a/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py b/products/warehouse_sources/sources/optimizely/tests/test_optimizely.pyindex aa8239cd8594..6d558108f019 100644--- a/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py+++ b/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py@@ -1,10 +1,16 @@from typing import Any+from unittest.mock import MagicMock+import requestsfrom parameterized import parameterizedfrom products.warehouse_sources.sources.optimizely._config import OptimizelySourceConfig-from products.warehouse_sources.sources.optimizely.optimizely import optimizely_source, validate_credentials+from products.warehouse_sources.sources.optimizely.optimizely import (+ OptimizelyResumeConfig,+ optimizely_source,+ validate_credentials,+)from products.warehouse_sources.sources.optimizely.settings import ENDPOINTS, OPTIMIZELY_ENDPOINTSfrom products.warehouse_sources.sources.optimizely.source import OptimizelySourcefrom products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, route, scripted_network@@ -89,6 +95,61 @@ class TestProjectScopedFanOut:assert result.requests[1].param("project_id") == "11"+class TestResume:+ @parameterized.expand(+ [+ (+ "list_endpoint_starts_at_saved_next_link",+ "projects",+ {"next_url": "https://api.optimizely.com/v2/projects?page=2&per_page=100"},+ {"/v2/projects": [_response([{"id": 2}])]},+ ["https://api.optimizely.com/v2/projects?page=2&per_page=100"],+ [2],+ ),+ (+ "fan_out_skips_finished_projects",+ "campaigns",+ {"completed": ["/campaigns?project_id=11"], "current": None, "child_state": None},+ {+ "/v2/projects": [_response([{"id": 11}, {"id": 22}])],+ "/v2/campaigns": [_response([{"id": "camp-22"}])],+ },+ [+ "https://api.optimizely.com/v2/projects?per_page=100",+ "https://api.optimizely.com/v2/campaigns?project_id=22&per_page=100",+ ],+ ["camp-22"],+ ),+ ]+ )+ def test_resumes_from_saved_state(+ self,+ _name: str,+ endpoint: str,+ saved: dict[str, Any],+ responses: dict[str, list[ScriptedResponse]],+ expected_urls: list[str],+ expected_ids: list[Any],+ ) -> None:+ result = _driver().run(endpoint, route(responses), resume_state=OptimizelyResumeConfig(paginator_state=saved))++ assert result.raised is None+ assert result.urls == expected_urls+ assert [row["id"] for row in result.rows] == expected_ids++ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:+ result = _driver().run("projects", route({"/v2/projects": [_response([{"id": 1}])]}))++ assert result.raised is None+ assert result.saved_states[-1] == OptimizelyResumeConfig(completed=True)++ resumed = _driver().run("projects", route({}), resume_state=OptimizelyResumeConfig(completed=True))++ assert resumed.raised is None+ assert resumed.urls == []+ assert resumed.rows == []++class TestPerParentResults:@parameterized.expand([@@ -139,7 +200,9 @@ class TestOptimizelySourceResponse:@parameterized.expand([(endpoint,) for endpoint in ENDPOINTS])def test_response_metadata_per_endpoint(self, endpoint: str) -> None:config = OPTIMIZELY_ENDPOINTS[endpoint]- response = optimizely_source("token", endpoint, team_id=1, job_id="j")+ manager = MagicMock()+ manager.can_resume.return_value = False+ response = optimizely_source("token", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)assert response.name == endpointassert response.primary_keys == [config.primary_key]diff --git a/products/warehouse_sources/sources/zapier_supported_storage/source.py b/products/warehouse_sources/sources/zapier_supported_storage/source.pyindex 5ac5c186fd2f..bfb8723f020e 100644--- a/products/warehouse_sources/sources/zapier_supported_storage/source.py+++ b/products/warehouse_sources/sources/zapier_supported_storage/source.py@@ -7,7 +7,8 @@ from products.warehouse_sources.sources.sdk import (ExternalDataSourceType,FieldType,ReleaseStatus,- SimpleSource,+ ResumableSource,+ ResumableSourceManager,SourceConfig,SourceFieldInputConfig,SourceFieldInputConfigType,@@ -23,6 +24,7 @@ from products.warehouse_sources.sources.zapier_supported_storage.settings importZAPIER_SUPPORTED_STORAGE_ENDPOINTS,)from products.warehouse_sources.sources.zapier_supported_storage.zapier_supported_storage import (+ ZapierSupportedStorageResumeConfig,validate_credentials as validate_zapier_supported_storage_credentials,zapier_supported_storage_source,)@@ -37,7 +39,9 @@ _ENDPOINT_DESCRIPTIONS: dict[str, str] = {@SourceRegistry.register-class ZapierSupportedStorageSource(SimpleSource[ZapierSupportedStorageSourceConfig]):+class ZapierSupportedStorageSource(+ ResumableSource[ZapierSupportedStorageSourceConfig, ZapierSupportedStorageResumeConfig]+):codebase_detection = NOT_DETECTABLElists_tables_without_credentials = True # static endpoint catalog — safe for public docs@@ -127,9 +131,20 @@ Only full-refresh syncing is supported: the store has no timestamps, so every sy) -> tuple[bool, str | None]:return validate_zapier_supported_storage_credentials(config.secret)- def source_for_pipeline(self, config: ZapierSupportedStorageSourceConfig, inputs: SourceInputs) -> SourceResponse:+ def get_resumable_source_manager(+ self, inputs: SourceInputs+ ) -> ResumableSourceManager[ZapierSupportedStorageResumeConfig]:+ return ResumableSourceManager[ZapierSupportedStorageResumeConfig](inputs, ZapierSupportedStorageResumeConfig)++ def source_for_pipeline(+ self,+ config: ZapierSupportedStorageSourceConfig,+ resumable_source_manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig],+ inputs: SourceInputs,+ ) -> SourceResponse:return zapier_supported_storage_source(secret=config.secret,endpoint=inputs.schema_name,logger=inputs.logger,+ resumable_source_manager=resumable_source_manager,)diff --git a/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py b/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.pyindex e6daee0baad5..811f5114ab0b 100644--- a/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py+++ b/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py@@ -16,6 +16,7 @@ from products.warehouse_sources.sources.sdk.testing import (from products.warehouse_sources.sources.zapier_supported_storage._config import ZapierSupportedStorageSourceConfigfrom products.warehouse_sources.sources.zapier_supported_storage.source import ZapierSupportedStorageSourcefrom products.warehouse_sources.sources.zapier_supported_storage.zapier_supported_storage import (+ ZapierSupportedStorageResumeConfig,ZapierSupportedStorageRetryableError,validate_credentials,zapier_supported_storage_source,@@ -59,6 +60,28 @@ class TestGetRows:rows = _rows_from_store({"k": stored_value})assert rows == [{"key": "k", "value": expected}]+ @pytest.mark.parametrize(+ ("resume_state", "expected_keys"),+ [+ (None, ["a", "b", "c"]),+ (ZapierSupportedStorageResumeConfig(last_key="a"), ["b", "c"]),+ (ZapierSupportedStorageResumeConfig(last_key="c"), []),+ ],+ )+ def test_resume_reads_only_keys_after_the_saved_one(+ self, resume_state: ZapierSupportedStorageResumeConfig | None, expected_keys: list[str]+ ) -> None:+ result = _driver().run("records", [ScriptedResponse(json={"c": 3, "a": 1, "b": 2})], resume_state=resume_state)++ assert result.raised is None+ assert [row["key"] for table in result.items for row in table.to_pylist()] == expected_keys++ def test_final_batch_stages_its_last_key(self) -> None:+ result = _driver().run("records", [ScriptedResponse(json={"c": 3, "a": 1, "b": 2})])++ assert result.raised is None+ assert result.saved_states == [ZapierSupportedStorageResumeConfig(last_key="c")]+class TestFetchStore:@pytest.mark.parametrize("status", [429, 500, 502, 503])@@ -134,7 +157,9 @@ class TestSourceResponse:def test_items_is_lazy(self) -> None:# Building the SourceResponse must not issue any request; only iterating items should.with scripted_network([]) as network:- zapier_supported_storage_source(secret="s", endpoint="records", logger=MagicMock())+ zapier_supported_storage_source(+ secret="s", endpoint="records", logger=MagicMock(), resumable_source_manager=MagicMock()+ )assert network.requests_log == []assert network.session_options == []diff --git a/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py b/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.pyindex fe8255bbbde3..6ea598dbf6d3 100644--- a/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py+++ b/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py@@ -5,9 +5,16 @@ from urllib.parse import urlencode, urlsplitimport requestsfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter--from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter++from products.warehouse_sources.sources.sdk import (+ ResumableSourceManager,+ SourceResponse,+ frozen,+ make_tracked_session,+ reach_safe_point,+ retry_with_deadline,+)from products.warehouse_sources.sources.sdk.internals import Batcherfrom products.warehouse_sources.sources.zapier_supported_storage.settings import ZAPIER_SUPPORTED_STORAGE_ENDPOINTS@@ -16,10 +23,21 @@ from products.warehouse_sources.sources.zapier_supported_storage.settings importZAPIER_SUPPORTED_STORAGE_URL = "https://store.zapier.com/api/records"+@frozen+class ZapierSupportedStorageResumeConfig:+ # The last key handed on. Keys are read in sorted order, so a resume reads only the keys after it.+ last_key: str++class ZapierSupportedStorageRetryableError(Exception):pass+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:+ # The store is read in one call before any row is handed on, so nothing is held during the wait.+ reach_safe_point()++def _headers(secret: str) -> dict[str, str]:return {"X-Secret": secret, "Accept": "application/json"}@@ -44,6 +62,7 @@ def _stringify_value(value: Any) -> str | None:),stop=stop_after_attempt(5),wait=wait_exponential_jitter(initial=1, max=30),+ before_sleep=_safe_point_before_retry_wait,reraise=True,)def _fetch_store(session: requests.Session, secret: str, logger: FilteringBoundLogger) -> dict[str, Any]:@@ -83,7 +102,9 @@ def _fetch_store(session: requests.Session, secret: str, logger: FilteringBoundLreturn data-def get_rows(secret: str, logger: FilteringBoundLogger) -> Iterator[Any]:+def get_rows(+ secret: str, logger: FilteringBoundLogger, manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig]+) -> Iterator[Any]:batcher = Batcher(logger=logger, chunk_size=5000, chunk_size_bytes=100 * 1024 * 1024)# capture=False: the entire response body is the store's arbitrary key/value contents, so HTTP# sample capture would serialize customer data to object storage where the name-based scrubbers@@ -94,12 +115,16 @@ def get_rows(secret: str, logger: FilteringBoundLogger) -> Iterator[Any]:# The store is fetched in a single call - there is no pagination or list endpoint.store = _fetch_store(session, secret, logger)- for key, value in store.items():- batcher.batch({"key": key, "value": _stringify_value(value)})+ resume = manager.load_state() if manager.can_resume() else None+ keys = sorted(key for key in store if resume is None or key > resume.last_key)+ for key in keys:+ batcher.batch({"key": key, "value": _stringify_value(store[key])})if batcher.should_yield():+ manager.save_state(ZapierSupportedStorageResumeConfig(last_key=key))yield batcher.get_table()if batcher.should_yield(include_incomplete_chunk=True):+ manager.save_state(ZapierSupportedStorageResumeConfig(last_key=keys[-1]))yield batcher.get_table()@@ -107,12 +132,14 @@ def zapier_supported_storage_source(secret: str,endpoint: str,logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig],) -> SourceResponse:config = ZAPIER_SUPPORTED_STORAGE_ENDPOINTS[endpoint]return SourceResponse(name=endpoint,- items=lambda: get_rows(secret=secret, logger=logger),+ on_complete=resumable_source_manager.clear_state,+ items=lambda: get_rows(secret=secret, logger=logger, manager=resumable_source_manager),primary_keys=config.primary_keys,# Full refresh only - the store exposes no timestamps to partition on.partition_count=1,