Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): resume 9 simple sources from a saved cursor (#127477)

Tom Owers2235415b7fa9
diff --git a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
index ee2aeff8e6e2..e7b32bb4968b 100644
--- a/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
+++ b/posthog/test/repo_invariants/source_wrapped_resource_baseline.txt
@@ -46,7 +46,6 @@ shutterstock/shutterstock.py::get_rows
telli/telli.py::telli_source.get_rows
twenty/twenty.py::twenty_source.items
twitter/twitter.py::_iter_pages
-vitally/vitally.py::vitally_source
who_gho/who_gho.py::_catalog_pages
who_gho/who_gho.py::_dimension_values_pages
who_gho/who_gho.py::_indicator_data_pages
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index eba4f8224e68..247149d422dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -1,7 +1,6 @@
AmazonS3 not-checkable:schemas
AppStoreConnect not-checkable:no-request
AppleSearchAds not-checkable:no-request
-AppsFlyer fail:safe-point
Attentive not-checkable:database
AwsAthena not-checkable:schemas
AwsDocumentDB not-checkable:driver:pymongo
@@ -17,7 +16,6 @@ CiscoDuo not-checkable:no-request
ClickHouse not-checkable:schemas
CloudflareD1 not-checkable:schemas
CloudflareR2 not-checkable:schemas
-Coda fail:safe-point
Convex not-checkable:schemas
CosmosDB not-checkable:schemas
Coveralls not-checkable:no-request
@@ -42,15 +40,12 @@ GooglePlayConsole not-checkable:no-request
GoogleSearchConsole not-checkable:database
GoogleSheets not-checkable:schemas
GrafanaIRM not-checkable:no-request
-Groq fail:safe-point
Hatchet not-checkable:no-request
HelpScout not-checkable:database
Hubspot not-checkable:database
Instagram not-checkable:database
Intercom not-checkable:database
Kafka not-checkable:driver:confluent_kafka
-Kandji fail:safe-point
-Kernel fail:safe-point
Kestra not-checkable:no-request
Linear not-checkable:database
LinkedinAds not-checkable:database
@@ -60,13 +55,11 @@ MetaAds not-checkable:database
Modal not-checkable:driver:grpclib
MongoDB not-checkable:driver:pymongo
Motherduck not-checkable:driver:duckdb
-MyHours fail:safe-point
MySQL not-checkable:driver:psycopg
Neo4j not-checkable:schemas
Neon not-checkable:driver:psycopg
OpenMeteo not-checkable:no-request
OpenWeather not-checkable:no-request
-Optimizely fail:safe-point
Pardot not-checkable:database
PinterestAds not-checkable:database
Pipedrive not-checkable:database
@@ -87,9 +80,7 @@ TemporalIO not-checkable:driver:temporalio.client
TikTokAds not-checkable:database
Trino not-checkable:schemas
TwitterAds not-checkable:database
-Vitally fail:safe-point
Wrike not-checkable:no-request
YouTubeAnalytics not-checkable:database
-ZapierSupportedStorage fail:safe-point
shared-path:rest-client-paginate fail:safe-point
shared-path:rest-framework-wrapped-resource fail:safe-point
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
index 693f0b4c1be8..9160273b608d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
@@ -12,7 +12,7 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
CanonicalDescriptions,
)
@@ -20,6 +20,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.err
is_transient_egress_proxy_error,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.vitally import (
@@ -32,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.se
UPDATED_AT_INCREMENTAL_FIELD,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally import (
+ VitallyResumeConfig,
list_custom_object_definitions,
validate_credentials as validate_vitally_credentials,
vitally_source,
@@ -40,7 +42,7 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
-class VitallySource(SimpleSource[VitallySourceConfig]):
+class VitallySource(ResumableSource[VitallySourceConfig, VitallyResumeConfig]):
codebase_detection = detect(env=("^VITALLY_",), script_hosts=("cdn.vitally.io",))
api_docs_url = "https://docs.vitally.io"
@@ -151,24 +153,32 @@ class VitallySource(SimpleSource[VitallySourceConfig]):
"403 Client Error: Forbidden for url": "Your Vitally secret token does not have permission to access this data. Please check the token's permissions and reconnect.",
}
- def source_for_pipeline(self, config: VitallySourceConfig, inputs: SourceInputs) -> SourceResponse:
- items = vitally_source(
- secret_token=config.secret_token,
- region=config.region.selection,
- subdomain=config.region.subdomain,
- endpoint=inputs.schema_name,
- team_id=inputs.team_id,
- job_id=inputs.job_id,
- should_use_incremental_field=inputs.should_use_incremental_field,
- db_incremental_field_last_value=inputs.db_incremental_field_last_value
- if inputs.should_use_incremental_field
- else None,
- logger=inputs.logger,
- )
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[VitallyResumeConfig]:
+ return ResumableSourceManager[VitallyResumeConfig](inputs, VitallyResumeConfig)
+ def source_for_pipeline(
+ self,
+ config: VitallySourceConfig,
+ resumable_source_manager: ResumableSourceManager[VitallyResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return SourceResponse(
name=inputs.schema_name,
- items=lambda: items,
+ on_complete=resumable_source_manager.clear_state,
+ items=lambda: vitally_source(
+ secret_token=config.secret_token,
+ region=config.region.selection,
+ subdomain=config.region.subdomain,
+ endpoint=inputs.schema_name,
+ team_id=inputs.team_id,
+ job_id=inputs.job_id,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value
+ if inputs.should_use_incremental_field
+ else None,
+ logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
+ ),
primary_keys=["id"],
partition_count=1, # this enables partitioning
partition_size=1, # this enables partitioning
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
index 77ce0b155a29..e3aa9d6ff3ca 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
@@ -7,15 +7,21 @@ from requests import HTTPError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
+ SourceDriver,
always,
scripted_network,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.vitally import (
+ VitallyRegionConfig,
+ VitallySourceConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.settings import (
CUSTOM_OBJECT_SCHEMA_PREFIX,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.source import VitallySource
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally import (
VitallyPaginator,
+ VitallyResumeConfig,
get_custom_object_records_resource,
get_messages,
list_custom_object_definitions,
@@ -23,6 +29,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vi
)
+def _no_resume() -> Mock:
+ return Mock(can_resume=Mock(return_value=False))
+
+
def _make_response(json_data: dict[str, Any], status_code: int = 200) -> Mock:
response = Mock()
response.status_code = status_code
@@ -120,12 +130,108 @@ class TestGetMessages:
# KeyError('results'); now raise_for_status lets the real HTTP error through.
with scripted_network(always(ScriptedResponse(status=401, json={"message": "Unauthorized"}))):
with pytest.raises(HTTPError, match="401"):
- list(get_messages("secret", "EU", None, None, False, MagicMock()))
+ list(get_messages("secret", "EU", None, None, False, MagicMock(), _no_resume()))
def test_handles_missing_results_key_without_keyerror(self):
# Defensive: a 200 response that omits "results" must end pagination cleanly, not KeyError.
with scripted_network([ScriptedResponse(json={"next": None})]):
- assert list(get_messages("secret", "EU", None, None, False, MagicMock())) == []
+ assert list(get_messages("secret", "EU", None, None, False, MagicMock(), _no_resume())) == []
+
+
+class TestResume:
+ @pytest.mark.parametrize(
+ "endpoint, saved, script, expected_paths, expected_from, expected_ids",
+ [
+ pytest.param(
+ "Organizations",
+ VitallyResumeConfig(cursor="cursor-2"),
+ [ScriptedResponse(json={"results": [{"id": "org-2"}], "next": None})],
+ ["/resources/organizations"],
+ ["cursor-2"],
+ ["org-2"],
+ id="list_endpoint_starts_at_saved_cursor",
+ ),
+ pytest.param(
+ "Messages",
+ VitallyResumeConfig(cursor="cursor-2", conversations_done=["c1"]),
+ [
+ ScriptedResponse(json={"results": [{"id": "c1"}, {"id": "c2"}], "next": None}),
+ ScriptedResponse(json={"messages": [{"id": "m2"}]}),
+ ],
+ ["/resources/conversations", "/resources/conversations/c2"],
+ ["cursor-2", None],
+ ["m2"],
+ id="messages_skip_conversations_already_handed_on",
+ ),
+ pytest.param(
+ "Messages",
+ VitallyResumeConfig(cursor=None, conversations_done=["c1"]),
+ [
+ ScriptedResponse(json={"results": [{"id": "c3"}, {"id": "c1"}, {"id": "c2"}], "next": None}),
+ ScriptedResponse(json={"messages": [{"id": "m3"}]}),
+ ScriptedResponse(json={"messages": [{"id": "m2"}]}),
+ ],
+ ["/resources/conversations", "/resources/conversations/c3", "/resources/conversations/c2"],
+ [None, None, None],
+ ["m3", "m2"],
+ id="messages_read_a_conversation_that_moved_ahead_of_finished_ones",
+ ),
+ pytest.param(
+ "Organizations",
+ VitallyResumeConfig(completed=True),
+ [],
+ [],
+ [],
+ [],
+ id="list_endpoint_reads_nothing_after_its_last_page",
+ ),
+ ],
+ )
+ def test_resumes_from_saved_state(
+ self,
+ endpoint: str,
+ saved: VitallyResumeConfig,
+ script: list[ScriptedResponse],
+ expected_paths: list[str],
+ expected_from: list[str | None],
+ expected_ids: list[str],
+ ) -> None:
+ driver = SourceDriver(
+ VitallySource(),
+ VitallySourceConfig(secret_token="secret", region=VitallyRegionConfig(subdomain="", selection="EU")),
+ )
+
+ result = driver.run(endpoint, script, resume_state=saved)
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.params("from") == expected_from
+ assert [row["id"] for row in result.rows] == expected_ids
+
+ def test_messages_stage_each_conversation_before_its_messages(self) -> None:
+ driver = SourceDriver(
+ VitallySource(),
+ VitallySourceConfig(secret_token="secret", region=VitallyRegionConfig(subdomain="", selection="EU")),
+ )
+
+ result = driver.run(
+ "Messages",
+ [
+ ScriptedResponse(json={"results": [{"id": "c1"}, {"id": "c2"}], "next": "cursor-2"}),
+ ScriptedResponse(json={"messages": [{"id": "m1"}]}),
+ ScriptedResponse(json={"messages": []}),
+ ScriptedResponse(json={"results": [], "next": None}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.items == [[{"id": "m1", "conversation_updated_at": None}]]
+ assert result.saved_states == [
+ VitallyResumeConfig(cursor=None, conversations_done=[]),
+ VitallyResumeConfig(cursor=None, conversations_done=["c1"]),
+ VitallyResumeConfig(cursor=None, conversations_done=["c1", "c2"]),
+ VitallyResumeConfig(cursor="cursor-2"),
+ ]
class TestGetCustomObjectRecordsResource:
@@ -172,6 +278,7 @@ class TestVitallySourceCustomObjectRouting:
job_id="job-1",
logger=MagicMock(),
db_incremental_field_last_value=None,
+ resumable_source_manager=_no_resume(),
should_use_incremental_field=False,
)
)
@@ -202,6 +309,7 @@ class TestVitallySourceCustomObjectRouting:
job_id="job-1",
logger=MagicMock(),
db_incremental_field_last_value=None,
+ resumable_source_manager=_no_resume(),
should_use_incremental_field=False,
)
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
index 75bfc5e8d975..f880b64611d1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
@@ -1,11 +1,15 @@
import base64
+from collections.abc import Iterable, Iterator
+from dataclasses import field
from datetime import datetime
from typing import Any, Optional
from dateutil import parser
-from requests import HTTPError, JSONDecodeError, Request, Response
+from requests import HTTPError, JSONDecodeError, Request, Response, Session
from structlog.types import FilteringBoundLogger
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
@@ -13,11 +17,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.settings import (
CUSTOM_OBJECT_SCHEMA_PREFIX,
)
+@frozen
+class VitallyResumeConfig:
+ # The `from` cursor of the page to read. None is the first page.
+ cursor: str | None = None
+ # Messages only: the ids of that page's conversations already handed on. Ids, not a count, because
+ # a conversation updated between attempts moves within the `updatedAt` order.
+ conversations_done: list[str] = field(default_factory=list)
+ # List endpoints only: the last page is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResource:
resources: dict[str, EndpointResource] = {
"Organizations": {
@@ -381,12 +397,26 @@ class VitallyPaginator(BasePaginator):
else:
self._has_next_page = False
+ def init_request(self, request: Request) -> None:
+ # Apply a resumed cursor to the first request.
+ if self._cursor is not None:
+ self.update_request(request)
+
def update_request(self, request: Request) -> None:
if request.params is None:
request.params = {}
request.params["from"] = self._cursor
+ def get_resume_state(self) -> Optional[dict[str, Any]]:
+ return {"cursor": self._cursor} if self._has_next_page and self._cursor else None
+
+ def set_resume_state(self, state: dict[str, Any]) -> None:
+ cursor = state.get("cursor")
+ if cursor:
+ self._cursor = str(cursor)
+ self._has_next_page = True
+
def get_messages(
secret_token: str,
@@ -395,7 +425,8 @@ def get_messages(
db_incremental_field_last_value: Optional[Any],
should_use_incremental_field: bool,
logger: FilteringBoundLogger,
-):
+ manager: ResumableSourceManager[VitallyResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
"""Messages are a field on conversations which only get returned
when you request each conversation individually. This queries
for conversations and then gets and yields the messages for each
@@ -429,10 +460,19 @@ def get_messages(
headers={"Authorization": f"Basic {basic_token}:"},
)
+ resume = manager.load_state() if manager.can_resume() else None
+ page_cursor = resume.cursor if resume is not None else None
+ done = list(resume.conversations_done) if resume is not None else []
+ if page_cursor is not None:
+ paginator.set_resume_state({"cursor": page_cursor})
+
logger.debug("Requesting first page")
with make_tracked_session() as session:
while paginator.has_next_page:
+ # Every conversation before this point is handed on, so an empty page can end the run here.
+ manager.save_state(VitallyResumeConfig(cursor=page_cursor, conversations_done=list(done)))
+ manager.safe_point()
paginator.update_request(request)
prepared_request = session.prepare_request(request)
response = session.send(prepared_request)
@@ -442,33 +482,60 @@ def get_messages(
results = response.json().get("results") or []
for conversation in results:
- id = conversation.get("id")
- conversation_updated_at = conversation.get("updatedAt")
- logger.debug(f"Requesting messages for conversation {id}")
-
- conversation_response = session.get(
- f"{get_base_url(region, subdomain)}resources/conversations/{id}",
- headers={"Authorization": f"Basic {basic_token}:"},
- )
-
- try:
- conversation_response.raise_for_status()
-
- messages = conversation_response.json().get("messages") or []
- logger.debug(f"Yielding {len(messages)} messages")
- for message in messages:
- message["conversation_updated_at"] = conversation_updated_at
- yield message
- except HTTPError as e:
- logger.debug(
- f"Failed to fetch messages for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"
- )
- except JSONDecodeError as e:
- logger.debug(
- f"Failed to decode JSON response for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"
- )
+ # A resumed page is read again from its start, so skip the conversations already handed on.
+ conversation_id = str(conversation.get("id"))
+ if conversation_id in done:
+ continue
+ messages = _conversation_messages(session, conversation, region, subdomain, basic_token, logger)
+ done.append(conversation_id)
+ manager.save_state(VitallyResumeConfig(cursor=page_cursor, conversations_done=list(done)))
+ if messages:
+ # One list per conversation, so its messages and the cursor past it commit together.
+ yield messages
+ else:
+ manager.safe_point()
+ done = []
paginator.update_state(response)
+ next_state = paginator.get_resume_state()
+ page_cursor = next_state["cursor"] if next_state else None
+
+
+def _conversation_messages(
+ session: Session,
+ conversation: dict[str, Any],
+ region: str,
+ subdomain: Optional[str],
+ basic_token: str,
+ logger: FilteringBoundLogger,
+) -> list[dict[str, Any]]:
+ id = conversation.get("id")
+ conversation_updated_at = conversation.get("updatedAt")
+ logger.debug(f"Requesting messages for conversation {id}")
+
+ conversation_response = session.get(
+ f"{get_base_url(region, subdomain)}resources/conversations/{id}",
+ headers={"Authorization": f"Basic {basic_token}:"},
+ timeout=60,
+ )
+
+ try:
+ conversation_response.raise_for_status()
+
+ messages = conversation_response.json().get("messages") or []
+ logger.debug(f"Yielding {len(messages)} messages")
+ for message in messages:
+ message["conversation_updated_at"] = conversation_updated_at
+ return messages
+ except HTTPError as e:
+ logger.debug(
+ f"Failed to fetch messages for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"
+ )
+ except JSONDecodeError as e:
+ logger.debug(
+ f"Failed to decode JSON response for conversation {id}: {conversation_response.status_code} {e}. Body: {conversation_response.text}"
+ )
+ return []
def get_base_url(region: str, subdomain: Optional[str]) -> str:
@@ -487,13 +554,19 @@ def vitally_source(
job_id: str,
logger: FilteringBoundLogger,
db_incremental_field_last_value: Optional[Any],
+ resumable_source_manager: ResumableSourceManager[VitallyResumeConfig],
should_use_incremental_field: bool = False,
-):
+) -> Iterable[Any]:
if endpoint == "Messages":
- yield from get_messages(
- secret_token, region, subdomain, db_incremental_field_last_value, should_use_incremental_field, logger
+ return get_messages(
+ secret_token,
+ region,
+ subdomain,
+ db_incremental_field_last_value,
+ should_use_incremental_field,
+ logger,
+ resumable_source_manager,
)
- return
if endpoint.startswith(CUSTOM_OBJECT_SCHEMA_PREFIX):
machine_name = endpoint[len(CUSTOM_OBJECT_SCHEMA_PREFIX) :]
@@ -532,7 +605,26 @@ def vitally_source(
"resources": [endpoint_resource],
}
- yield from rest_api_resource(config, team_id, job_id, db_incremental_field_last_value)
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(VitallyResumeConfig(cursor=str(state["cursor"])))
+ elif state is None:
+ resumable_source_manager.save_state(VitallyResumeConfig(completed=True))
+
+ if resume is not None and resume.completed:
+ return []
+
+ # Returned, not wrapped, so the framework reaches a safe point after each page.
+ return rest_api_resource(
+ config,
+ team_id,
+ job_id,
+ db_incremental_field_last_value,
+ resume_hook=save_checkpoint,
+ initial_paginator_state={"cursor": resume.cursor} if resume is not None and resume.cursor else None,
+ )
def validate_credentials(secret_token: str, region: str, subdomain: Optional[str]) -> bool:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
index 61b8db733a74..6c8f1d0d6798 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/e2e/test_end_to_end.py
@@ -3060,12 +3060,8 @@ async def test_worker_shutdown_desc_sort_order(team):
def mock_raise_if_is_worker_shutdown(self):
raise WorkerShuttingDownError("test_id", "test_type", "test_queue", 1, "test_workflow", "test_workflow_type")
- def mock_get_messages(*args, **kwargs):
- yield {
- "id": "test-message-id",
- "conversation_updated_at": datetime.now().isoformat(),
- "created_at": datetime.now().isoformat(),
- }
+ def mock_get_rows(*args, **kwargs):
+ yield [{"_board_id": "1", "id": "test-log-id", "created_at": datetime.now().isoformat()}]
with (
mock.patch.object(ShutdownMonitor, "raise_if_is_worker_shutdown", mock_raise_if_is_worker_shutdown),
@@ -3076,22 +3072,19 @@ async def test_worker_shutdown_desc_sort_order(team):
"products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher.DEFAULT_CHUNK_SIZE", 1
),
mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally.get_messages",
- mock_get_messages,
+ "products.warehouse_sources.sources.monday.monday.get_rows",
+ mock_get_rows,
),
):
_, inputs = await _run(
team=team,
- schema_name="Messages",
- table_name="vitally_messages",
- source_type="Vitally",
- job_inputs={
- "secret_token": "test_token",
- "region": {"selection": "EU", "subdomain": ""},
- },
+ schema_name="board_activity_logs",
+ table_name="monday_board_activity_logs",
+ source_type="Monday",
+ job_inputs={"api_token": "test_token"},
mock_data_response=[],
sync_type=ExternalDataSchemaSyncType.INCREMENTAL,
- sync_type_config={"incremental_field": "conversation_updated_at", "incremental_field_type": "datetime"},
+ sync_type_config={"incremental_field": "created_at", "incremental_field_type": "datetime"},
ignore_assertions=True,
)
diff --git a/products/warehouse_sources/sources/appsflyer/appsflyer.py b/products/warehouse_sources/sources/appsflyer/appsflyer.py
index 0649785daa1c..9b39512b61a1 100644
--- a/products/warehouse_sources/sources/appsflyer/appsflyer.py
+++ b/products/warehouse_sources/sources/appsflyer/appsflyer.py
@@ -15,7 +15,13 @@ from products.warehouse_sources.sources.appsflyer.settings import (
AppsFlyerEndpointConfig,
AppsFlyerReportKind,
)
-from products.warehouse_sources.sources.sdk import SourceResponse, frozen, make_tracked_session, retry_with_deadline
+from products.warehouse_sources.sources.sdk import (
+ ResumableSourceManager,
+ SourceResponse,
+ frozen,
+ make_tracked_session,
+ retry_with_deadline,
+)
APPSFLYER_BASE_URL = "https://hq1.appsflyer.com"
# Aggregate pull requests cap the date range at ~1000 days.
@@ -54,6 +60,12 @@ MAX_REJECTION_REASON_CHARS = 300
_HTML_MARKUP = re.compile(r"<\s*(!doctype|html|head|body)\b", re.IGNORECASE)
+@frozen
+class AppsFlyerResumeConfig:
+ # The first day of the next request window, as an ISO date.
+ next_window_start: str
+
+
@frozen
class _ReportWindow:
start: date
@@ -351,6 +363,7 @@ def get_rows(
app_id: str,
endpoint: str,
logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[AppsFlyerResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Any = None,
) -> Iterator[list[dict[str, Any]]]:
@@ -360,6 +373,13 @@ def get_rows(
today = datetime.now(UTC).date()
start = _window_start(config, today, should_use_incremental_field, db_incremental_field_last_value)
+ # A full refresh appends, so the part of a window read before a hand-off would load twice.
+ resume = manager.load_state() if should_use_incremental_field and manager.can_resume() else None
+ if resume is not None:
+ # Never earlier than the computed start, which keeps a raw pull inside its lookback limit.
+ start = max(start, date.fromisoformat(resume.next_window_start))
+ if start > today:
+ return
chunk: list[dict[str, Any]] = []
for window in _request_windows(config.kind, start, today):
@@ -379,8 +399,14 @@ def get_rows(
window_end=window.end.isoformat(),
row_cap=RAW_MAX_ROWS,
)
- if chunk:
- yield chunk
+ # A window is the resume unit: rows have no documented order inside one, so a resume reads
+ # a part-way window again. The rest of the window goes out with the cursor past it.
+ manager.save_state(AppsFlyerResumeConfig(next_window_start=(window.end + timedelta(days=1)).isoformat()))
+ if chunk:
+ yield chunk
+ chunk = []
+ else:
+ manager.safe_point()
def appsflyer_source(
@@ -388,6 +414,7 @@ def appsflyer_source(
app_id: str,
endpoint: str,
logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[AppsFlyerResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Any = None,
) -> SourceResponse:
@@ -395,11 +422,13 @@ def appsflyer_source(
return SourceResponse(
name=endpoint,
+ on_complete=resumable_source_manager.clear_state,
items=lambda: get_rows(
api_token=api_token,
app_id=app_id,
endpoint=endpoint,
logger=logger,
+ manager=resumable_source_manager,
should_use_incremental_field=should_use_incremental_field,
db_incremental_field_last_value=db_incremental_field_last_value,
),
@@ -409,6 +438,7 @@ def appsflyer_source(
partition_mode="datetime",
partition_format=config.partition_format,
partition_keys=[config.partition_key],
+ supports_resume=should_use_incremental_field,
# Dimension keys can collide (e.g. blank campaign values) — an expected trait of report
# data, not something the user can fix. Don't set has_duplicate_primary_keys: that flag
# tells validate_incremental_sync to block incremental syncing altogether. The merge's own
diff --git a/products/warehouse_sources/sources/appsflyer/source.py b/products/warehouse_sources/sources/appsflyer/source.py
index 31791af47822..9139341ac707 100644
--- a/products/warehouse_sources/sources/appsflyer/source.py
+++ b/products/warehouse_sources/sources/appsflyer/source.py
@@ -5,6 +5,7 @@ import requests
from products.warehouse_sources.sources.appsflyer._config import AppsFlyerSourceConfig
from products.warehouse_sources.sources.appsflyer.appsflyer import (
AppsFlyerCredentialsError,
+ AppsFlyerResumeConfig,
AppsFlyerRetryableError,
appsflyer_source,
validate_credentials as validate_appsflyer_credentials,
@@ -16,7 +17,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -30,7 +32,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class AppsFlyerSource(SimpleSource[AppsFlyerSourceConfig]):
+class AppsFlyerSource(ResumableSource[AppsFlyerSourceConfig, AppsFlyerResumeConfig]):
codebase_detection = detect(
env=("^APPSFLYER_",),
script_hosts=("websdk.appsflyersdk.com",),
@@ -190,12 +192,24 @@ Raw data tables (installs, in-app events, uninstalls, retargeting conversions, a
"Could not reach AppsFlyer to validate credentials. This may be a temporary rate-limit or network issue — please try again.",
)
- def source_for_pipeline(self, config: AppsFlyerSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def resume_covers_run(self, *, incremental_or_append: bool, schema_name: str | None = None) -> bool:
+ return incremental_or_append
+
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[AppsFlyerResumeConfig]:
+ return ResumableSourceManager[AppsFlyerResumeConfig](inputs, AppsFlyerResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: AppsFlyerSourceConfig,
+ resumable_source_manager: ResumableSourceManager[AppsFlyerResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return appsflyer_source(
api_token=config.api_token,
app_id=config.app_id,
endpoint=inputs.schema_name,
logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
diff --git a/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py b/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py
index 197741b72628..b9347113e8ab 100644
--- a/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py
+++ b/products/warehouse_sources/sources/appsflyer/tests/test_appsflyer.py
@@ -13,6 +13,7 @@ from products.warehouse_sources.sources.appsflyer.appsflyer import (
RAW_MAX_LOOKBACK_DAYS,
RAW_MAX_REQUEST_DAYS,
AppsFlyerCredentialsError,
+ AppsFlyerResumeConfig,
AppsFlyerRetryableError,
_normalize_header,
_parse_csv_rows,
@@ -283,6 +284,45 @@ class TestRawAndMasterWindows:
for window_start, window_end in windows:
assert 0 <= (window_end - window_start).days < MASTER_MAX_REQUEST_DAYS
+ @pytest.mark.parametrize(
+ "endpoint, incremental, saved_days_ago, expected_first_from_days_ago, expected_requests",
+ [
+ # A raw pull resumes at the saved window, and each window stages the start of the next.
+ ("installs", True, 20, 20, 3),
+ # The saved day is clamped to the lookback limit, which moves forward between attempts.
+ ("installs", True, RAW_MAX_LOOKBACK_DAYS + 5, RAW_MAX_LOOKBACK_DAYS, 13),
+ ("master_report", True, 40, 40, 2),
+ # A single-window report whose window is already written reads nothing.
+ ("daily_report", True, -1, None, 0),
+ # A full refresh appends a replayed window part twice, so it reads from the start.
+ ("installs", False, 20, RAW_MAX_LOOKBACK_DAYS, 13),
+ ],
+ )
+ def test_resumes_at_the_saved_window(
+ self,
+ endpoint: str,
+ incremental: bool,
+ saved_days_ago: int,
+ expected_first_from_days_ago: int | None,
+ expected_requests: int,
+ ) -> None:
+ today = datetime.now(UTC).date()
+ saved = AppsFlyerResumeConfig(next_window_start=(today - timedelta(days=saved_days_ago)).isoformat())
+
+ result = _driver().run(
+ endpoint, always(_csv_response()), resume_state=saved, should_use_incremental_field=incremental
+ )
+
+ assert result.raised is None
+ assert result.response is not None
+ assert result.response.supports_resume is incremental
+ assert len(result.requests) == expected_requests
+ if expected_first_from_days_ago is not None:
+ assert result.params("from")[0] == (today - timedelta(days=expected_first_from_days_ago)).isoformat()
+ assert result.final_state == AppsFlyerResumeConfig(
+ next_window_start=(today + timedelta(days=1)).isoformat()
+ )
+
@pytest.mark.parametrize("endpoint", ["installs", "in_app_events", "ad_revenue"])
def test_raw_incremental_repulls_a_whole_window_before_the_watermark(self, endpoint: str) -> None:
# A worker shutdown can interrupt a window part-way through, and AppsFlyer documents no row
@@ -371,5 +411,7 @@ class TestRawAndMasterWindows:
class TestAppsFlyerSourceResponse:
def test_geo_report_key_includes_country(self):
- response = appsflyer_source("token", "id123", "geo_report", mock.MagicMock())
+ response = appsflyer_source(
+ "token", "id123", "geo_report", mock.MagicMock(), resumable_source_manager=mock.MagicMock()
+ )
assert "country" in (response.primary_keys or [])
diff --git a/products/warehouse_sources/sources/coda/coda.py b/products/warehouse_sources/sources/coda/coda.py
index a69fb783b801..d2f1d339e1ac 100644
--- a/products/warehouse_sources/sources/coda/coda.py
+++ b/products/warehouse_sources/sources/coda/coda.py
@@ -1,4 +1,4 @@
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
from typing import Any, cast
from products.warehouse_sources.sources.coda.settings import CODA_ENDPOINTS
@@ -7,7 +7,9 @@ from products.warehouse_sources.sources.sdk import (
JSONResponseCursorPaginator,
ResponseAction,
RESTAPIConfig,
+ ResumableSourceManager,
SourceResponse,
+ frozen,
make_tracked_session,
rest_api_resources,
validate_via_probe,
@@ -17,6 +19,16 @@ CODA_BASE_URL = "https://coda.io/apis/v1"
PAGE_SIZE = 100
ROWS_PAGE_SIZE = 200
+
+@frozen
+class CodaResumeConfig:
+ # What the framework handed the resume hook: the next page token of a list endpoint, or the
+ # finished and in-progress parents of a fan-out. Stored unchanged.
+ paginator_state: dict[str, Any] | None = None
+ # The last page of a list endpoint is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
# The docs list also returns docs the token user only opened (shared or published docs from other
# workspaces). Their sharing settings and their workspace's members are not readable, so skip those
# parents instead of failing the whole fan-out.
@@ -305,6 +317,7 @@ def coda_source(
endpoint: str,
team_id: int,
job_id: str,
+ resumable_source_manager: ResumableSourceManager[CodaResumeConfig],
) -> SourceResponse:
resource_chain = _resource_chain(endpoint)
config = CODA_ENDPOINTS[endpoint]
@@ -318,12 +331,34 @@ def coda_source(
"resources": cast("list[str | EndpointResource]", resource_chain),
}
- resources = rest_api_resources(rest_config, team_id, job_id, None)
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state:
+ resumable_source_manager.save_state(CodaResumeConfig(paginator_state=state))
+ elif state is None:
+ resumable_source_manager.save_state(CodaResumeConfig(completed=True))
+
+ # The framework resumes a fan-out with one dependent level only. Rows and columns chain two
+ # levels, so they read every doc and table again after a resume.
+ resources = rest_api_resources(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=resume.paginator_state if resume is not None else None,
+ )
resource = next(r for r in resources if r.name == endpoint)
+ completed = resume is not None and resume.completed
+
+ def items() -> Iterable[Any]:
+ return [] if completed else resource
return SourceResponse(
name=endpoint,
- items=lambda: resource,
+ on_complete=resumable_source_manager.clear_state,
+ items=items,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/sources/coda/source.py b/products/warehouse_sources/sources/coda/source.py
index 08dc6f374a70..3db04440d0ba 100644
--- a/products/warehouse_sources/sources/coda/source.py
+++ b/products/warehouse_sources/sources/coda/source.py
@@ -2,6 +2,7 @@ from typing import Optional, cast
from products.warehouse_sources.sources.coda._config import CodaSourceConfig
from products.warehouse_sources.sources.coda.coda import (
+ CodaResumeConfig,
coda_source,
validate_credentials as validate_coda_credentials,
)
@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -26,7 +28,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class CodaSource(SimpleSource[CodaSourceConfig]):
+class CodaSource(ResumableSource[CodaSourceConfig, CodaResumeConfig]):
codebase_detection = detect(env=("^CODA_API_",), npm=("@codahq/packs-sdk",), pypi=("codaio",))
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
@@ -97,10 +99,19 @@ You can generate an API token in [Coda account settings](https://coda.io/account
return False, "Invalid Coda API token"
- def source_for_pipeline(self, config: CodaSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[CodaResumeConfig]:
+ return ResumableSourceManager[CodaResumeConfig](inputs, CodaResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: CodaSourceConfig,
+ resumable_source_manager: ResumableSourceManager[CodaResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return coda_source(
api_token=config.api_token,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
)
diff --git a/products/warehouse_sources/sources/coda/tests/test_coda.py b/products/warehouse_sources/sources/coda/tests/test_coda.py
index 44138777b012..6b2c91855683 100644
--- a/products/warehouse_sources/sources/coda/tests/test_coda.py
+++ b/products/warehouse_sources/sources/coda/tests/test_coda.py
@@ -1,9 +1,10 @@
from typing import Any
import pytest
+from unittest.mock import MagicMock
from products.warehouse_sources.sources.coda._config import CodaSourceConfig
-from products.warehouse_sources.sources.coda.coda import coda_source, validate_credentials
+from products.warehouse_sources.sources.coda.coda import CodaResumeConfig, coda_source, validate_credentials
from products.warehouse_sources.sources.coda.settings import CODA_ENDPOINTS, ENDPOINTS
from products.warehouse_sources.sources.coda.source import CodaSource
from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
@@ -161,6 +162,55 @@ class TestGetRows:
key_column, parent_id = key
assert [row[key_column] for row in result.rows] == [parent_id]
+ @pytest.mark.parametrize(
+ "endpoint, saved, responses, expected_paths, expected_tokens, expected_ids",
+ [
+ (
+ "docs",
+ {"cursor": "token-2"},
+ [_response([{"id": "doc2"}])],
+ ["/apis/v1/docs"],
+ ["token-2"],
+ ["doc2"],
+ ),
+ (
+ "pages",
+ {"completed": ["/docs/doc1/pages"], "current": None, "child_state": None},
+ [_response([{"id": "doc1"}, {"id": "doc2"}]), _response([{"id": "canvas-2"}])],
+ ["/apis/v1/docs", "/apis/v1/docs/doc2/pages"],
+ [None, None],
+ ["canvas-2"],
+ ),
+ ],
+ )
+ def test_resumes_from_saved_state(
+ self,
+ endpoint: str,
+ saved: dict[str, Any],
+ responses: list[ScriptedResponse],
+ expected_paths: list[str],
+ expected_tokens: list[str | None],
+ expected_ids: list[str],
+ ) -> None:
+ result = _driver().run(endpoint, responses, resume_state=CodaResumeConfig(paginator_state=saved))
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.params("pageToken") == expected_tokens
+ assert [row["id"] for row in result.rows] == expected_ids
+
+ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:
+ result = _driver().run("docs", [_response([{"id": "doc1"}])])
+
+ assert result.raised is None
+ assert result.saved_states[-1] == CodaResumeConfig(completed=True)
+
+ resumed = _driver().run("docs", [], resume_state=CodaResumeConfig(completed=True))
+
+ assert resumed.raised is None
+ assert resumed.paths == []
+ assert resumed.rows == []
+
def test_table_without_id_fails_fast_in_rows(self) -> None:
result = _driver().run("rows", [_response([{"id": "doc1"}]), _response([{"name": "broken"}])])
@@ -174,7 +224,9 @@ class TestCodaSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = CODA_ENDPOINTS[endpoint]
- response = coda_source("token", endpoint, team_id=1, job_id="j")
+ manager = MagicMock()
+ manager.can_resume.return_value = False
+ response = coda_source("token", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)
assert response.name == endpoint
assert response.primary_keys == config.primary_keys
diff --git a/products/warehouse_sources/sources/groq/groq.py b/products/warehouse_sources/sources/groq/groq.py
index 9d1fcf24745f..85ec29ed7e76 100644
--- a/products/warehouse_sources/sources/groq/groq.py
+++ b/products/warehouse_sources/sources/groq/groq.py
@@ -1,10 +1,15 @@
+from collections.abc import Iterable
+from typing import Any
+
from products.warehouse_sources.sources.groq.settings import GROQ_ENDPOINTS, GroqEndpointConfig
from products.warehouse_sources.sources.sdk import (
BasePaginator,
JSONResponseCursorPaginator,
RESTAPIConfig,
+ ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
+ frozen,
make_tracked_session,
rest_api_resource,
validate_via_probe,
@@ -13,6 +18,14 @@ from products.warehouse_sources.sources.sdk import (
GROQ_BASE_URL = "https://api.groq.com/openai/v1"
+@frozen
+class GroqResumeConfig:
+ # The `paging.next_cursor` of the last page handed on. Only `batches` paginates.
+ cursor: str | None = None
+ # The last page is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
def _get_headers(api_key: str) -> dict[str, str]:
return {
"Authorization": f"Bearer {api_key}",
@@ -34,6 +47,7 @@ def groq_source(
endpoint: str,
team_id: int,
job_id: str,
+ resumable_source_manager: ResumableSourceManager[GroqResumeConfig],
) -> SourceResponse:
config: GroqEndpointConfig = GROQ_ENDPOINTS[endpoint]
@@ -63,11 +77,31 @@ def groq_source(
],
}
- resource = rest_api_resource(rest_config, team_id, job_id, None)
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state and state.get("cursor"):
+ resumable_source_manager.save_state(GroqResumeConfig(cursor=str(state["cursor"])))
+ elif state is None:
+ resumable_source_manager.save_state(GroqResumeConfig(completed=True))
+
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state={"cursor": resume.cursor} if resume is not None and resume.cursor else None,
+ )
+ completed = resume is not None and resume.completed
+
+ def items() -> Iterable[Any]:
+ return [] if completed else resource
return SourceResponse(
name=endpoint,
- items=lambda: resource,
+ on_complete=resumable_source_manager.clear_state,
+ items=items,
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/sources/groq/source.py b/products/warehouse_sources/sources/groq/source.py
index bea39185cdff..99007d6577bb 100644
--- a/products/warehouse_sources/sources/groq/source.py
+++ b/products/warehouse_sources/sources/groq/source.py
@@ -2,6 +2,7 @@ from typing import Optional, cast
from products.warehouse_sources.sources.groq._config import GroqSourceConfig
from products.warehouse_sources.sources.groq.groq import (
+ GroqResumeConfig,
groq_source,
validate_credentials as validate_groq_credentials,
)
@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -25,7 +27,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class GroqSource(SimpleSource[GroqSourceConfig]):
+class GroqSource(ResumableSource[GroqSourceConfig, GroqResumeConfig]):
codebase_detection = detect(env=("^GROQ_",), npm=("groq-sdk",), pypi=("groq",))
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
@@ -120,10 +122,19 @@ Create an API key in the [Groq console](https://console.groq.com/keys). Groq exp
return False, "Your Groq API key is missing the permissions needed to sync this data"
return False, "Could not connect to Groq with the provided API key"
- def source_for_pipeline(self, config: GroqSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[GroqResumeConfig]:
+ return ResumableSourceManager[GroqResumeConfig](inputs, GroqResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: GroqSourceConfig,
+ resumable_source_manager: ResumableSourceManager[GroqResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return groq_source(
api_key=config.api_key,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
)
diff --git a/products/warehouse_sources/sources/groq/tests/test_groq.py b/products/warehouse_sources/sources/groq/tests/test_groq.py
index 715c8d52bfcb..3ce5c394b842 100644
--- a/products/warehouse_sources/sources/groq/tests/test_groq.py
+++ b/products/warehouse_sources/sources/groq/tests/test_groq.py
@@ -1,8 +1,10 @@
+from unittest.mock import MagicMock
+
import requests
from parameterized import parameterized
from products.warehouse_sources.sources.groq._config import GroqSourceConfig
-from products.warehouse_sources.sources.groq.groq import groq_source, validate_credentials
+from products.warehouse_sources.sources.groq.groq import GroqResumeConfig, groq_source, validate_credentials
from products.warehouse_sources.sources.groq.source import GroqSource
from products.warehouse_sources.sources.sdk import RESTClientRetryableError
from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
@@ -26,6 +28,41 @@ class TestGroq:
assert [row["id"] for row in result.rows] == ["a", "b"]
assert result.paths == [f"/openai/v1/{endpoint}"]
+ @parameterized.expand(
+ [
+ (
+ "fresh_run",
+ None,
+ [None, "c2"],
+ ["a", "b"],
+ [GroqResumeConfig(cursor="c2"), GroqResumeConfig(completed=True)],
+ ),
+ ("resumed_run", GroqResumeConfig(cursor="c2"), ["c2"], ["b"], [GroqResumeConfig(completed=True)]),
+ # A hand-off after the last page must not read that page again.
+ ("completed_run", GroqResumeConfig(completed=True), [], [], []),
+ ]
+ )
+ def test_batches_resume_from_saved_cursor(
+ self,
+ _name: str,
+ resume_state: GroqResumeConfig | None,
+ expected_cursors: list[str | None],
+ expected_ids: list[str],
+ expected_saved: list[GroqResumeConfig],
+ ) -> None:
+ pages = {
+ None: ScriptedResponse(json={"data": [{"id": "a"}], "paging": {"next_cursor": "c2"}}),
+ "c2": ScriptedResponse(json={"data": [{"id": "b"}], "paging": {}}),
+ }
+ result = self._driver().run(
+ "batches", lambda request: pages[request.param("cursor")], resume_state=resume_state
+ )
+
+ assert result.raised is None
+ assert result.params("cursor") == expected_cursors
+ assert [row["id"] for row in result.rows] == expected_ids
+ assert result.saved_states == expected_saved
+
def test_non_dict_body_is_retried_then_reraises(self) -> None:
# A non-object body (bare array) is an unexpected shape; the request is reissued and, if it
# never recovers, surfaces as a retryable error after the attempt cap.
@@ -72,7 +109,9 @@ class TestGroq:
)
def test_groq_source_maps_primary_keys_and_partitioning(self, endpoint: str, partition_key: str) -> None:
# No network: SourceResponse metadata is built eagerly, rows only on iteration.
- response = groq_source("gsk_k", endpoint, team_id=1, job_id="j")
+ manager = MagicMock()
+ manager.can_resume.return_value = False
+ response = groq_source("gsk_k", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)
assert response.name == endpoint
assert response.primary_keys == ["id"]
assert response.partition_mode == "datetime"
diff --git a/products/warehouse_sources/sources/kandji/kandji.py b/products/warehouse_sources/sources/kandji/kandji.py
index b57dbb4dec6d..2733f5a05996 100644
--- a/products/warehouse_sources/sources/kandji/kandji.py
+++ b/products/warehouse_sources/sources/kandji/kandji.py
@@ -19,13 +19,25 @@ from products.warehouse_sources.sources.sdk import (
EndpointResource,
OffsetPaginator,
RESTAPIConfig,
+ ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
build_dependent_resource,
+ frozen,
make_tracked_session,
rest_api_resource,
)
+
+@frozen
+class KandjiResumeConfig:
+ # What the framework handed the resume hook: the next offset or `next` query of a list endpoint,
+ # or the device listing position and finished devices of a fan-out. Stored unchanged.
+ paginator_state: dict[str, Any] | None = None
+ # The last page of a list endpoint is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
REGION_TEMPLATES = {
"us": US_API_HOST_TEMPLATE,
"eu": EU_API_HOST_TEMPLATE,
@@ -87,10 +99,23 @@ class KandjiNextLinkPaginator(BasePaginator):
self._next_params = next_params
self._has_next_page = True
+ def init_request(self, request: Request) -> None:
+ # Apply a resumed `next` query to the first request.
+ self.update_request(request)
+
def update_request(self, request: Request) -> None:
if self._next_params is not None:
request.params = {**(request.params or {}), **self._next_params}
+ def get_resume_state(self) -> Optional[dict[str, Any]]:
+ return {"next_params": self._next_params} if self._has_next_page and self._next_params else None
+
+ def set_resume_state(self, state: dict[str, Any]) -> None:
+ next_params = state.get("next_params")
+ if isinstance(next_params, dict) and next_params:
+ self._next_params = {str(key): str(value) for key, value in next_params.items()}
+ self._has_next_page = True
+
def _list_paginator(config: KandjiEndpointConfig) -> BasePaginator:
if config.pagination == "single":
@@ -157,9 +182,12 @@ def get_resource(config: KandjiEndpointConfig) -> EndpointResource:
}
-def _make_source_response(config: KandjiEndpointConfig, items_fn) -> SourceResponse:
+def _make_source_response(
+ config: KandjiEndpointConfig, items_fn, manager: ResumableSourceManager[KandjiResumeConfig]
+) -> SourceResponse:
return SourceResponse(
name=config.name,
+ on_complete=manager.clear_state,
items=items_fn,
primary_keys=config.primary_key if isinstance(config.primary_key, list) else [config.primary_key],
# Full refresh only — Kandji's list endpoints expose no stable updated-since cursor.
@@ -174,11 +202,21 @@ def kandji_source(
endpoint: str,
team_id: int,
job_id: str,
+ resumable_source_manager: ResumableSourceManager[KandjiResumeConfig],
) -> SourceResponse:
config = KANDJI_ENDPOINTS[endpoint]
base_url = build_base_url(subdomain, region)
client_config = _rest_api_client_config(base_url, api_token)
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ initial_paginator_state = resume.paginator_state if resume is not None else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state:
+ resumable_source_manager.save_state(KandjiResumeConfig(paginator_state=state))
+ elif state is None:
+ resumable_source_manager.save_state(KandjiResumeConfig(completed=True))
+
if config.fanout is not None:
parent_config = KANDJI_ENDPOINTS[config.fanout.parent_name]
dependent_resource = cast(
@@ -202,14 +240,25 @@ def kandji_source(
"paginator": _list_paginator(config),
"data_selector": config.data_selector,
},
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ resume_parent_pagination=True,
),
)
- return _make_source_response(config, lambda: dependent_resource)
+ return _make_source_response(config, lambda: dependent_resource, resumable_source_manager)
rest_config: RESTAPIConfig = {
"client": client_config,
"resource_defaults": {"write_disposition": "replace"},
"resources": [get_resource(config)],
}
- resource = rest_api_resource(rest_config, team_id, job_id, None)
- return _make_source_response(config, lambda: resource)
+ resource = rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=initial_paginator_state,
+ )
+ completed = resume is not None and resume.completed
+ return _make_source_response(config, lambda: [] if completed else resource, resumable_source_manager)
diff --git a/products/warehouse_sources/sources/kandji/source.py b/products/warehouse_sources/sources/kandji/source.py
index 334ee40e9480..1edf9ee4f0c0 100644
--- a/products/warehouse_sources/sources/kandji/source.py
+++ b/products/warehouse_sources/sources/kandji/source.py
@@ -2,6 +2,7 @@ from typing import Optional, cast
from products.warehouse_sources.sources.kandji._config import KandjiSourceConfig
from products.warehouse_sources.sources.kandji.kandji import (
+ KandjiResumeConfig,
kandji_source,
validate_credentials as validate_kandji_credentials,
)
@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -27,7 +29,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class KandjiSource(SimpleSource[KandjiSourceConfig]):
+class KandjiSource(ResumableSource[KandjiSourceConfig, KandjiResumeConfig]):
codebase_detection = detect(env=("^KANDJI_",))
# `get_schemas` iterates a static endpoint catalog with no I/O, so the table list is safe to
@@ -95,7 +97,15 @@ class KandjiSource(SimpleSource[KandjiSourceConfig]):
schema_name=schema_name,
)
- def source_for_pipeline(self, config: KandjiSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KandjiResumeConfig]:
+ return ResumableSourceManager[KandjiResumeConfig](inputs, KandjiResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: KandjiSourceConfig,
+ resumable_source_manager: ResumableSourceManager[KandjiResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return kandji_source(
api_token=config.api_token,
subdomain=config.subdomain,
@@ -103,6 +113,7 @@ class KandjiSource(SimpleSource[KandjiSourceConfig]):
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
)
@property
diff --git a/products/warehouse_sources/sources/kandji/tests/test_kandji.py b/products/warehouse_sources/sources/kandji/tests/test_kandji.py
index 0a40d9e340c3..8d2eeb9be8fc 100644
--- a/products/warehouse_sources/sources/kandji/tests/test_kandji.py
+++ b/products/warehouse_sources/sources/kandji/tests/test_kandji.py
@@ -1,3 +1,5 @@
+from typing import Any
+
import pytest
from unittest.mock import Mock, patch
@@ -7,6 +9,7 @@ from parameterized import parameterized
from products.warehouse_sources.sources.kandji._config import KandjiSourceConfig
from products.warehouse_sources.sources.kandji.kandji import (
KandjiNextLinkPaginator,
+ KandjiResumeConfig,
build_base_url,
kandji_source,
validate_credentials,
@@ -93,6 +96,7 @@ class TestKandjiTransport:
endpoint="devices",
team_id=1,
job_id="job-1",
+ resumable_source_manager=Mock(can_resume=Mock(return_value=False)),
)
assert response.name == "devices"
@@ -118,6 +122,67 @@ class TestKandjiTransport:
assert result.response is not None
assert result.response.primary_keys == ["device_id", "bundle_id"]
+ @parameterized.expand(
+ [
+ (
+ "offset_list",
+ "devices",
+ {"offset": 300},
+ [ScriptedResponse(json=[{"device_id": "dev_301"}])],
+ ["https://accuhive.api.kandji.io/api/v1/devices?offset=300&limit=300"],
+ ),
+ (
+ "next_link_list",
+ "users",
+ {"next_params": {"cursor": "abc"}},
+ [ScriptedResponse(json={"results": [{"id": "u2"}], "next": None})],
+ ["https://accuhive.api.kandji.io/api/v1/users?sizePerPage=300&cursor=abc"],
+ ),
+ (
+ "fan_out",
+ "device_apps",
+ {"completed": ["/devices/dev_1/apps"], "current": None, "child_state": None, "parent": None},
+ [
+ ScriptedResponse(json=[{"device_id": "dev_1"}, {"device_id": "dev_2"}]),
+ ScriptedResponse(json={"apps": [{"bundle_id": "com.apple.Safari"}]}),
+ ],
+ [
+ "https://accuhive.api.kandji.io/api/v1/devices?limit=300&offset=0",
+ "https://accuhive.api.kandji.io/api/v1/devices/dev_2/apps?limit=300",
+ ],
+ ),
+ ]
+ )
+ def test_resumes_from_saved_state(
+ self,
+ _name: str,
+ endpoint: str,
+ saved: dict[str, Any],
+ script: list[ScriptedResponse],
+ expected_urls: list[str],
+ ) -> None:
+ result = SourceDriver(
+ KandjiSource(), KandjiSourceConfig(api_token="tok", subdomain="accuhive", region="us")
+ ).run(endpoint, script, resume_state=KandjiResumeConfig(paginator_state=saved))
+
+ assert result.raised is None
+ assert result.urls == expected_urls
+ assert len(result.rows) == 1
+
+ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:
+ driver = SourceDriver(KandjiSource(), KandjiSourceConfig(api_token="tok", subdomain="accuhive", region="us"))
+
+ result = driver.run("users", [ScriptedResponse(json={"results": [{"id": "u1"}], "next": None})])
+
+ assert result.raised is None
+ assert result.saved_states[-1] == KandjiResumeConfig(completed=True)
+
+ resumed = driver.run("users", [], resume_state=KandjiResumeConfig(completed=True))
+
+ assert resumed.raised is None
+ assert resumed.urls == []
+ assert resumed.rows == []
+
class TestKandjiNextLinkPaginator:
def test_copies_next_query_onto_original_request(self) -> None:
diff --git a/products/warehouse_sources/sources/kernel/kernel.py b/products/warehouse_sources/sources/kernel/kernel.py
index 527df5ee08e0..83b089ef84a0 100644
--- a/products/warehouse_sources/sources/kernel/kernel.py
+++ b/products/warehouse_sources/sources/kernel/kernel.py
@@ -19,8 +19,13 @@ from products.warehouse_sources.sources.kernel.settings import (
TELEMETRY_RETENTION_DAYS,
KernelEndpointConfig,
)
-from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline
-from products.warehouse_sources.sources.sdk.internals import Batcher
+from products.warehouse_sources.sources.sdk import (
+ ResumableSourceManager,
+ SourceResponse,
+ frozen,
+ make_tracked_session,
+ retry_with_deadline,
+)
KERNEL_BASE_URL = "https://api.onkernel.com"
@@ -33,6 +38,20 @@ AUDIT_LOG_MIN_WINDOW = timedelta(hours=1)
AUDIT_LOG_RETENTION = timedelta(days=365)
+@frozen
+class KernelResumeConfig:
+ # List endpoints: the offset of the next page.
+ offset: int | None = None
+ # audit_logs: the start of the next time window, as an ISO timestamp.
+ audit_log_start: str | None = None
+ # browser_telemetry_events: the browser session to read next, and the offset cursor of its next
+ # page. Sessions are read in id order, and a None cursor starts the session at its first event.
+ telemetry_session_id: str | None = None
+ telemetry_cursor: int | None = None
+ # List and telemetry endpoints: the last page is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
class KernelRetryableError(Exception):
pass
@@ -228,8 +247,14 @@ def _fetch_audit_log_page(
def _iter_audit_log_windows(
- session: requests.Session, headers: dict[str, str], start: datetime, end: datetime, logger: FilteringBoundLogger
-) -> Iterator[list[dict[str, Any]]]:
+ session: requests.Session,
+ headers: dict[str, str],
+ start: datetime,
+ end: datetime,
+ logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[KernelResumeConfig],
+) -> Iterator[tuple[datetime, list[dict[str, Any]]]]:
+ """Yield the end of each window with its rows, oldest window first. A window with no rows is yielded too."""
windows: list[tuple[datetime, datetime]] = []
window_start = start
while window_start < end:
@@ -243,52 +268,57 @@ def _iter_audit_log_windows(
midpoint = window_start + (window_end - window_start) / 2
pending.append((midpoint, window_end))
pending.append((window_start, midpoint))
+ # The rows of the window are dropped and read again in the halves, so none is held.
+ manager.safe_point()
continue
while next_token is not None:
page, next_token = _fetch_audit_log_page(session, headers, window_start, window_end, next_token, logger)
rows.extend(page)
- if rows:
- rows.sort(key=lambda row: _parse_datetime(row.get("timestamp")) or datetime.min.replace(tzinfo=UTC))
- yield _add_audit_log_ids(rows)
+ rows.sort(key=lambda row: _parse_datetime(row.get("timestamp")) or datetime.min.replace(tzinfo=UTC))
+ yield window_end, _add_audit_log_ids(rows)
def get_audit_log_rows(
- api_key: str, logger: FilteringBoundLogger, db_incremental_field_last_value: Optional[Any] = None
+ api_key: str,
+ logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[KernelResumeConfig],
+ db_incremental_field_last_value: Optional[Any] = None,
) -> Iterator[Any]:
headers = _get_headers(api_key)
- batcher = Batcher(logger=logger, chunk_size=2000, chunk_size_bytes=100 * 1024 * 1024)
session = make_tracked_session(capture=False)
end = datetime.now(UTC)
start = end - AUDIT_LOG_RETENTION
last_value = _parse_datetime(db_incremental_field_last_value)
if last_value is not None and last_value > start:
start = last_value
- for rows in _iter_audit_log_windows(session, headers, start, end, logger):
- for row in rows:
- batcher.batch(row)
- if batcher.should_yield():
- yield batcher.get_table()
- if batcher.should_yield(include_incomplete_chunk=True):
- yield batcher.get_table()
+ resume = manager.load_state() if manager.can_resume() else None
+ resume_start = _parse_datetime(resume.audit_log_start) if resume is not None else None
+ if resume_start is not None and resume_start > start:
+ start = resume_start
+ for window_end, rows in _iter_audit_log_windows(session, headers, start, end, logger, manager):
+ manager.save_state(KernelResumeConfig(audit_log_start=_format_datetime(window_end)))
+ if rows:
+ # One item per window, so a window and the cursor past it commit together.
+ yield rows
+ else:
+ manager.safe_point()
-def _iter_list_items(
+def _iter_list_pages(
session: requests.Session,
headers: dict[str, str],
config: KernelEndpointConfig,
logger: FilteringBoundLogger,
-) -> Iterator[Any]:
- # Full refresh only: no resumable offset state. A crashed sync restarts from offset 0 and
- # the pipeline overwrites the table on the first chunk, so re-fetched pages never duplicate
- # rows (full-refresh appends have no primary-key dedupe, so resuming mid-table would).
- offset = 0
+ offset: int = 0,
+) -> Iterator[tuple[list[Any], int | None]]:
+ """Yield each page with the offset of the page after it, or None after the last page."""
while True:
params: dict[str, Any] = {"limit": PAGE_SIZE, "offset": offset, **config.extra_params}
url = _build_url(config.path, params)
try:
response = _fetch_page(session, url, headers, logger, empty_on_error_code=config.empty_on_error_code)
except KernelResourceDisabledError:
- break
+ return
items = _extract_items(response.json())
has_more, next_offset = _next_page(response.headers, offset, len(items))
@@ -298,14 +328,15 @@ def _iter_list_items(
# via X-Has-More. Keep going, but stop if the offset can't advance (an empty page
# with no X-Next-Offset would otherwise loop forever on the same request).
if not has_more or next_offset <= offset:
- break
+ return
+ yield [], next_offset
offset = next_offset
continue
- yield from items
+ yield items, next_offset if has_more else None
if not has_more:
- break
+ return
offset = next_offset
@@ -325,24 +356,29 @@ def _telemetry_session_ids(
headers: dict[str, str],
retention_start: datetime,
logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[KernelResumeConfig],
) -> list[str]:
- """Session ids of every browser that can still have retained telemetry events.
+ """Session ids of every browser that can still have retained telemetry events, in id order.
Ids are collected before fanning out so the offset-paginated /browsers walk isn't stretched
- across thousands of child requests while new sessions shift its pages.
+ across thousands of child requests while new sessions shift its pages. The id order is the
+ order a resume relies on.
"""
session_ids: list[str] = []
- for browser in _iter_list_items(session, headers, KERNEL_ENDPOINTS["browsers"], logger):
- if not isinstance(browser, dict):
- continue
- session_id = browser.get("session_id")
- if not isinstance(session_id, str) or not session_id:
- continue
- deleted_at = _parse_timestamp(browser.get("deleted_at"))
- if deleted_at is not None and deleted_at < retention_start:
- continue
- session_ids.append(session_id)
- return session_ids
+ for browsers, _next_offset in _iter_list_pages(session, headers, KERNEL_ENDPOINTS["browsers"], logger):
+ for browser in browsers:
+ if not isinstance(browser, dict):
+ continue
+ session_id = browser.get("session_id")
+ if not isinstance(session_id, str) or not session_id:
+ continue
+ deleted_at = _parse_timestamp(browser.get("deleted_at"))
+ if deleted_at is not None and deleted_at < retention_start:
+ continue
+ session_ids.append(session_id)
+ # No row is handed on while the ids are collected, so the run can stop here.
+ manager.safe_point()
+ return sorted(set(session_ids))
def _next_telemetry_cursor(headers: Any) -> int | None:
@@ -371,15 +407,16 @@ def _telemetry_row(session_id: str, envelope: dict[str, Any]) -> dict[str, Any]:
return row
-def _iter_telemetry_events(
+def _iter_telemetry_pages(
session: requests.Session,
headers: dict[str, str],
session_id: str,
since: str,
logger: FilteringBoundLogger,
-) -> Iterator[dict[str, Any]]:
+ cursor: int | None,
+) -> Iterator[tuple[list[dict[str, Any]], int | None]]:
+ """Yield each page of a session's events with the cursor of the page after it, or None after the last."""
path = KERNEL_ENDPOINTS[BROWSER_TELEMETRY_EVENTS].path.format(session_id=quote(session_id, safe=""))
- cursor: int | None = None
while True:
# `since` defaults to the last 5 minutes, so it must be set to read the archive.
# Kernel ignores it once an offset cursor is passed.
@@ -390,15 +427,19 @@ def _iter_telemetry_events(
if response.status_code == 404:
# The session can be purged between listing browsers and reading its events.
logger.debug(f"Kernel browser session {session_id} not found, skipping its telemetry events")
+ yield [], None
return
- for envelope in _extract_items(response.json()):
- if isinstance(envelope, dict):
- yield _telemetry_row(session_id, envelope)
-
+ rows = [
+ _telemetry_row(session_id, envelope)
+ for envelope in _extract_items(response.json())
+ if isinstance(envelope, dict)
+ ]
next_cursor = _next_telemetry_cursor(response.headers)
if next_cursor is None or next_cursor == cursor:
+ yield rows, None
return
+ yield rows, next_cursor
cursor = next_cursor
@@ -406,47 +447,70 @@ def _iter_browser_telemetry_events(
session: requests.Session,
headers: dict[str, str],
logger: FilteringBoundLogger,
-) -> Iterator[dict[str, Any]]:
+ manager: ResumableSourceManager[KernelResumeConfig],
+) -> Iterator[list[dict[str, Any]]]:
retention_start = datetime.now(UTC) - timedelta(days=TELEMETRY_RETENTION_DAYS)
since = retention_start.strftime("%Y-%m-%dT%H:%M:%SZ")
- for session_id in _telemetry_session_ids(session, headers, retention_start, logger):
- yield from _iter_telemetry_events(session, headers, session_id, since, logger)
+ resume = manager.load_state() if manager.can_resume() else None
+ if resume is not None and resume.completed:
+ return
+ resume_session_id = resume.telemetry_session_id if resume is not None else None
+ session_ids = _telemetry_session_ids(session, headers, retention_start, logger, manager)
+ for index, session_id in enumerate(session_ids):
+ if resume_session_id is not None and session_id < resume_session_id:
+ continue
+ cursor = resume.telemetry_cursor if resume is not None and session_id == resume_session_id else None
+ following = session_ids[index + 1] if index + 1 < len(session_ids) else None
+ for rows, next_cursor in _iter_telemetry_pages(session, headers, session_id, since, logger, cursor):
+ if next_cursor is not None:
+ manager.save_state(KernelResumeConfig(telemetry_session_id=session_id, telemetry_cursor=next_cursor))
+ elif following is not None:
+ manager.save_state(KernelResumeConfig(telemetry_session_id=following))
+ else:
+ manager.save_state(KernelResumeConfig(completed=True))
+ if rows:
+ yield rows
+ else:
+ manager.safe_point()
def get_rows(
api_key: str,
endpoint: str,
logger: FilteringBoundLogger,
+ manager: ResumableSourceManager[KernelResumeConfig],
) -> Iterator[Any]:
config = KERNEL_ENDPOINTS[endpoint]
headers = _get_headers(api_key)
- batcher = Batcher(logger=logger, chunk_size=2000, chunk_size_bytes=100 * 1024 * 1024)
# One session reused across every page so urllib3 keeps the connection alive.
# capture=False: Kernel responses carry secret-bearing fields (see SENSITIVE_FIELDS) that the
# generic HTTP-sample scrubber does not know to redact, and sampling happens before redaction.
session = make_tracked_session(capture=False)
+ # Each page goes out as its own item, so the cursor staged for a page covers every row handed on.
if endpoint == BROWSER_TELEMETRY_EVENTS:
- items: Iterator[Any] = _iter_browser_telemetry_events(session, headers, logger)
- else:
- items = (
- _redact_sensitive_fields(item, config.nested_sensitive_fields)
- for item in _iter_list_items(session, headers, config, logger)
+ yield from _iter_browser_telemetry_events(session, headers, logger, manager)
+ return
+
+ resume = manager.load_state() if manager.can_resume() else None
+ if resume is not None and resume.completed:
+ return
+ offset = resume.offset if resume is not None and resume.offset is not None else 0
+ for items, next_offset in _iter_list_pages(session, headers, config, logger, offset):
+ manager.save_state(
+ KernelResumeConfig(offset=next_offset) if next_offset is not None else KernelResumeConfig(completed=True)
)
-
- for item in items:
- batcher.batch(item)
- if batcher.should_yield():
- yield batcher.get_table()
-
- if batcher.should_yield(include_incomplete_chunk=True):
- yield batcher.get_table()
+ if items:
+ yield [_redact_sensitive_fields(item, config.nested_sensitive_fields) for item in items]
+ else:
+ manager.safe_point()
def kernel_source(
api_key: str,
endpoint: str,
logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[KernelResumeConfig],
db_incremental_field_last_value: Optional[Any] = None,
) -> SourceResponse:
endpoint_config: KernelEndpointConfig = KERNEL_ENDPOINTS[endpoint]
@@ -455,13 +519,15 @@ def kernel_source(
get_audit_log_rows,
api_key=api_key,
logger=logger,
+ manager=resumable_source_manager,
db_incremental_field_last_value=db_incremental_field_last_value,
)
if endpoint == AUDIT_LOGS_ENDPOINT
- else partial(get_rows, api_key=api_key, endpoint=endpoint, logger=logger)
+ else partial(get_rows, api_key=api_key, endpoint=endpoint, logger=logger, manager=resumable_source_manager)
)
return SourceResponse(
name=endpoint,
+ on_complete=resumable_source_manager.clear_state,
items=items,
primary_keys=endpoint_config.primary_keys,
sort_mode="asc",
diff --git a/products/warehouse_sources/sources/kernel/source.py b/products/warehouse_sources/sources/kernel/source.py
index 853b1e8fe9eb..2ca17259d446 100644
--- a/products/warehouse_sources/sources/kernel/source.py
+++ b/products/warehouse_sources/sources/kernel/source.py
@@ -2,6 +2,7 @@ from typing import Optional, cast
from products.warehouse_sources.sources.kernel._config import KernelSourceConfig
from products.warehouse_sources.sources.kernel.kernel import (
+ KernelResumeConfig,
kernel_source,
validate_credentials as validate_kernel_credentials,
)
@@ -12,7 +13,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -25,7 +27,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class KernelSource(SimpleSource[KernelSourceConfig]):
+class KernelSource(ResumableSource[KernelSourceConfig, KernelResumeConfig]):
codebase_detection = detect(
env=("^KERNEL_API_KEY$", "^KERNEL_ORG_API_KEY$"),
npm=("@onkernel/sdk",),
@@ -128,11 +130,20 @@ You can create an API key in your [Kernel dashboard](https://dashboard.onkernel.
return False, "Invalid Kernel API key"
- def source_for_pipeline(self, config: KernelSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[KernelResumeConfig]:
+ return ResumableSourceManager[KernelResumeConfig](inputs, KernelResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: KernelSourceConfig,
+ resumable_source_manager: ResumableSourceManager[KernelResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return kernel_source(
api_key=config.api_key,
endpoint=inputs.schema_name,
logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
db_incremental_field_last_value=(
inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
),
diff --git a/products/warehouse_sources/sources/kernel/tests/test_kernel.py b/products/warehouse_sources/sources/kernel/tests/test_kernel.py
index 47048a0330c3..db5c6521e7c1 100644
--- a/products/warehouse_sources/sources/kernel/tests/test_kernel.py
+++ b/products/warehouse_sources/sources/kernel/tests/test_kernel.py
@@ -1,4 +1,3 @@
-import json
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -13,6 +12,7 @@ from products.warehouse_sources.sources.kernel._config import KernelSourceConfig
from products.warehouse_sources.sources.kernel.kernel import (
AUDIT_LOG_MAX_WINDOW,
PAGE_SIZE,
+ KernelResumeConfig,
KernelRetryableError,
KernelUnexpectedResponseError,
_extract_items,
@@ -37,10 +37,6 @@ def _driver() -> SourceDriver:
return SourceDriver(KernelSource(), KernelSourceConfig(api_key="sk_test"))
-def _rows(result: DriveResult) -> list[dict[str, Any]]:
- return [row for table in result.rows for row in table.to_pylist()]
-
-
def _response(
items: Any, *, status_code: int = 200, has_more: bool | None = None, next_offset: int | None = None
) -> ScriptedResponse:
@@ -118,13 +114,16 @@ class TestGetRows:
)
assert result.raised is None
- assert _rows(result) == [{"id": "a1"}, {"id": "a2"}]
+ # Row lists, not tables, so the pipeline batches pages together.
+ assert result.items == [[{"id": "a1"}], [{"id": "a2"}]]
assert result.params("offset") == ["0", "100"]
+ # The last page carries a terminal cursor, so a hand-off after it reads nothing again.
+ assert result.committed_states == [KernelResumeConfig(offset=100), KernelResumeConfig(completed=True)]
def test_empty_first_page_yields_nothing(self) -> None:
result = _driver().run("profiles", [_response([], has_more=False)])
assert result.raised is None
- assert _rows(result) == []
+ assert result.rows == []
def test_empty_page_with_more_pages_keeps_paging(self) -> None:
# An empty page that still reports X-Has-More must not end the sync early.
@@ -136,7 +135,7 @@ class TestGetRows:
],
)
assert result.raised is None
- assert _rows(result) == [{"id": "a1"}]
+ assert result.rows == [{"id": "a1"}]
assert result.params("offset") == ["0", "100"]
@pytest.mark.parametrize(
@@ -150,13 +149,79 @@ class TestGetRows:
assert isinstance(result.raised, requests.HTTPError)
else:
assert result.raised is None
- assert _rows(result) == []
+ assert result.rows == []
def test_unexpected_response_shape_raises(self) -> None:
result = _driver().run("apps", [_response({"unexpected": [{"id": "a1"}]}, has_more=False)])
assert isinstance(result.raised, KernelUnexpectedResponseError)
+ @pytest.mark.parametrize(
+ "endpoint, saved, responses, expected_paths, expected_offsets, row_key, expected_keys",
+ [
+ pytest.param(
+ "deployments",
+ KernelResumeConfig(offset=100),
+ {"/deployments": [_response([{"id": "a2"}], has_more=False)]},
+ ["/deployments"],
+ ["100"],
+ "id",
+ ["a2"],
+ id="list_starts_at_saved_offset",
+ ),
+ pytest.param(
+ "browser_telemetry_events",
+ KernelResumeConfig(telemetry_session_id="b2", telemetry_cursor=987),
+ {
+ "/browsers": [_response([{"session_id": "b3"}, {"session_id": "b1"}, {"session_id": "b2"}])],
+ "/browsers/b2/telemetry/events": [_response([{"seq": 2, "event": {}}], has_more=False)],
+ "/browsers/b3/telemetry/events": [_response([{"seq": 3, "event": {}}], has_more=False)],
+ },
+ ["/browsers", "/browsers/b2/telemetry/events", "/browsers/b3/telemetry/events"],
+ ["0", "987", None],
+ "seq",
+ [2, 3],
+ id="telemetry_skips_finished_sessions_and_resumes_the_saved_one",
+ ),
+ pytest.param(
+ "deployments",
+ KernelResumeConfig(completed=True),
+ {},
+ [],
+ [],
+ "id",
+ [],
+ id="list_after_its_last_page_reads_nothing",
+ ),
+ pytest.param(
+ "browser_telemetry_events",
+ KernelResumeConfig(completed=True),
+ {},
+ [],
+ [],
+ "seq",
+ [],
+ id="telemetry_after_its_last_page_reads_nothing",
+ ),
+ ],
+ )
+ def test_resumes_from_saved_state(
+ self,
+ endpoint: str,
+ saved: KernelResumeConfig,
+ responses: dict[str, list[ScriptedResponse]],
+ expected_paths: list[str],
+ expected_offsets: list[str | None],
+ row_key: str,
+ expected_keys: list[Any],
+ ) -> None:
+ result = _driver().run(endpoint, route(responses), resume_state=saved)
+
+ assert result.raised is None
+ assert result.paths == expected_paths
+ assert result.params("offset") == expected_offsets
+ assert [row[row_key] for row in result.rows] == expected_keys
+
def test_retries_exhausted_raises(self) -> None:
result = _driver().run("apps", always(_response(None, status_code=503)))
@@ -205,7 +270,7 @@ class TestBrowserTelemetryEvents:
)
assert result.raised is None
- assert [{**row, "data": json.loads(row["data"])} for row in _rows(result)] == [
+ assert result.rows == [
{
"browser_session_id": "active",
"seq": 1,
@@ -253,7 +318,7 @@ class TestBrowserTelemetryEvents:
)
assert result.raised is None
- assert [row["seq"] for row in _rows(result)] == [1]
+ assert [row["seq"] for row in result.rows] == [1]
assert len(result.requests) == 2
def test_skips_browser_purged_after_listing(self) -> None:
@@ -267,14 +332,14 @@ class TestBrowserTelemetryEvents:
)
assert result.raised is None
- assert [(row["browser_session_id"], row["seq"]) for row in _rows(result)] == [("b2", 7)]
+ assert [(row["browser_session_id"], row["seq"]) for row in result.rows] == [("b2", 7)]
class TestKernelSourceResponse:
@pytest.mark.parametrize("endpoint", list(ENDPOINTS))
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = KERNEL_ENDPOINTS[endpoint]
- response = kernel_source("sk_test", endpoint, mock.MagicMock())
+ response = kernel_source("sk_test", endpoint, mock.MagicMock(), resumable_source_manager=mock.MagicMock())
assert response.name == endpoint
assert response.primary_keys == config.primary_keys
@@ -341,7 +406,7 @@ class TestAuditLogRows:
_audit_record(_NOW - timedelta(hours=2), path="/apps"),
]
result = _driver().run("audit_logs", _fake_audit_log_api(records))
- rows = _rows(result)
+ rows = result.rows
assert result.raised is None
assert len(rows) == len(records)
@@ -366,23 +431,29 @@ class TestAuditLogRows:
db_incremental_field_last_value=_NOW - timedelta(hours=2),
)
assert rerun.raised is None
- assert {r["id"] for r in _rows(rerun)} == {r["id"] for r in rows[-2:]}
+ assert {r["id"] for r in rerun.rows} == {r["id"] for r in rows[-2:]}
@pytest.mark.parametrize(
- "last_value, expected_start",
+ "last_value, resume_start, expected_start",
[
- (None, _NOW - timedelta(days=365)),
- (_NOW - timedelta(days=2), _NOW - timedelta(days=2)),
- ("2026-05-30T00:00:00Z", datetime(2026, 5, 30, tzinfo=UTC)),
+ (None, None, _NOW - timedelta(days=365)),
+ (_NOW - timedelta(days=2), None, _NOW - timedelta(days=2)),
+ ("2026-05-30T00:00:00Z", None, datetime(2026, 5, 30, tzinfo=UTC)),
# Kernel keeps one year of audit logs, so an older watermark starts at the retention edge.
- (_NOW - timedelta(days=500), _NOW - timedelta(days=365)),
+ (_NOW - timedelta(days=500), None, _NOW - timedelta(days=365)),
+ # A resumed run starts after the last window it handed on.
+ (None, "2026-05-10T00:00:00.000000Z", datetime(2026, 5, 10, tzinfo=UTC)),
+ (_NOW - timedelta(days=2), "2026-05-10T00:00:00.000000Z", _NOW - timedelta(days=2)),
],
)
@time_machine.travel(_NOW, tick=False)
- def test_search_starts_at_watermark_or_retention_edge(self, last_value: Any, expected_start: datetime) -> None:
+ def test_search_starts_at_watermark_or_retention_edge(
+ self, last_value: Any, resume_start: str | None, expected_start: datetime
+ ) -> None:
result = _driver().run(
"audit_logs",
always(_response([], has_more=False)),
+ resume_state=KernelResumeConfig(audit_log_start=resume_start) if resume_start else None,
incremental_field="timestamp" if last_value is not None else None,
db_incremental_field_last_value=last_value,
)
diff --git a/products/warehouse_sources/sources/my_hours/my_hours.py b/products/warehouse_sources/sources/my_hours/my_hours.py
index eadbe73a3b53..e9b25083788e 100644
--- a/products/warehouse_sources/sources/my_hours/my_hours.py
+++ b/products/warehouse_sources/sources/my_hours/my_hours.py
@@ -17,12 +17,12 @@ from products.warehouse_sources.sources.sdk import (
Resource,
RESTAPIConfig,
RESTClient,
+ ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
build_dependent_resource,
frozen,
make_tracked_session,
- reach_safe_point,
rest_api_resource,
)
@@ -63,6 +63,14 @@ def _client_config(api_key: str) -> ClientConfig:
}
+@frozen
+class MyHoursResumeConfig:
+ # time_logs: the first day of the next date window, as an ISO date.
+ next_window_start: str | None = None
+ # Fan-out endpoints: the finished and in-progress parents, as the framework hands them over.
+ paginator_state: dict[str, Any] | None = None
+
+
@frozen
class _TimeLogWindow:
date_from: date
@@ -88,9 +96,17 @@ def _time_log_windows(start: date, today: date) -> Iterator[_TimeLogWindow]:
start = end + timedelta(days=1)
-def _time_log_pages(client: RESTClient, start: date) -> Iterator[list[dict[str, Any]]]:
+def _time_log_pages(
+ client: RESTClient, start: date, manager: ResumableSourceManager[MyHoursResumeConfig]
+) -> Iterator[list[dict[str, Any]]]:
+ resume = manager.load_state() if manager.can_resume() else None
# One day past today in UTC covers accounts in timezones already on the next day.
today = datetime.now(UTC).date() + timedelta(days=1)
+ if resume is not None and resume.next_window_start:
+ start = date.fromisoformat(resume.next_window_start)
+ if start > today:
+ # The last window is handed on. The window walk would clamp to today and read it again.
+ return
for window in _time_log_windows(start, today):
rows: list[dict[str, Any]] = []
for page in client.paginate(
@@ -113,8 +129,11 @@ def _time_log_pages(client: RESTClient, start: date) -> Iterator[list[dict[str,
data_selector_malformed_retryable=True,
):
rows.extend(page)
+ # Each window is one request, so the next window is where a resume starts.
+ next_start = window.date_to + timedelta(days=1)
+ manager.save_state(MyHoursResumeConfig(next_window_start=next_start.isoformat()))
if not rows:
- reach_safe_point()
+ manager.safe_point()
continue
# The report's row order is not documented. Sorting each window keeps the incremental
# watermark ascending across batches.
@@ -141,6 +160,7 @@ def my_hours_source(
endpoint: str,
team_id: int,
job_id: str,
+ resumable_source_manager: ResumableSourceManager[MyHoursResumeConfig],
should_use_incremental_field: bool = False,
db_incremental_field_last_value: Any = None,
) -> SourceResponse:
@@ -156,13 +176,20 @@ def my_hours_source(
start = _start_date(db_incremental_field_last_value if should_use_incremental_field else None)
return SourceResponse(
name=endpoint,
- items=lambda: _time_log_pages(client, start),
+ on_complete=resumable_source_manager.clear_state,
+ items=lambda: _time_log_pages(client, start, resumable_source_manager),
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
)
if config.fanout is not None:
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state:
+ resumable_source_manager.save_state(MyHoursResumeConfig(paginator_state=state))
+
resource = cast(
Resource,
build_dependent_resource(
@@ -175,12 +202,15 @@ def my_hours_source(
job_id=job_id,
db_incremental_field_last_value=None,
page_size_param=None,
+ resume_hook=save_checkpoint,
+ initial_paginator_state=resume.paginator_state if resume is not None else None,
),
)
if endpoint == PROJECT_TASKS_ENDPOINT:
resource = resource.add_map(_flatten_task_lists)
return SourceResponse(
name=endpoint,
+ on_complete=resumable_source_manager.clear_state,
items=lambda: resource,
primary_keys=config.primary_keys,
partition_count=1,
@@ -208,6 +238,7 @@ def my_hours_source(
return SourceResponse(
name=endpoint,
+ on_complete=resumable_source_manager.clear_state,
items=lambda: resource,
primary_keys=config.primary_keys,
partition_count=1,
diff --git a/products/warehouse_sources/sources/my_hours/source.py b/products/warehouse_sources/sources/my_hours/source.py
index 0b3c2b1ab011..35ff0102c133 100644
--- a/products/warehouse_sources/sources/my_hours/source.py
+++ b/products/warehouse_sources/sources/my_hours/source.py
@@ -1,7 +1,7 @@
from typing import Optional, cast
from products.warehouse_sources.sources.my_hours._config import MyHoursSourceConfig
-from products.warehouse_sources.sources.my_hours.my_hours import check_access, my_hours_source
+from products.warehouse_sources.sources.my_hours.my_hours import MyHoursResumeConfig, check_access, my_hours_source
from products.warehouse_sources.sources.my_hours.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
@@ -18,7 +18,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -32,7 +33,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class MyHoursSource(SimpleSource[MyHoursSourceConfig]):
+class MyHoursSource(ResumableSource[MyHoursSourceConfig, MyHoursResumeConfig]):
codebase_detection = detect(env=("^(MY_HOURS|MYHOURS)_",))
api_docs_url = "https://documenter.getpostman.com/view/8879268/TVmV4YYU"
@@ -129,7 +130,15 @@ API keys are available on paid plans. Create one under **Settings → Integratio
return False, "Invalid My Hours API key"
return False, message or "Could not validate My Hours API key"
- def source_for_pipeline(self, config: MyHoursSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[MyHoursResumeConfig]:
+ return ResumableSourceManager[MyHoursResumeConfig](inputs, MyHoursResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: MyHoursSourceConfig,
+ resumable_source_manager: ResumableSourceManager[MyHoursResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
if inputs.schema_name not in MY_HOURS_ENDPOINTS:
raise ValueError(f"Unknown My Hours schema '{inputs.schema_name}'")
@@ -138,6 +147,7 @@ API keys are available on paid plans. Create one under **Settings → Integratio
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)
diff --git a/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py b/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py
index a72614f0b262..7214972b137d 100644
--- a/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py
+++ b/products/warehouse_sources/sources/my_hours/tests/test_my_hours.py
@@ -6,7 +6,12 @@ import requests
from parameterized import parameterized
from products.warehouse_sources.sources.my_hours._config import MyHoursSourceConfig
-from products.warehouse_sources.sources.my_hours.my_hours import MY_HOURS_BASE_URL, MyHoursApiKeyAuth, check_access
+from products.warehouse_sources.sources.my_hours.my_hours import (
+ MY_HOURS_BASE_URL,
+ MyHoursApiKeyAuth,
+ MyHoursResumeConfig,
+ check_access,
+)
from products.warehouse_sources.sources.my_hours.settings import ENDPOINTS, MY_HOURS_ENDPOINTS, TEAMS_UNAVAILABLE_ERROR
from products.warehouse_sources.sources.my_hours.source import MyHoursSource
from products.warehouse_sources.sources.sdk import RESTClientRetryableError
@@ -143,9 +148,12 @@ class TestMyHoursSourceResponse:
class TestTimeLogs:
@parameterized.expand(
[
- ("full_refresh", False, None, [("2010-01-01", "2010-01-31"), ("2010-02-01", "2010-02-21")]),
- ("incremental_from_watermark", True, datetime(2010, 2, 5), [("2010-02-05", "2010-02-21")]),
- ("incremental_watermark_after_today", True, datetime(2010, 3, 10), [("2010-02-21", "2010-02-21")]),
+ ("full_refresh", False, None, None, [("2010-01-01", "2010-01-31"), ("2010-02-01", "2010-02-21")]),
+ ("incremental_from_watermark", True, datetime(2010, 2, 5), None, [("2010-02-05", "2010-02-21")]),
+ ("incremental_watermark_after_today", True, datetime(2010, 3, 10), None, [("2010-02-21", "2010-02-21")]),
+ ("resumed_full_refresh", False, None, "2010-02-01", [("2010-02-01", "2010-02-21")]),
+ ("resumed_incremental", True, datetime(2010, 1, 5), "2010-02-04", [("2010-02-04", "2010-02-21")]),
+ ("resumed_after_last_window", False, None, "2010-02-22", []),
]
)
@time_machine.travel("2010-02-20 12:00:00", tick=False)
@@ -154,16 +162,26 @@ class TestTimeLogs:
_name: str,
incremental: bool,
last_value: datetime | None,
+ resume_from: str | None,
expected_windows: list[tuple[str, str]],
) -> None:
inputs = {"incremental_field": "date", "db_incremental_field_last_value": last_value} if incremental else {}
+ resume_state = MyHoursResumeConfig(next_window_start=resume_from) if resume_from else None
- result = _driver().run("time_logs", [ScriptedResponse(json={"data": []}) for _ in expected_windows], **inputs)
+ result = _driver().run(
+ "time_logs",
+ [ScriptedResponse(json={"data": []}) for _ in expected_windows],
+ resume_state=resume_state,
+ **inputs,
+ )
assert result.raised is None
assert all(r.method == "PUT" and r.url == f"{MY_HOURS_BASE_URL}/Reports/activitydx" for r in result.requests)
options = [r.json()["options"] for r in result.requests]
assert [(o["dateFrom"], o["dateTo"]) for o in options] == expected_windows
+ # Each window stages the start of the next one, so a resume never reads a finished window again.
+ assert result.final_state == MyHoursResumeConfig(next_window_start="2010-02-22")
+ assert result.completed_state is None
@time_machine.travel("2010-02-20 12:00:00", tick=False)
def test_yields_each_non_empty_window_sorted_by_date(self) -> None:
diff --git a/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py b/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py
index aad273de2123..78dd6be90012 100644
--- a/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py
+++ b/products/warehouse_sources/sources/my_hours/tests/test_my_hours_source.py
@@ -34,9 +34,11 @@ class TestMyHoursSource:
def test_source_for_pipeline_plumbs_arguments(self, mock_source: mock.MagicMock) -> None:
inputs = _make_inputs(schema_name="clients")
- self.source.source_for_pipeline(self.config, inputs)
+ manager = mock.MagicMock()
+ self.source.source_for_pipeline(self.config, manager, inputs)
mock_source.assert_called_once()
kwargs = mock_source.call_args.kwargs
assert kwargs["api_key"] == "mh-key"
assert kwargs["endpoint"] == "clients"
+ assert kwargs["resumable_source_manager"] is manager
diff --git a/products/warehouse_sources/sources/optimizely/optimizely.py b/products/warehouse_sources/sources/optimizely/optimizely.py
index 7145f3eb757a..d1d711569f30 100644
--- a/products/warehouse_sources/sources/optimizely/optimizely.py
+++ b/products/warehouse_sources/sources/optimizely/optimizely.py
@@ -1,3 +1,6 @@
+from collections.abc import Callable
+from typing import Any
+
from products.warehouse_sources.sources.optimizely.settings import OPTIMIZELY_ENDPOINTS, OptimizelyEndpointConfig
from products.warehouse_sources.sources.sdk import (
ClientConfig,
@@ -6,8 +9,10 @@ from products.warehouse_sources.sources.sdk import (
Resource,
ResponseAction,
RESTAPIConfig,
+ ResumableSourceManager,
SinglePagePaginator,
SourceResponse,
+ frozen,
make_tracked_session,
rest_api_resource,
rest_api_resources,
@@ -19,6 +24,16 @@ OPTIMIZELY_BASE_URL = f"https://{OPTIMIZELY_API_HOST}/v2"
# Optimizely list pages cap at 100 items.
PAGE_SIZE = 100
+
+@frozen
+class OptimizelyResumeConfig:
+ # What the framework handed the resume hook: the next `Link` URL of a list endpoint, or the
+ # finished and in-progress projects of a project fan-out. Stored unchanged.
+ paginator_state: dict[str, Any] | None = None
+ # The last page of a list endpoint is handed on, so a resume reads nothing again.
+ completed: bool = False
+
+
_SKIP_INACCESSIBLE: list[ResponseAction] = [
{"status_code": 400, "action": "ignore"},
{"status_code": 403, "action": "ignore"},
@@ -42,7 +57,12 @@ def _client_config(api_token: str) -> ClientConfig:
def _simple_source(
- api_token: str, endpoint: str, config: OptimizelyEndpointConfig, team_id: int, job_id: str
+ api_token: str,
+ endpoint: str,
+ config: OptimizelyEndpointConfig,
+ team_id: int,
+ job_id: str,
+ manager: ResumableSourceManager[OptimizelyResumeConfig],
) -> Resource:
rest_config: RESTAPIConfig = {
"client": _client_config(api_token),
@@ -59,7 +79,14 @@ def _simple_source(
}
],
}
- return rest_api_resource(rest_config, team_id, job_id, None)
+ return rest_api_resource(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=_resume_hook(manager),
+ initial_paginator_state=_initial_state(manager),
+ )
def _project_scoped_resource(name: str, path: str) -> EndpointResource:
@@ -107,7 +134,12 @@ def _per_parent_resource(endpoint: str, path: str, parent: str, parent_id_param:
def _fan_out_source(
- api_token: str, endpoint: str, config: OptimizelyEndpointConfig, team_id: int, job_id: str
+ api_token: str,
+ endpoint: str,
+ config: OptimizelyEndpointConfig,
+ team_id: int,
+ job_id: str,
+ manager: ResumableSourceManager[OptimizelyResumeConfig],
) -> Resource:
resources: list[str | EndpointResource] = [
{
@@ -129,22 +161,55 @@ def _fan_out_source(
"resource_defaults": {},
"resources": resources,
}
- built = rest_api_resources(rest_config, team_id, job_id, None)
+ # The framework resumes a fan-out with one dependent level only. The per-parent results tables
+ # chain two levels, so they read every project again after a resume (the merge dedupes).
+ built = rest_api_resources(
+ rest_config,
+ team_id,
+ job_id,
+ None,
+ resume_hook=_resume_hook(manager),
+ initial_paginator_state=_initial_state(manager),
+ )
# Only the requested endpoint's rows are emitted; its ancestors are iterated internally to drive the fan-out.
return next(resource for resource in built if resource.name == endpoint)
-def optimizely_source(api_token: str, endpoint: str, team_id: int, job_id: str) -> SourceResponse:
+def _initial_state(manager: ResumableSourceManager[OptimizelyResumeConfig]) -> dict[str, Any] | None:
+ resume = manager.load_state() if manager.can_resume() else None
+ return resume.paginator_state if resume is not None else None
+
+
+def _resume_hook(manager: ResumableSourceManager[OptimizelyResumeConfig]) -> Callable[[dict[str, Any] | None], None]:
+ def save_checkpoint(state: dict[str, Any] | None) -> None:
+ if state:
+ manager.save_state(OptimizelyResumeConfig(paginator_state=state))
+ elif state is None:
+ manager.save_state(OptimizelyResumeConfig(completed=True))
+
+ return save_checkpoint
+
+
+def optimizely_source(
+ api_token: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[OptimizelyResumeConfig],
+) -> SourceResponse:
config = OPTIMIZELY_ENDPOINTS[endpoint]
+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
+ completed = resume is not None and resume.completed
if config.project_scoped:
- resource = _fan_out_source(api_token, endpoint, config, team_id, job_id)
+ resource = _fan_out_source(api_token, endpoint, config, team_id, job_id, resumable_source_manager)
else:
- resource = _simple_source(api_token, endpoint, config, team_id, job_id)
+ resource = _simple_source(api_token, endpoint, config, team_id, job_id, resumable_source_manager)
return SourceResponse(
name=endpoint,
- items=lambda: resource,
+ on_complete=resumable_source_manager.clear_state,
+ items=lambda: [] if completed else resource,
primary_keys=[config.primary_key],
partition_count=1,
partition_size=1,
diff --git a/products/warehouse_sources/sources/optimizely/source.py b/products/warehouse_sources/sources/optimizely/source.py
index 5647609f7c78..d444dbc49ee8 100644
--- a/products/warehouse_sources/sources/optimizely/source.py
+++ b/products/warehouse_sources/sources/optimizely/source.py
@@ -2,6 +2,7 @@ from typing import Optional, cast
from products.warehouse_sources.sources.optimizely._config import OptimizelySourceConfig
from products.warehouse_sources.sources.optimizely.optimizely import (
+ OptimizelyResumeConfig,
optimizely_source,
validate_credentials as validate_optimizely_credentials,
)
@@ -13,7 +14,8 @@ from products.warehouse_sources.sources.sdk import (
FieldType,
ProductKey,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -26,7 +28,7 @@ from products.warehouse_sources.sources.sdk import (
@SourceRegistry.register
-class OptimizelySource(SimpleSource[OptimizelySourceConfig]):
+class OptimizelySource(ResumableSource[OptimizelySourceConfig, OptimizelyResumeConfig]):
codebase_detection = detect(
env=("^OPTIMIZELY_", "^NEXT_PUBLIC_OPTIMIZELY_SDK_KEY$", "^EXPO_PUBLIC_OPTIMIZELY_SDK_KEY$"),
script_hosts=("cdn.optimizely.com", "cdn-pci.optimizely.com", "optimizely-edge.com"),
@@ -165,10 +167,19 @@ An admin can generate a personal access token in Optimizely under Account Settin
return False, "Invalid Optimizely personal access token"
- def source_for_pipeline(self, config: OptimizelySourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[OptimizelyResumeConfig]:
+ return ResumableSourceManager[OptimizelyResumeConfig](inputs, OptimizelyResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: OptimizelySourceConfig,
+ resumable_source_manager: ResumableSourceManager[OptimizelyResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return optimizely_source(
api_token=config.api_token,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
+ resumable_source_manager=resumable_source_manager,
)
diff --git a/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py b/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py
index aa8239cd8594..6d558108f019 100644
--- a/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py
+++ b/products/warehouse_sources/sources/optimizely/tests/test_optimizely.py
@@ -1,10 +1,16 @@
from typing import Any
+from unittest.mock import MagicMock
+
import requests
from parameterized import parameterized
from products.warehouse_sources.sources.optimizely._config import OptimizelySourceConfig
-from products.warehouse_sources.sources.optimizely.optimizely import optimizely_source, validate_credentials
+from products.warehouse_sources.sources.optimizely.optimizely import (
+ OptimizelyResumeConfig,
+ optimizely_source,
+ validate_credentials,
+)
from products.warehouse_sources.sources.optimizely.settings import ENDPOINTS, OPTIMIZELY_ENDPOINTS
from products.warehouse_sources.sources.optimizely.source import OptimizelySource
from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, route, scripted_network
@@ -89,6 +95,61 @@ class TestProjectScopedFanOut:
assert result.requests[1].param("project_id") == "11"
+class TestResume:
+ @parameterized.expand(
+ [
+ (
+ "list_endpoint_starts_at_saved_next_link",
+ "projects",
+ {"next_url": "https://api.optimizely.com/v2/projects?page=2&per_page=100"},
+ {"/v2/projects": [_response([{"id": 2}])]},
+ ["https://api.optimizely.com/v2/projects?page=2&per_page=100"],
+ [2],
+ ),
+ (
+ "fan_out_skips_finished_projects",
+ "campaigns",
+ {"completed": ["/campaigns?project_id=11"], "current": None, "child_state": None},
+ {
+ "/v2/projects": [_response([{"id": 11}, {"id": 22}])],
+ "/v2/campaigns": [_response([{"id": "camp-22"}])],
+ },
+ [
+ "https://api.optimizely.com/v2/projects?per_page=100",
+ "https://api.optimizely.com/v2/campaigns?project_id=22&per_page=100",
+ ],
+ ["camp-22"],
+ ),
+ ]
+ )
+ def test_resumes_from_saved_state(
+ self,
+ _name: str,
+ endpoint: str,
+ saved: dict[str, Any],
+ responses: dict[str, list[ScriptedResponse]],
+ expected_urls: list[str],
+ expected_ids: list[Any],
+ ) -> None:
+ result = _driver().run(endpoint, route(responses), resume_state=OptimizelyResumeConfig(paginator_state=saved))
+
+ assert result.raised is None
+ assert result.urls == expected_urls
+ assert [row["id"] for row in result.rows] == expected_ids
+
+ def test_list_endpoint_marks_completion_before_its_last_page(self) -> None:
+ result = _driver().run("projects", route({"/v2/projects": [_response([{"id": 1}])]}))
+
+ assert result.raised is None
+ assert result.saved_states[-1] == OptimizelyResumeConfig(completed=True)
+
+ resumed = _driver().run("projects", route({}), resume_state=OptimizelyResumeConfig(completed=True))
+
+ assert resumed.raised is None
+ assert resumed.urls == []
+ assert resumed.rows == []
+
+
class TestPerParentResults:
@parameterized.expand(
[
@@ -139,7 +200,9 @@ class TestOptimizelySourceResponse:
@parameterized.expand([(endpoint,) for endpoint in ENDPOINTS])
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
config = OPTIMIZELY_ENDPOINTS[endpoint]
- response = optimizely_source("token", endpoint, team_id=1, job_id="j")
+ manager = MagicMock()
+ manager.can_resume.return_value = False
+ response = optimizely_source("token", endpoint, team_id=1, job_id="j", resumable_source_manager=manager)
assert response.name == endpoint
assert response.primary_keys == [config.primary_key]
diff --git a/products/warehouse_sources/sources/zapier_supported_storage/source.py b/products/warehouse_sources/sources/zapier_supported_storage/source.py
index 5ac5c186fd2f..bfb8723f020e 100644
--- a/products/warehouse_sources/sources/zapier_supported_storage/source.py
+++ b/products/warehouse_sources/sources/zapier_supported_storage/source.py
@@ -7,7 +7,8 @@ from products.warehouse_sources.sources.sdk import (
ExternalDataSourceType,
FieldType,
ReleaseStatus,
- SimpleSource,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
@@ -23,6 +24,7 @@ from products.warehouse_sources.sources.zapier_supported_storage.settings import
ZAPIER_SUPPORTED_STORAGE_ENDPOINTS,
)
from products.warehouse_sources.sources.zapier_supported_storage.zapier_supported_storage import (
+ ZapierSupportedStorageResumeConfig,
validate_credentials as validate_zapier_supported_storage_credentials,
zapier_supported_storage_source,
)
@@ -37,7 +39,9 @@ _ENDPOINT_DESCRIPTIONS: dict[str, str] = {
@SourceRegistry.register
-class ZapierSupportedStorageSource(SimpleSource[ZapierSupportedStorageSourceConfig]):
+class ZapierSupportedStorageSource(
+ ResumableSource[ZapierSupportedStorageSourceConfig, ZapierSupportedStorageResumeConfig]
+):
codebase_detection = NOT_DETECTABLE
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
@@ -127,9 +131,20 @@ Only full-refresh syncing is supported: the store has no timestamps, so every sy
) -> tuple[bool, str | None]:
return validate_zapier_supported_storage_credentials(config.secret)
- def source_for_pipeline(self, config: ZapierSupportedStorageSourceConfig, inputs: SourceInputs) -> SourceResponse:
+ def get_resumable_source_manager(
+ self, inputs: SourceInputs
+ ) -> ResumableSourceManager[ZapierSupportedStorageResumeConfig]:
+ return ResumableSourceManager[ZapierSupportedStorageResumeConfig](inputs, ZapierSupportedStorageResumeConfig)
+
+ def source_for_pipeline(
+ self,
+ config: ZapierSupportedStorageSourceConfig,
+ resumable_source_manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig],
+ inputs: SourceInputs,
+ ) -> SourceResponse:
return zapier_supported_storage_source(
secret=config.secret,
endpoint=inputs.schema_name,
logger=inputs.logger,
+ resumable_source_manager=resumable_source_manager,
)
diff --git a/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py b/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py
index e6daee0baad5..811f5114ab0b 100644
--- a/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py
+++ b/products/warehouse_sources/sources/zapier_supported_storage/tests/test_zapier_supported_storage.py
@@ -16,6 +16,7 @@ from products.warehouse_sources.sources.sdk.testing import (
from products.warehouse_sources.sources.zapier_supported_storage._config import ZapierSupportedStorageSourceConfig
from products.warehouse_sources.sources.zapier_supported_storage.source import ZapierSupportedStorageSource
from products.warehouse_sources.sources.zapier_supported_storage.zapier_supported_storage import (
+ ZapierSupportedStorageResumeConfig,
ZapierSupportedStorageRetryableError,
validate_credentials,
zapier_supported_storage_source,
@@ -59,6 +60,28 @@ class TestGetRows:
rows = _rows_from_store({"k": stored_value})
assert rows == [{"key": "k", "value": expected}]
+ @pytest.mark.parametrize(
+ ("resume_state", "expected_keys"),
+ [
+ (None, ["a", "b", "c"]),
+ (ZapierSupportedStorageResumeConfig(last_key="a"), ["b", "c"]),
+ (ZapierSupportedStorageResumeConfig(last_key="c"), []),
+ ],
+ )
+ def test_resume_reads_only_keys_after_the_saved_one(
+ self, resume_state: ZapierSupportedStorageResumeConfig | None, expected_keys: list[str]
+ ) -> None:
+ result = _driver().run("records", [ScriptedResponse(json={"c": 3, "a": 1, "b": 2})], resume_state=resume_state)
+
+ assert result.raised is None
+ assert [row["key"] for table in result.items for row in table.to_pylist()] == expected_keys
+
+ def test_final_batch_stages_its_last_key(self) -> None:
+ result = _driver().run("records", [ScriptedResponse(json={"c": 3, "a": 1, "b": 2})])
+
+ assert result.raised is None
+ assert result.saved_states == [ZapierSupportedStorageResumeConfig(last_key="c")]
+
class TestFetchStore:
@pytest.mark.parametrize("status", [429, 500, 502, 503])
@@ -134,7 +157,9 @@ class TestSourceResponse:
def test_items_is_lazy(self) -> None:
# Building the SourceResponse must not issue any request; only iterating items should.
with scripted_network([]) as network:
- zapier_supported_storage_source(secret="s", endpoint="records", logger=MagicMock())
+ zapier_supported_storage_source(
+ secret="s", endpoint="records", logger=MagicMock(), resumable_source_manager=MagicMock()
+ )
assert network.requests_log == []
assert network.session_options == []
diff --git a/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py b/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py
index fe8255bbbde3..6ea598dbf6d3 100644
--- a/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py
+++ b/products/warehouse_sources/sources/zapier_supported_storage/zapier_supported_storage.py
@@ -5,9 +5,16 @@ from urllib.parse import urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-
-from products.warehouse_sources.sources.sdk import SourceResponse, make_tracked_session, retry_with_deadline
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+
+from products.warehouse_sources.sources.sdk import (
+ ResumableSourceManager,
+ SourceResponse,
+ frozen,
+ make_tracked_session,
+ reach_safe_point,
+ retry_with_deadline,
+)
from products.warehouse_sources.sources.sdk.internals import Batcher
from products.warehouse_sources.sources.zapier_supported_storage.settings import ZAPIER_SUPPORTED_STORAGE_ENDPOINTS
@@ -16,10 +23,21 @@ from products.warehouse_sources.sources.zapier_supported_storage.settings import
ZAPIER_SUPPORTED_STORAGE_URL = "https://store.zapier.com/api/records"
+@frozen
+class ZapierSupportedStorageResumeConfig:
+ # The last key handed on. Keys are read in sorted order, so a resume reads only the keys after it.
+ last_key: str
+
+
class ZapierSupportedStorageRetryableError(Exception):
pass
+def _safe_point_before_retry_wait(_state: RetryCallState) -> None:
+ # The store is read in one call before any row is handed on, so nothing is held during the wait.
+ reach_safe_point()
+
+
def _headers(secret: str) -> dict[str, str]:
return {"X-Secret": secret, "Accept": "application/json"}
@@ -44,6 +62,7 @@ def _stringify_value(value: Any) -> str | None:
),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
+ before_sleep=_safe_point_before_retry_wait,
reraise=True,
)
def _fetch_store(session: requests.Session, secret: str, logger: FilteringBoundLogger) -> dict[str, Any]:
@@ -83,7 +102,9 @@ def _fetch_store(session: requests.Session, secret: str, logger: FilteringBoundL
return data
-def get_rows(secret: str, logger: FilteringBoundLogger) -> Iterator[Any]:
+def get_rows(
+ secret: str, logger: FilteringBoundLogger, manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig]
+) -> Iterator[Any]:
batcher = Batcher(logger=logger, chunk_size=5000, chunk_size_bytes=100 * 1024 * 1024)
# capture=False: the entire response body is the store's arbitrary key/value contents, so HTTP
# sample capture would serialize customer data to object storage where the name-based scrubbers
@@ -94,12 +115,16 @@ def get_rows(secret: str, logger: FilteringBoundLogger) -> Iterator[Any]:
# The store is fetched in a single call - there is no pagination or list endpoint.
store = _fetch_store(session, secret, logger)
- for key, value in store.items():
- batcher.batch({"key": key, "value": _stringify_value(value)})
+ resume = manager.load_state() if manager.can_resume() else None
+ keys = sorted(key for key in store if resume is None or key > resume.last_key)
+ for key in keys:
+ batcher.batch({"key": key, "value": _stringify_value(store[key])})
if batcher.should_yield():
+ manager.save_state(ZapierSupportedStorageResumeConfig(last_key=key))
yield batcher.get_table()
if batcher.should_yield(include_incomplete_chunk=True):
+ manager.save_state(ZapierSupportedStorageResumeConfig(last_key=keys[-1]))
yield batcher.get_table()
@@ -107,12 +132,14 @@ def zapier_supported_storage_source(
secret: str,
endpoint: str,
logger: FilteringBoundLogger,
+ resumable_source_manager: ResumableSourceManager[ZapierSupportedStorageResumeConfig],
) -> SourceResponse:
config = ZAPIER_SUPPORTED_STORAGE_ENDPOINTS[endpoint]
return SourceResponse(
name=endpoint,
- items=lambda: get_rows(secret=secret, logger=logger),
+ on_complete=resumable_source_manager.clear_state,
+ items=lambda: get_rows(secret=secret, logger=logger, manager=resumable_source_manager),
primary_keys=config.primary_keys,
# Full refresh only - the store exposes no timestamps to partition on.
partition_count=1,