:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse_sources): read shopify orders in parallel updated_at windows (#120420)
Daniel RC23e7fbf1cc13
diff --git a/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt b/posthog/test/repo_invariants/source_request_without_timeout_baseline.txtindex 73c2e34562c2..442b8012a4b5 100644--- a/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt+++ b/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt@@ -26,7 +26,6 @@ rokt_ads/rokt_ads.py::RoktAdsClient.getrokt_ads/rokt_ads.py::RoktAdsClient.postsalesforce/auth.py::salesforce_refresh_access_tokenshopify/shopify.py::_get_granted_scopes-shopify/shopify.py::_make_paginated_shopify_request.executeshopify/shopify.py::_probe_resource_permissionshopify/shopify.py::validate_credentialsstytch/stytch.py::_get_session_rowsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.pyindex 6297e2c7e51d..611edecb99d6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py@@ -149,6 +149,15 @@ def _boundaries(settings: Any) -> Iterator[None]:):stack.enter_context(patch(f"{_PROCESSOR}.{target}"))stack.enter_context(patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False))+ # The completion check reads the queue DB, whose tables only exist in suites that create+ # them, so leaving it live makes this one order-dependent on whichever shard it lands in.+ # Every run here hands over all of its batches, which is what count=0 stands for.+ stack.enter_context(+ patch(+ f"{_PROCESSOR}.BatchQueue.get_unfinished_earlier_batches_sync",+ return_value=UnfinishedEarlierBatches(count=0, first_batch_index=None, states=()),+ )+ )stack.enter_context(patch(f"{_PROCESSOR}.finish_row_tracking", new_callable=AsyncMock))# The completion check reads the queue DB, which these tests do not have.stack.enter_context(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.pyindex 9f738f66bcb9..4344bd342a98 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py@@ -49,6 +49,9 @@ SUBSCRIPTION_CONTRACTS = "subscriptionContracts"# page size. An object listed here starts at a page size that is known to fit. The request layer# then grows the page from the cost Shopify reports.SHOPIFY_COST_LIMITED_START_PAGE_SIZES = {ORDERS: 20}+# The objects a full read can split into `updated_at` windows read at once, with the row field that+# holds `updated_at`. Each must sort by `updated_at` and filter on it (see `parallel.py`).+SHOPIFY_PARALLEL_READ_TIME_FIELDS = {ORDERS: "updatedAt"}SHOPIFY_MAX_QUERY_COST = 1000# The largest `first` Shopify accepts on a connection.SHOPIFY_MAX_PAGE_SIZE = 250diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.pynew file mode 100644index 000000000000..739ce1c6cb91--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.py@@ -0,0 +1,204 @@+"""Reads one Shopify object with several requests in flight, each over its own range of `updated_at`.++One paged read sends a request only after the one before it answers, because each page needs the+cursor of the last. A large store can serve more than that: the read uses a fraction of the store's+query cost budget. This reader splits the time range into windows and reads several windows at once.++Shopify returns each window sorted by `updated_at`, so a window that has read up to a time can hand+the rest of its range to another window. When a worker is free, the reader splits the window with the+most time left. A range that holds many rows is split again and again until every worker has work.+"""++import queue+import threading+import contextvars+import dataclasses+from collections.abc import Callable, Iterator+from concurrent.futures import Future, ThreadPoolExecutor+from datetime import datetime, timedelta+from typing import Any++from posthog.dataclasses import frozen++PARALLEL_READ_WORKERS = 3+# A range shorter than this is not split. Shopify filters on whole seconds.+_MIN_SPLIT = timedelta(seconds=2)+_QUEUE_POLL_SECONDS = 1.0++[email protected](frozen=False)+class ReadWindow:+ """A range of `updated_at` that one request chain reads.++ The query covers `lo` to `query_hi`. After a split the window stops at `stop_hi` and leaves the+ rest to the new window, but it keeps its query, because its cursor belongs to that query.+ """++ lo: datetime+ query_hi: datetime+ stop_hi: datetime+ # The cursor after the last page the pipeline received, or None before the first page.+ cursor: str | None = None+ done: bool = False+ # The newest `updated_at` a worker has fetched. A split only happens above it.+ progress: datetime | None = None++ def to_state(self) -> dict[str, Any]:+ return {+ "lo": self.lo.isoformat(),+ "query_hi": self.query_hi.isoformat(),+ "stop_hi": self.stop_hi.isoformat(),+ "cursor": self.cursor,+ "done": self.done,+ "progress": self.progress.isoformat() if self.progress is not None else None,+ }++ @classmethod+ def from_state(cls, state: dict[str, Any]) -> "ReadWindow":+ progress = state.get("progress")+ return cls(+ lo=datetime.fromisoformat(state["lo"]),+ query_hi=datetime.fromisoformat(state["query_hi"]),+ stop_hi=datetime.fromisoformat(state["stop_hi"]),+ cursor=state.get("cursor"),+ done=bool(state.get("done")),+ # A resumed window splits above the rows it already handed over, not from `lo`.+ progress=datetime.fromisoformat(progress) if progress else None,+ )+++@frozen+class FetchedPage:+ rows: list[dict[str, Any]]+ # None when this is the last page of the window's query.+ next_cursor: str | None+++@frozen+class _WindowPage:+ window: ReadWindow+ rows: list[dict[str, Any]]+ finished: bool+ next_cursor: str | None+ error: BaseException | None = None+++# Reads the pages of one window, from its cursor, and owns the session it uses. It stops asking+# Shopify for pages once the given event is set, so a stopped read does not keep spending the+# store's query budget against the run that replaces it.+FetchWindowPages = Callable[[ReadWindow, threading.Event], Iterator[FetchedPage]]+++def _split(windows: list[ReadWindow], active: list[ReadWindow]) -> ReadWindow | None:+ """Give half of the remaining range of the busiest active window to a new window."""+ best: ReadWindow | None = None+ best_span = _MIN_SPLIT+ for window in active:+ span = window.stop_hi - (window.progress or window.lo)+ if span >= best_span:+ best, best_span = window, span+ if best is None:+ return None+ start = best.progress or best.lo+ # A whole-second boundary, because Shopify compares `updated_at` at that precision.+ middle = (start + best_span / 2).replace(microsecond=0)+ if middle <= start:+ return None+ new_window = ReadWindow(lo=middle, query_hi=best.stop_hi, stop_hi=best.stop_hi)+ best.stop_hi = middle+ windows.append(new_window)+ return new_window+++def _next_page(pages: "queue.Queue[_WindowPage]", tasks: list[Future[None]]) -> _WindowPage:+ """The next page from the workers. A worker that died without handing over its error fails the read."""+ while True:+ try:+ return pages.get(timeout=_QUEUE_POLL_SECONDS)+ except queue.Empty:+ for task in tasks:+ error = task.exception() if task.done() else None+ if error is not None:+ raise error+++def read_windows(+ windows: list[ReadWindow],+ fetch_pages: FetchWindowPages,+ row_time: Callable[[dict[str, Any]], datetime],+ save_windows: Callable[[list[dict[str, Any]]], None],+ safe_point: Callable[[], None],+ workers: int = PARALLEL_READ_WORKERS,+) -> Iterator[list[dict[str, Any]]]:+ """Yield the rows of every window, reading up to `workers` windows at once.++ Rows come in the order the windows answer, not in `updated_at` order. `save_windows` receives+ the state of every window before each page goes to the pipeline, so a resume reads each window+ from the cursor after the last page the pipeline received.+ """+ lock = threading.Lock()+ pages: queue.Queue[_WindowPage] = queue.Queue(maxsize=workers * 2)+ stop = threading.Event()++ def hand_over(page: _WindowPage) -> None:+ while not stop.is_set():+ try:+ pages.put(page, timeout=_QUEUE_POLL_SECONDS)+ return+ except queue.Full:+ continue++ def read(window: ReadWindow) -> None:+ try:+ for fetched in fetch_pages(window, stop):+ with lock:+ # Rows are sorted by `updated_at`, so the first row at or above the stop ends+ # the window. A split can lower the stop while this page is in flight.+ kept = [row for row in fetched.rows if row_time(row) < window.stop_hi]+ finished = fetched.next_cursor is None or len(kept) < len(fetched.rows)+ if kept:+ window.progress = row_time(kept[-1])+ hand_over(_WindowPage(window=window, rows=kept, finished=finished, next_cursor=fetched.next_cursor))+ if finished or stop.is_set():+ return+ hand_over(_WindowPage(window=window, rows=[], finished=True, next_cursor=None))+ except BaseException as error:+ hand_over(_WindowPage(window=window, rows=[], finished=False, next_cursor=None, error=error))++ pending = [window for window in windows if not window.done]+ active: list[ReadWindow] = []+ tasks: list[Future[None]] = []+ executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="shopify-window")+ try:+ while True:+ with lock:+ while len(active) < workers:+ window = pending.pop(0) if pending else _split(windows, active)+ if window is None:+ break+ active.append(window)+ # A copy of the context keeps the job's log and request tracking in the worker.+ tasks.append(executor.submit(contextvars.copy_context().run, read, window))+ if not active:+ return+ page = _next_page(pages, tasks)+ if page.error is not None:+ raise page.error+ with lock:+ if page.finished:+ page.window.done = True+ active.remove(page.window)+ else:+ page.window.cursor = page.next_cursor+ state = [window.to_state() for window in windows]+ save_windows(state)+ if page.rows:+ yield page.rows+ else:+ # An empty page leaves no row in the reader, so the run can stop and resume here.+ safe_point()+ finally:+ stop.set()+ while not pages.empty():+ pages.get_nowait()+ executor.shutdown(wait=False, cancel_futures=True)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.pyindex 93468210fc0d..93fc880cc619 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py@@ -1,7 +1,8 @@import reimport json+import threadingimport dataclasses-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom datetime import UTC, datetimefrom typing import Any@@ -10,11 +11,15 @@ import structlogfrom requests.exceptions import ChunkedEncodingErrorfrom structlog.types import FilteringBoundLoggerfrom tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity.nap import sleep as retry_sleepfrom posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (+ default_request_timeout,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.predicates import ValidatedRowFilterfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -23,6 +28,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.coID,resolve_schema_name,)+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.parallel import (+ FetchedPage,+ ReadWindow,+ read_windows,+)from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.settings import ENDPOINT_CONFIGSfrom products.warehouse_sources.backend.temporal.data_imports.sources.shopify.utils import (ShopifyGraphQLObject,@@ -42,6 +52,7 @@ from .constants import (SHOPIFY_GRAPHQL_OBJECTS,SHOPIFY_MAX_PAGE_SIZE,SHOPIFY_MAX_QUERY_COST,+ SHOPIFY_PARALLEL_READ_TIME_FIELDS,)logger = structlog.get_logger(__name__)@@ -51,6 +62,11 @@ logger = structlog.get_logger(__name__)PHASE_ALL = "all"PHASE_EARLIEST = "earliest"PHASE_LATEST = "latest"+# The "all" read split into `updated_at` windows that are read at once.+PHASE_PARALLEL = "parallel"++# The start of the first window of a parallel read, before any Shopify store existed.+_PARALLEL_READ_EARLIEST = datetime(2000, 1, 1, tzinfo=UTC)# Raised when Shopify's OAuth token endpoint returns a 4xx that we can't attribute to a more# specific cause below. The app credentials are invalid or revoked, so re-entering them is the@@ -188,10 +204,12 @@ SHOPIFY_CREDENTIALS_CHECK_ERROR = ()[email protected](frozen=True)class ShopifyResumeConfig:phase: strcursor: str+ # Set only in the parallel phase: the state of every window (see `ReadWindow.to_state`).+ windows: list[dict[str, Any]] | None = Noneclass ShopifyPermissionError(Exception):@@ -316,6 +334,10 @@ class ShopifyQueryCostExceededError(Exception):"""Shopify rejected the query because its requested cost is above the single-query limit."""+class ShopifyReadStopped(Exception):+ """Raised inside a paged read whose caller stopped wanting rows, to end it without more requests."""++def _is_max_cost_exceeded(errors: Any) -> bool:if not isinstance(errors, list):return False@@ -347,18 +369,36 @@ def _page_size_for_cost(page_size: int, payload: Any) -> int:return min(SHOPIFY_MAX_PAGE_SIZE, int(page_size * target / requested))-def _make_paginated_shopify_request(+@frozen+class ShopifyPage:+ rows: list[Any]+ # The cursor of the page after this one, or None when this is the last page.+ next_cursor: str | None+++def iter_shopify_pages(url: str,sess: requests.Session,graphql_object: ShopifyGraphQLObject,logger: FilteringBoundLogger,query: str | None = None,- phase: str = PHASE_ALL,initial_cursor: str | None = None,- resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig] | None = None,-) -> Iterator[list[Any]]:+ stop: threading.Event | None = None,+) -> Iterator[ShopifyPage]:+ """Yield the pages of one query.++ `stop`, when given, ends the read: no request or retry starts once it is set, and a backoff+ already waiting gives up as soon as it is. A read that only the caller abandons keeps spending+ the store's query budget for as long as its retries last, which a parallel read cannot afford.+ """endpoint_config = ENDPOINT_CONFIGS.get(graphql_object.name)+ def wait_unless_stopped(seconds: float) -> None:+ if stop is None:+ retry_sleep(seconds)+ else:+ stop.wait(seconds)+@retry(# Transient network failures (proxy/egress hiccups, connect/read timeouts) surface from# `sess.post` as requests ConnectionError/Timeout — e.g. a 504 from the egress proxy tunnel.@@ -369,13 +409,18 @@ def _make_paginated_shopify_request(),stop=stop_after_attempt(5),wait=_shopify_retry_wait,+ sleep=wait_unless_stopped,reraise=True,)def execute(vars: dict[str, Any]):+ if stop is not None and stop.is_set():+ raise ShopifyReadStopped(f"Shopify: read of {graphql_object.name} stopped")# `post` reads the body eagerly (stream=False), so a connection dropped mid-stream# surfaces here as ChunkedEncodingError. It's transient — retry it like a 5xx below.try:- response = sess.post(url, json={"query": graphql_object.query, "variables": vars})+ response = sess.post(+ url, json={"query": graphql_object.query, "variables": vars}, timeout=default_request_timeout()+ )except ChunkedEncodingError as e:raise ShopifyRetryableError(f"Shopify: connection broken while reading response: {e}") from eif response.status_code >= 500:@@ -447,19 +492,102 @@ def _make_paginated_shopify_request(data = [endpoint_config.incremental_field_resolver(row) for row in data if row.get("discount")]if endpoint_config.partition_key_resolver:data = [endpoint_config.partition_key_resolver(row) for row in data if row.get("discount")]- yield datapage_info = unwrap(payload, path=f"data.{graphql_object.name}.pageInfo")has_next_page = page_info.get("hasNextPage", False)- if has_next_page:- # this is intentionally an unsafe lookup so errors surface if expectations aren't met- next_cursor = page_info["endCursor"]+ # this is intentionally an unsafe lookup so errors surface if expectations aren't met+ next_cursor = page_info["endCursor"] if has_next_page else None+ yield ShopifyPage(rows=data, next_cursor=next_cursor)+ if next_cursor is not None:vars.update({"cursor": next_cursor})+++def _make_paginated_shopify_request(+ url: str,+ sess: requests.Session,+ graphql_object: ShopifyGraphQLObject,+ logger: FilteringBoundLogger,+ query: str | None = None,+ phase: str = PHASE_ALL,+ initial_cursor: str | None = None,+ resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig] | None = None,+) -> Iterator[list[Any]]:+ for page in iter_shopify_pages(url, sess, graphql_object, logger, query=query, initial_cursor=initial_cursor):+ yield page.rows+ if page.next_cursor is not None:# Checkpoint points to the NEXT unfetched page. Because this save happens# only after the generator is resumed past `yield data`, a shutdown/crash# after yielding can still cause the last yielded page to be re-fetched on# resume; primary-key merge semantics dedupe if that happens.if resumable_source_manager is not None:- resumable_source_manager.save_state(ShopifyResumeConfig(phase=phase, cursor=next_cursor))+ resumable_source_manager.save_state(ShopifyResumeConfig(phase=phase, cursor=page.next_cursor))+++def _parallel_read_windows(+ schema_name: str,+ resume_config: ShopifyResumeConfig | None,+ should_use_incremental_field: bool,+ use_parallel_read: Callable[[], bool] | None,+) -> list[ReadWindow] | None:+ """The windows of a parallel read, or None when the read is paged.++ A read in progress stays on the path that started it, because the cursor of one path means+ nothing to the other. Only a new read asks the flag.+ """+ if resume_config is not None:+ if resume_config.phase != PHASE_PARALLEL or resume_config.windows is None:+ return None+ return [ReadWindow.from_state(state) for state in resume_config.windows]+ # A full refresh is excluded: its windows end at the start time, so a row that changes during+ # the read would be missing from the table until the next run. An incremental run reads it next.+ if (+ not should_use_incremental_field+ or schema_name not in SHOPIFY_PARALLEL_READ_TIME_FIELDS+ or use_parallel_read is None+ or not use_parallel_read()+ ):+ return None+ started_at = datetime.now(UTC).replace(microsecond=0)+ return [ReadWindow(lo=_PARALLEL_READ_EARLIEST, query_hi=started_at, stop_hi=started_at)]+++def _read_in_parallel(+ url: str,+ make_session: Callable[[], requests.Session],+ graphql_object: ShopifyGraphQLObject,+ logger: FilteringBoundLogger,+ windows: list[ReadWindow],+ filter_terms: list[str],+ resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig],+) -> Iterator[list[Any]]:+ endpoint_config = ENDPOINT_CONFIGS[graphql_object.name]+ time_field = SHOPIFY_PARALLEL_READ_TIME_FIELDS[graphql_object.name]++ def fetch_pages(window: ReadWindow, stop: threading.Event) -> Iterator[FetchedPage]:+ query = " AND ".join(+ [+ f"{endpoint_config.query_filter}:>='{window.lo.isoformat()}'",+ f"{endpoint_config.query_filter}:<'{window.query_hi.isoformat()}'",+ *filter_terms,+ ]+ )+ # Each window runs on its own thread, and a `requests` session is not safe to share.+ with make_session() as window_sess:+ for page in iter_shopify_pages(+ url, window_sess, graphql_object, logger, query=query, initial_cursor=window.cursor, stop=stop+ ):+ yield FetchedPage(rows=page.rows, next_cursor=page.next_cursor)++ def save_windows(state: list[dict[str, Any]]) -> None:+ resumable_source_manager.save_state(ShopifyResumeConfig(phase=PHASE_PARALLEL, cursor="", windows=state))++ logger.debug(f"Shopify: reading {graphql_object.name} in parallel windows of {endpoint_config.query_filter}")+ yield from read_windows(+ windows,+ fetch_pages,+ row_time=lambda row: datetime.fromisoformat(row[time_field]),+ save_windows=save_windows,+ safe_point=resumable_source_manager.safe_point,+ )# A Shopify store subdomain is lowercase alphanumerics and hyphens.@@ -682,6 +810,7 @@ def shopify_source(should_use_incremental_field: bool = False,shopify_access_token: str | None = None,row_filters: list[ValidatedRowFilter] | None = None,+ use_parallel_read: Callable[[], bool] | None = None,):store_id = normalize_store_id(shopify_store_id)api_url = SHOPIFY_API_URL.format(store_id, api_version)@@ -689,11 +818,14 @@ def shopify_source(schema_name = resolve_schema_name(graphql_object_name)filter_terms = row_filter_search_terms(schema_name, row_filters)- def get_rows():- sess = make_tracked_session(+ def make_session() -> requests.Session:+ return make_tracked_session(headers={"X-Shopify-Access-Token": access_token, "Content-Type": "application/json"},redact_values=(access_token,),)++ def get_rows():+ sess = make_session()graphql_object = SHOPIFY_GRAPHQL_OBJECTS.get(schema_name)if not graphql_object:raise Exception(f"Shopify object does not exist: {schema_name}")@@ -724,6 +856,20 @@ def shopify_source(db_incremental_field_last_value is None and db_incremental_field_earliest_value is None):logger.debug(f"Shopify: iterating all objects from source for {schema_name}")+ windows = _parallel_read_windows(+ schema_name, resume_config, should_use_incremental_field, use_parallel_read+ )+ if windows is not None:+ yield from _read_in_parallel(+ api_url,+ make_session,+ graphql_object,+ logger,+ windows,+ filter_terms,+ resumable_source_manager,+ )+ returninitial_cursor = (resume_config.cursor if resume_config is not None and resume_config.phase == PHASE_ALL else None)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.pyindex c8a708b213eb..13f1a5df0ba7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py@@ -1,3 +1,4 @@+import functoolsfrom datetime import datefrom typing import Optional, cast@@ -11,6 +12,8 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldSelectConfig,SourceFieldSelectConfigOption,)+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabledfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (FieldType,ResumableSource,@@ -61,6 +64,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.sh)from products.warehouse_sources.backend.types import ExternalDataSourceType+WAREHOUSE_SHOPIFY_PARALLEL_READ_FLAG = "warehouse-shopify-parallel-read"+++def _is_parallel_read_enabled(team_id: int, schema_id: str) -> bool:+ schema = ExternalDataSchema.objects.filter(team_id=team_id, id=schema_id).select_related("source").first()+ return schema is not None and is_schema_flag_enabled(schema, WAREHOUSE_SHOPIFY_PARALLEL_READ_FLAG)+@SourceRegistry.registerclass ShopifySource(ResumableSource[ShopifySourceConfig, ShopifyResumeConfig]):@@ -456,4 +466,5 @@ class ShopifySource(ResumableSource[ShopifySourceConfig, ShopifyResumeConfig]):logger=inputs.logger,resumable_source_manager=resumable_source_manager,row_filters=inputs.row_filters,+ use_parallel_read=functools.partial(_is_parallel_read_enabled, inputs.team_id, inputs.schema_id),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.pynew file mode 100644index 000000000000..337c22c70a72--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.py@@ -0,0 +1,12 @@+import pytest+from unittest.mock import patch++[email protected](autouse=True)+def _parallel_read_flag_off():+ # The flag lookup reads the schema row, and the contract tests run without a database.+ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.shopify.source._is_parallel_read_enabled",+ return_value=False,+ ):+ yielddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.pynew file mode 100644index 000000000000..8d763533777c--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.py@@ -0,0 +1,190 @@+import re+import threading+from datetime import UTC, datetime, timedelta+from typing import Any++import pytest+from unittest.mock import MagicMock, patch++from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (+ DriveResult,+ RecordedRequest,+ ScriptedResponse,+ SourceDriver,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.shopify import (+ ShopifyAuthMethodConfig,+ ShopifySourceConfig,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.constants import (+ ABANDONED_CHECKOUTS,+ SHOPIFY_GRAPHQL_OBJECTS,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.parallel import ReadWindow, _split+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.shopify import (+ PHASE_PARALLEL,+ ShopifyReadStopped,+ ShopifyResumeConfig,+ iter_shopify_pages,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.source import ShopifySource++_SHOPIFY = "products.warehouse_sources.backend.temporal.data_imports.sources.shopify"+_CLUSTER_START = datetime(2022, 6, 20, tzinfo=UTC)+++def _orders() -> list[dict[str, Any]]:+ # Most orders share a few minutes, the way a bulk update leaves them, and the rest spread over years.+ clustered = [_CLUSTER_START + timedelta(seconds=3 * index) for index in range(200)]+ spread = [datetime(2019, 1, 1, tzinfo=UTC) + timedelta(days=21 * index) for index in range(100)]+ return [+ {"id": f"order-{index}", "updatedAt": moment.isoformat().replace("+00:00", "Z")}+ for index, moment in enumerate(sorted(clustered + spread))+ ]+++_ORDERS = _orders()+_ORDER_IDS = {order["id"] for order in _ORDERS}+++class _Store:+ """Answers the orders search the way Shopify does: filtered on `updated_at`, sorted by it, paged by cursor."""++ def __init__(self, fail_windows_from: datetime | None = None):+ self.fail_windows_from = fail_windows_from++ def __call__(self, request: RecordedRequest) -> ScriptedResponse:+ if request.path == "/admin/oauth/access_token":+ return ScriptedResponse(json={"access_token": "tok"})+ if request.method == "GET":+ return ScriptedResponse(json={"access_scopes": []})+ variables = request.json()["variables"]+ search = variables.get("query") or ""+ lower = re.search(r"updated_at:>='([^']+)'", search)+ upper = re.search(r"updated_at:<'([^']+)'", search)+ low = datetime.fromisoformat(lower.group(1)) if lower else None+ high = datetime.fromisoformat(upper.group(1)) if upper else None+ if self.fail_windows_from is not None and low is not None and low >= self.fail_windows_from:+ return ScriptedResponse(status=500)+ matching = [+ order+ for order in _ORDERS+ if (low is None or datetime.fromisoformat(order["updatedAt"]) >= low)+ and (high is None or datetime.fromisoformat(order["updatedAt"]) < high)+ ]+ offset = int(variables.get("cursor") or 0)+ page = matching[offset : offset + variables["pageSize"]]+ end = offset + len(page)+ return ScriptedResponse(+ json={+ "data": {+ "orders": {+ "nodes": page,+ "pageInfo": {"hasNextPage": end < len(matching), "endCursor": str(end)},+ }+ }+ }+ )+++def _run(store: _Store, *, flag_enabled: bool = True, incremental: bool = True, **inputs: Any) -> DriveResult:+ config = ShopifySourceConfig(+ shopify_store_id="my-store",+ auth_method=ShopifyAuthMethodConfig(+ selection="client_credentials", shopify_client_id="cid", shopify_client_secret="secret"+ ),+ )+ with patch(f"{_SHOPIFY}.source._is_parallel_read_enabled", return_value=flag_enabled):+ return SourceDriver(ShopifySource(), config).run(+ "orders", store, should_use_incremental_field=incremental, **inputs+ )+++def _searches(result: DriveResult) -> list[str]:+ return [+ request.json()["variables"].get("query") or ""+ for request in result.requests+ if request.path.endswith("/graphql.json")+ ]+++def test_parallel_read_returns_every_order_once_and_splits_a_dense_range() -> None:+ result = _run(_Store())++ assert result.raised is None+ ids = [row["id"] for row in result.rows]+ assert sorted(ids) == sorted(_ORDER_IDS)+ lower_bounds = {re.search(r"updated_at:>='([^']+)'", search).group(1) for search in _searches(result)} # type: ignore[union-attr]+ # The dense minutes get windows of their own, so more than the first window runs.+ assert sum(1 for bound in lower_bounds if datetime.fromisoformat(bound) >= _CLUSTER_START) >= 2+ assert all("updated_at:<'" in search for search in _searches(result))+++def test_resumed_parallel_read_returns_exactly_the_orders_the_first_attempt_did_not_hand_over() -> None:+ first = _run(_Store(fail_windows_from=_CLUSTER_START + timedelta(seconds=150)))++ assert first.raised is not None+ assert isinstance(first.final_state, ShopifyResumeConfig)+ assert first.final_state.phase == PHASE_PARALLEL++ second = _run(_Store(), resume_state=first.final_state, flag_enabled=False)++ assert second.raised is None+ first_ids = {row["id"] for row in first.rows}+ second_ids = [row["id"] for row in second.rows]+ assert len(second_ids) == len(set(second_ids))+ assert first_ids.isdisjoint(second_ids)+ assert first_ids | set(second_ids) == _ORDER_IDS+++ "kwargs",+ [{"flag_enabled": False}, {"incremental": False}],+ ids=["flag_off", "full_refresh"],+)+def test_reads_pages_in_one_chain_when_the_parallel_read_does_not_apply(kwargs: dict[str, Any]) -> None:+ result = _run(_Store(), **kwargs)++ assert sorted(row["id"] for row in result.rows) == sorted(_ORDER_IDS)+ assert not any("updated_at" in search for search in _searches(result))+++def test_a_resumed_window_splits_above_the_rows_it_already_handed_over() -> None:+ window = ReadWindow(+ lo=_CLUSTER_START, query_hi=_CLUSTER_START + timedelta(hours=4), stop_hi=_CLUSTER_START + timedelta(hours=4)+ )+ window.progress = _CLUSTER_START + timedelta(hours=3)+ window.cursor = "40"++ resumed = ReadWindow.from_state(window.to_state())+ new_window = _split([resumed], [resumed])++ assert new_window is not None+ # Splitting from `lo` would give the new window a range the resumed window already read.+ assert new_window.lo > window.progress+ assert resumed.stop_hi == new_window.lo+++def test_a_stopped_read_asks_shopify_for_no_more_pages() -> None:+ stop = threading.Event()+ response = MagicMock()+ response.status_code = 200+ response.json.return_value = {+ "data": {ABANDONED_CHECKOUTS: {"nodes": [{"id": "1"}], "pageInfo": {"hasNextPage": True, "endCursor": "2"}}}+ }+ sess = MagicMock()+ sess.post.return_value = response++ pages = iter_shopify_pages(+ "https://example.invalid/graphql",+ sess,+ SHOPIFY_GRAPHQL_OBJECTS[ABANDONED_CHECKOUTS],+ MagicMock(),+ stop=stop,+ )++ assert next(pages).next_cursor == "2"+ stop.set()+ with pytest.raises(ShopifyReadStopped):+ next(pages)+ assert sess.post.call_count == 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.pyindex 79f764eb4bce..d54ef5382d38 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py@@ -164,7 +164,7 @@ class TestMakePaginatedShopifyRequest:captured: list[tuple[int, str | None]] = []sess = MagicMock()- def post(url: str, json: dict[str, Any]) -> MagicMock:+ def post(url: str, json: dict[str, Any], **kwargs: Any) -> MagicMock:captured.append((json["variables"]["pageSize"], json["variables"].get("cursor")))outcome = outcomes[len(captured) - 1]if isinstance(outcome, Exception):