Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(warehouse_sources): read shopify orders in parallel updated_at windows (#120420)

Daniel RC23e7fbf1cc13
diff --git a/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt b/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt
index 73c2e34562c2..442b8012a4b5 100644
--- a/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt
+++ b/posthog/test/repo_invariants/source_request_without_timeout_baseline.txt
@@ -26,7 +26,6 @@ rokt_ads/rokt_ads.py::RoktAdsClient.get
rokt_ads/rokt_ads.py::RoktAdsClient.post
salesforce/auth.py::salesforce_refresh_access_token
shopify/shopify.py::_get_granted_scopes
-shopify/shopify.py::_make_paginated_shopify_request.execute
shopify/shopify.py::_probe_resource_permission
shopify/shopify.py::validate_credentials
stytch/stytch.py::_get_session_rows
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
index 6297e2c7e51d..611edecb99d6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_append_rollback.py
@@ -149,6 +149,15 @@ def _boundaries(settings: Any) -> Iterator[None]:
):
stack.enter_context(patch(f"{_PROCESSOR}.{target}"))
stack.enter_context(patch(f"{_PROCESSOR}.is_batch_already_processed", return_value=False))
+ # The completion check reads the queue DB, whose tables only exist in suites that create
+ # them, so leaving it live makes this one order-dependent on whichever shard it lands in.
+ # Every run here hands over all of its batches, which is what count=0 stands for.
+ stack.enter_context(
+ patch(
+ f"{_PROCESSOR}.BatchQueue.get_unfinished_earlier_batches_sync",
+ return_value=UnfinishedEarlierBatches(count=0, first_batch_index=None, states=()),
+ )
+ )
stack.enter_context(patch(f"{_PROCESSOR}.finish_row_tracking", new_callable=AsyncMock))
# The completion check reads the queue DB, which these tests do not have.
stack.enter_context(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py
index 9f738f66bcb9..4344bd342a98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/constants.py
@@ -49,6 +49,9 @@ SUBSCRIPTION_CONTRACTS = "subscriptionContracts"
# page size. An object listed here starts at a page size that is known to fit. The request layer
# then grows the page from the cost Shopify reports.
SHOPIFY_COST_LIMITED_START_PAGE_SIZES = {ORDERS: 20}
+# The objects a full read can split into `updated_at` windows read at once, with the row field that
+# holds `updated_at`. Each must sort by `updated_at` and filter on it (see `parallel.py`).
+SHOPIFY_PARALLEL_READ_TIME_FIELDS = {ORDERS: "updatedAt"}
SHOPIFY_MAX_QUERY_COST = 1000
# The largest `first` Shopify accepts on a connection.
SHOPIFY_MAX_PAGE_SIZE = 250
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.py
new file mode 100644
index 000000000000..739ce1c6cb91
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/parallel.py
@@ -0,0 +1,204 @@
+"""Reads one Shopify object with several requests in flight, each over its own range of `updated_at`.
+
+One paged read sends a request only after the one before it answers, because each page needs the
+cursor of the last. A large store can serve more than that: the read uses a fraction of the store's
+query cost budget. This reader splits the time range into windows and reads several windows at once.
+
+Shopify returns each window sorted by `updated_at`, so a window that has read up to a time can hand
+the rest of its range to another window. When a worker is free, the reader splits the window with the
+most time left. A range that holds many rows is split again and again until every worker has work.
+"""
+
+import queue
+import threading
+import contextvars
+import dataclasses
+from collections.abc import Callable, Iterator
+from concurrent.futures import Future, ThreadPoolExecutor
+from datetime import datetime, timedelta
+from typing import Any
+
+from posthog.dataclasses import frozen
+
+PARALLEL_READ_WORKERS = 3
+# A range shorter than this is not split. Shopify filters on whole seconds.
+_MIN_SPLIT = timedelta(seconds=2)
+_QUEUE_POLL_SECONDS = 1.0
+
+
[email protected](frozen=False)
+class ReadWindow:
+ """A range of `updated_at` that one request chain reads.
+
+ The query covers `lo` to `query_hi`. After a split the window stops at `stop_hi` and leaves the
+ rest to the new window, but it keeps its query, because its cursor belongs to that query.
+ """
+
+ lo: datetime
+ query_hi: datetime
+ stop_hi: datetime
+ # The cursor after the last page the pipeline received, or None before the first page.
+ cursor: str | None = None
+ done: bool = False
+ # The newest `updated_at` a worker has fetched. A split only happens above it.
+ progress: datetime | None = None
+
+ def to_state(self) -> dict[str, Any]:
+ return {
+ "lo": self.lo.isoformat(),
+ "query_hi": self.query_hi.isoformat(),
+ "stop_hi": self.stop_hi.isoformat(),
+ "cursor": self.cursor,
+ "done": self.done,
+ "progress": self.progress.isoformat() if self.progress is not None else None,
+ }
+
+ @classmethod
+ def from_state(cls, state: dict[str, Any]) -> "ReadWindow":
+ progress = state.get("progress")
+ return cls(
+ lo=datetime.fromisoformat(state["lo"]),
+ query_hi=datetime.fromisoformat(state["query_hi"]),
+ stop_hi=datetime.fromisoformat(state["stop_hi"]),
+ cursor=state.get("cursor"),
+ done=bool(state.get("done")),
+ # A resumed window splits above the rows it already handed over, not from `lo`.
+ progress=datetime.fromisoformat(progress) if progress else None,
+ )
+
+
+@frozen
+class FetchedPage:
+ rows: list[dict[str, Any]]
+ # None when this is the last page of the window's query.
+ next_cursor: str | None
+
+
+@frozen
+class _WindowPage:
+ window: ReadWindow
+ rows: list[dict[str, Any]]
+ finished: bool
+ next_cursor: str | None
+ error: BaseException | None = None
+
+
+# Reads the pages of one window, from its cursor, and owns the session it uses. It stops asking
+# Shopify for pages once the given event is set, so a stopped read does not keep spending the
+# store's query budget against the run that replaces it.
+FetchWindowPages = Callable[[ReadWindow, threading.Event], Iterator[FetchedPage]]
+
+
+def _split(windows: list[ReadWindow], active: list[ReadWindow]) -> ReadWindow | None:
+ """Give half of the remaining range of the busiest active window to a new window."""
+ best: ReadWindow | None = None
+ best_span = _MIN_SPLIT
+ for window in active:
+ span = window.stop_hi - (window.progress or window.lo)
+ if span >= best_span:
+ best, best_span = window, span
+ if best is None:
+ return None
+ start = best.progress or best.lo
+ # A whole-second boundary, because Shopify compares `updated_at` at that precision.
+ middle = (start + best_span / 2).replace(microsecond=0)
+ if middle <= start:
+ return None
+ new_window = ReadWindow(lo=middle, query_hi=best.stop_hi, stop_hi=best.stop_hi)
+ best.stop_hi = middle
+ windows.append(new_window)
+ return new_window
+
+
+def _next_page(pages: "queue.Queue[_WindowPage]", tasks: list[Future[None]]) -> _WindowPage:
+ """The next page from the workers. A worker that died without handing over its error fails the read."""
+ while True:
+ try:
+ return pages.get(timeout=_QUEUE_POLL_SECONDS)
+ except queue.Empty:
+ for task in tasks:
+ error = task.exception() if task.done() else None
+ if error is not None:
+ raise error
+
+
+def read_windows(
+ windows: list[ReadWindow],
+ fetch_pages: FetchWindowPages,
+ row_time: Callable[[dict[str, Any]], datetime],
+ save_windows: Callable[[list[dict[str, Any]]], None],
+ safe_point: Callable[[], None],
+ workers: int = PARALLEL_READ_WORKERS,
+) -> Iterator[list[dict[str, Any]]]:
+ """Yield the rows of every window, reading up to `workers` windows at once.
+
+ Rows come in the order the windows answer, not in `updated_at` order. `save_windows` receives
+ the state of every window before each page goes to the pipeline, so a resume reads each window
+ from the cursor after the last page the pipeline received.
+ """
+ lock = threading.Lock()
+ pages: queue.Queue[_WindowPage] = queue.Queue(maxsize=workers * 2)
+ stop = threading.Event()
+
+ def hand_over(page: _WindowPage) -> None:
+ while not stop.is_set():
+ try:
+ pages.put(page, timeout=_QUEUE_POLL_SECONDS)
+ return
+ except queue.Full:
+ continue
+
+ def read(window: ReadWindow) -> None:
+ try:
+ for fetched in fetch_pages(window, stop):
+ with lock:
+ # Rows are sorted by `updated_at`, so the first row at or above the stop ends
+ # the window. A split can lower the stop while this page is in flight.
+ kept = [row for row in fetched.rows if row_time(row) < window.stop_hi]
+ finished = fetched.next_cursor is None or len(kept) < len(fetched.rows)
+ if kept:
+ window.progress = row_time(kept[-1])
+ hand_over(_WindowPage(window=window, rows=kept, finished=finished, next_cursor=fetched.next_cursor))
+ if finished or stop.is_set():
+ return
+ hand_over(_WindowPage(window=window, rows=[], finished=True, next_cursor=None))
+ except BaseException as error:
+ hand_over(_WindowPage(window=window, rows=[], finished=False, next_cursor=None, error=error))
+
+ pending = [window for window in windows if not window.done]
+ active: list[ReadWindow] = []
+ tasks: list[Future[None]] = []
+ executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix="shopify-window")
+ try:
+ while True:
+ with lock:
+ while len(active) < workers:
+ window = pending.pop(0) if pending else _split(windows, active)
+ if window is None:
+ break
+ active.append(window)
+ # A copy of the context keeps the job's log and request tracking in the worker.
+ tasks.append(executor.submit(contextvars.copy_context().run, read, window))
+ if not active:
+ return
+ page = _next_page(pages, tasks)
+ if page.error is not None:
+ raise page.error
+ with lock:
+ if page.finished:
+ page.window.done = True
+ active.remove(page.window)
+ else:
+ page.window.cursor = page.next_cursor
+ state = [window.to_state() for window in windows]
+ save_windows(state)
+ if page.rows:
+ yield page.rows
+ else:
+ # An empty page leaves no row in the reader, so the run can stop and resume here.
+ safe_point()
+ finally:
+ stop.set()
+ while not pages.empty():
+ pages.get_nowait()
+ executor.shutdown(wait=False, cancel_futures=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
index 93468210fc0d..93fc880cc619 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
@@ -1,7 +1,8 @@
import re
import json
+import threading
import dataclasses
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from typing import Any
@@ -10,11 +11,15 @@ import structlog
from requests.exceptions import ChunkedEncodingError
from structlog.types import FilteringBoundLogger
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity.nap import sleep as retry_sleep
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (
+ default_request_timeout,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sql.predicates import ValidatedRowFilter
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -23,6 +28,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.co
ID,
resolve_schema_name,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.parallel import (
+ FetchedPage,
+ ReadWindow,
+ read_windows,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.settings import ENDPOINT_CONFIGS
from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.utils import (
ShopifyGraphQLObject,
@@ -42,6 +52,7 @@ from .constants import (
SHOPIFY_GRAPHQL_OBJECTS,
SHOPIFY_MAX_PAGE_SIZE,
SHOPIFY_MAX_QUERY_COST,
+ SHOPIFY_PARALLEL_READ_TIME_FIELDS,
)
logger = structlog.get_logger(__name__)
@@ -51,6 +62,11 @@ logger = structlog.get_logger(__name__)
PHASE_ALL = "all"
PHASE_EARLIEST = "earliest"
PHASE_LATEST = "latest"
+# The "all" read split into `updated_at` windows that are read at once.
+PHASE_PARALLEL = "parallel"
+
+# The start of the first window of a parallel read, before any Shopify store existed.
+_PARALLEL_READ_EARLIEST = datetime(2000, 1, 1, tzinfo=UTC)
# Raised when Shopify's OAuth token endpoint returns a 4xx that we can't attribute to a more
# specific cause below. The app credentials are invalid or revoked, so re-entering them is the
@@ -188,10 +204,12 @@ SHOPIFY_CREDENTIALS_CHECK_ERROR = (
)
[email protected](frozen=True)
class ShopifyResumeConfig:
phase: str
cursor: str
+ # Set only in the parallel phase: the state of every window (see `ReadWindow.to_state`).
+ windows: list[dict[str, Any]] | None = None
class ShopifyPermissionError(Exception):
@@ -316,6 +334,10 @@ class ShopifyQueryCostExceededError(Exception):
"""Shopify rejected the query because its requested cost is above the single-query limit."""
+class ShopifyReadStopped(Exception):
+ """Raised inside a paged read whose caller stopped wanting rows, to end it without more requests."""
+
+
def _is_max_cost_exceeded(errors: Any) -> bool:
if not isinstance(errors, list):
return False
@@ -347,18 +369,36 @@ def _page_size_for_cost(page_size: int, payload: Any) -> int:
return min(SHOPIFY_MAX_PAGE_SIZE, int(page_size * target / requested))
-def _make_paginated_shopify_request(
+@frozen
+class ShopifyPage:
+ rows: list[Any]
+ # The cursor of the page after this one, or None when this is the last page.
+ next_cursor: str | None
+
+
+def iter_shopify_pages(
url: str,
sess: requests.Session,
graphql_object: ShopifyGraphQLObject,
logger: FilteringBoundLogger,
query: str | None = None,
- phase: str = PHASE_ALL,
initial_cursor: str | None = None,
- resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig] | None = None,
-) -> Iterator[list[Any]]:
+ stop: threading.Event | None = None,
+) -> Iterator[ShopifyPage]:
+ """Yield the pages of one query.
+
+ `stop`, when given, ends the read: no request or retry starts once it is set, and a backoff
+ already waiting gives up as soon as it is. A read that only the caller abandons keeps spending
+ the store's query budget for as long as its retries last, which a parallel read cannot afford.
+ """
endpoint_config = ENDPOINT_CONFIGS.get(graphql_object.name)
+ def wait_unless_stopped(seconds: float) -> None:
+ if stop is None:
+ retry_sleep(seconds)
+ else:
+ stop.wait(seconds)
+
@retry(
# Transient network failures (proxy/egress hiccups, connect/read timeouts) surface from
# `sess.post` as requests ConnectionError/Timeout — e.g. a 504 from the egress proxy tunnel.
@@ -369,13 +409,18 @@ def _make_paginated_shopify_request(
),
stop=stop_after_attempt(5),
wait=_shopify_retry_wait,
+ sleep=wait_unless_stopped,
reraise=True,
)
def execute(vars: dict[str, Any]):
+ if stop is not None and stop.is_set():
+ raise ShopifyReadStopped(f"Shopify: read of {graphql_object.name} stopped")
# `post` reads the body eagerly (stream=False), so a connection dropped mid-stream
# surfaces here as ChunkedEncodingError. It's transient — retry it like a 5xx below.
try:
- response = sess.post(url, json={"query": graphql_object.query, "variables": vars})
+ response = sess.post(
+ url, json={"query": graphql_object.query, "variables": vars}, timeout=default_request_timeout()
+ )
except ChunkedEncodingError as e:
raise ShopifyRetryableError(f"Shopify: connection broken while reading response: {e}") from e
if response.status_code >= 500:
@@ -447,19 +492,102 @@ def _make_paginated_shopify_request(
data = [endpoint_config.incremental_field_resolver(row) for row in data if row.get("discount")]
if endpoint_config.partition_key_resolver:
data = [endpoint_config.partition_key_resolver(row) for row in data if row.get("discount")]
- yield data
page_info = unwrap(payload, path=f"data.{graphql_object.name}.pageInfo")
has_next_page = page_info.get("hasNextPage", False)
- if has_next_page:
- # this is intentionally an unsafe lookup so errors surface if expectations aren't met
- next_cursor = page_info["endCursor"]
+ # this is intentionally an unsafe lookup so errors surface if expectations aren't met
+ next_cursor = page_info["endCursor"] if has_next_page else None
+ yield ShopifyPage(rows=data, next_cursor=next_cursor)
+ if next_cursor is not None:
vars.update({"cursor": next_cursor})
+
+
+def _make_paginated_shopify_request(
+ url: str,
+ sess: requests.Session,
+ graphql_object: ShopifyGraphQLObject,
+ logger: FilteringBoundLogger,
+ query: str | None = None,
+ phase: str = PHASE_ALL,
+ initial_cursor: str | None = None,
+ resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig] | None = None,
+) -> Iterator[list[Any]]:
+ for page in iter_shopify_pages(url, sess, graphql_object, logger, query=query, initial_cursor=initial_cursor):
+ yield page.rows
+ if page.next_cursor is not None:
# Checkpoint points to the NEXT unfetched page. Because this save happens
# only after the generator is resumed past `yield data`, a shutdown/crash
# after yielding can still cause the last yielded page to be re-fetched on
# resume; primary-key merge semantics dedupe if that happens.
if resumable_source_manager is not None:
- resumable_source_manager.save_state(ShopifyResumeConfig(phase=phase, cursor=next_cursor))
+ resumable_source_manager.save_state(ShopifyResumeConfig(phase=phase, cursor=page.next_cursor))
+
+
+def _parallel_read_windows(
+ schema_name: str,
+ resume_config: ShopifyResumeConfig | None,
+ should_use_incremental_field: bool,
+ use_parallel_read: Callable[[], bool] | None,
+) -> list[ReadWindow] | None:
+ """The windows of a parallel read, or None when the read is paged.
+
+ A read in progress stays on the path that started it, because the cursor of one path means
+ nothing to the other. Only a new read asks the flag.
+ """
+ if resume_config is not None:
+ if resume_config.phase != PHASE_PARALLEL or resume_config.windows is None:
+ return None
+ return [ReadWindow.from_state(state) for state in resume_config.windows]
+ # A full refresh is excluded: its windows end at the start time, so a row that changes during
+ # the read would be missing from the table until the next run. An incremental run reads it next.
+ if (
+ not should_use_incremental_field
+ or schema_name not in SHOPIFY_PARALLEL_READ_TIME_FIELDS
+ or use_parallel_read is None
+ or not use_parallel_read()
+ ):
+ return None
+ started_at = datetime.now(UTC).replace(microsecond=0)
+ return [ReadWindow(lo=_PARALLEL_READ_EARLIEST, query_hi=started_at, stop_hi=started_at)]
+
+
+def _read_in_parallel(
+ url: str,
+ make_session: Callable[[], requests.Session],
+ graphql_object: ShopifyGraphQLObject,
+ logger: FilteringBoundLogger,
+ windows: list[ReadWindow],
+ filter_terms: list[str],
+ resumable_source_manager: ResumableSourceManager[ShopifyResumeConfig],
+) -> Iterator[list[Any]]:
+ endpoint_config = ENDPOINT_CONFIGS[graphql_object.name]
+ time_field = SHOPIFY_PARALLEL_READ_TIME_FIELDS[graphql_object.name]
+
+ def fetch_pages(window: ReadWindow, stop: threading.Event) -> Iterator[FetchedPage]:
+ query = " AND ".join(
+ [
+ f"{endpoint_config.query_filter}:>='{window.lo.isoformat()}'",
+ f"{endpoint_config.query_filter}:<'{window.query_hi.isoformat()}'",
+ *filter_terms,
+ ]
+ )
+ # Each window runs on its own thread, and a `requests` session is not safe to share.
+ with make_session() as window_sess:
+ for page in iter_shopify_pages(
+ url, window_sess, graphql_object, logger, query=query, initial_cursor=window.cursor, stop=stop
+ ):
+ yield FetchedPage(rows=page.rows, next_cursor=page.next_cursor)
+
+ def save_windows(state: list[dict[str, Any]]) -> None:
+ resumable_source_manager.save_state(ShopifyResumeConfig(phase=PHASE_PARALLEL, cursor="", windows=state))
+
+ logger.debug(f"Shopify: reading {graphql_object.name} in parallel windows of {endpoint_config.query_filter}")
+ yield from read_windows(
+ windows,
+ fetch_pages,
+ row_time=lambda row: datetime.fromisoformat(row[time_field]),
+ save_windows=save_windows,
+ safe_point=resumable_source_manager.safe_point,
+ )
# A Shopify store subdomain is lowercase alphanumerics and hyphens.
@@ -682,6 +810,7 @@ def shopify_source(
should_use_incremental_field: bool = False,
shopify_access_token: str | None = None,
row_filters: list[ValidatedRowFilter] | None = None,
+ use_parallel_read: Callable[[], bool] | None = None,
):
store_id = normalize_store_id(shopify_store_id)
api_url = SHOPIFY_API_URL.format(store_id, api_version)
@@ -689,11 +818,14 @@ def shopify_source(
schema_name = resolve_schema_name(graphql_object_name)
filter_terms = row_filter_search_terms(schema_name, row_filters)
- def get_rows():
- sess = make_tracked_session(
+ def make_session() -> requests.Session:
+ return make_tracked_session(
headers={"X-Shopify-Access-Token": access_token, "Content-Type": "application/json"},
redact_values=(access_token,),
)
+
+ def get_rows():
+ sess = make_session()
graphql_object = SHOPIFY_GRAPHQL_OBJECTS.get(schema_name)
if not graphql_object:
raise Exception(f"Shopify object does not exist: {schema_name}")
@@ -724,6 +856,20 @@ def shopify_source(
db_incremental_field_last_value is None and db_incremental_field_earliest_value is None
):
logger.debug(f"Shopify: iterating all objects from source for {schema_name}")
+ windows = _parallel_read_windows(
+ schema_name, resume_config, should_use_incremental_field, use_parallel_read
+ )
+ if windows is not None:
+ yield from _read_in_parallel(
+ api_url,
+ make_session,
+ graphql_object,
+ logger,
+ windows,
+ filter_terms,
+ resumable_source_manager,
+ )
+ return
initial_cursor = (
resume_config.cursor if resume_config is not None and resume_config.phase == PHASE_ALL else None
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py
index c8a708b213eb..13f1a5df0ba7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/source.py
@@ -1,3 +1,4 @@
+import functools
from datetime import date
from typing import Optional, cast
@@ -11,6 +12,8 @@ from products.warehouse_sources.backend.facade.source_config import (
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
)
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_schema_flag_enabled
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
FieldType,
ResumableSource,
@@ -61,6 +64,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.sh
)
from products.warehouse_sources.backend.types import ExternalDataSourceType
+WAREHOUSE_SHOPIFY_PARALLEL_READ_FLAG = "warehouse-shopify-parallel-read"
+
+
+def _is_parallel_read_enabled(team_id: int, schema_id: str) -> bool:
+ schema = ExternalDataSchema.objects.filter(team_id=team_id, id=schema_id).select_related("source").first()
+ return schema is not None and is_schema_flag_enabled(schema, WAREHOUSE_SHOPIFY_PARALLEL_READ_FLAG)
+
@SourceRegistry.register
class ShopifySource(ResumableSource[ShopifySourceConfig, ShopifyResumeConfig]):
@@ -456,4 +466,5 @@ class ShopifySource(ResumableSource[ShopifySourceConfig, ShopifyResumeConfig]):
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
row_filters=inputs.row_filters,
+ use_parallel_read=functools.partial(_is_parallel_read_enabled, inputs.team_id, inputs.schema_id),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.py
new file mode 100644
index 000000000000..337c22c70a72
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/conftest.py
@@ -0,0 +1,12 @@
+import pytest
+from unittest.mock import patch
+
+
[email protected](autouse=True)
+def _parallel_read_flag_off():
+ # The flag lookup reads the schema row, and the contract tests run without a database.
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.shopify.source._is_parallel_read_enabled",
+ return_value=False,
+ ):
+ yield
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.py
new file mode 100644
index 000000000000..8d763533777c
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_parallel_read.py
@@ -0,0 +1,190 @@
+import re
+import threading
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+import pytest
+from unittest.mock import MagicMock, patch
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+ DriveResult,
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.shopify import (
+ ShopifyAuthMethodConfig,
+ ShopifySourceConfig,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.constants import (
+ ABANDONED_CHECKOUTS,
+ SHOPIFY_GRAPHQL_OBJECTS,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.parallel import ReadWindow, _split
+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.shopify import (
+ PHASE_PARALLEL,
+ ShopifyReadStopped,
+ ShopifyResumeConfig,
+ iter_shopify_pages,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.shopify.source import ShopifySource
+
+_SHOPIFY = "products.warehouse_sources.backend.temporal.data_imports.sources.shopify"
+_CLUSTER_START = datetime(2022, 6, 20, tzinfo=UTC)
+
+
+def _orders() -> list[dict[str, Any]]:
+ # Most orders share a few minutes, the way a bulk update leaves them, and the rest spread over years.
+ clustered = [_CLUSTER_START + timedelta(seconds=3 * index) for index in range(200)]
+ spread = [datetime(2019, 1, 1, tzinfo=UTC) + timedelta(days=21 * index) for index in range(100)]
+ return [
+ {"id": f"order-{index}", "updatedAt": moment.isoformat().replace("+00:00", "Z")}
+ for index, moment in enumerate(sorted(clustered + spread))
+ ]
+
+
+_ORDERS = _orders()
+_ORDER_IDS = {order["id"] for order in _ORDERS}
+
+
+class _Store:
+ """Answers the orders search the way Shopify does: filtered on `updated_at`, sorted by it, paged by cursor."""
+
+ def __init__(self, fail_windows_from: datetime | None = None):
+ self.fail_windows_from = fail_windows_from
+
+ def __call__(self, request: RecordedRequest) -> ScriptedResponse:
+ if request.path == "/admin/oauth/access_token":
+ return ScriptedResponse(json={"access_token": "tok"})
+ if request.method == "GET":
+ return ScriptedResponse(json={"access_scopes": []})
+ variables = request.json()["variables"]
+ search = variables.get("query") or ""
+ lower = re.search(r"updated_at:>='([^']+)'", search)
+ upper = re.search(r"updated_at:<'([^']+)'", search)
+ low = datetime.fromisoformat(lower.group(1)) if lower else None
+ high = datetime.fromisoformat(upper.group(1)) if upper else None
+ if self.fail_windows_from is not None and low is not None and low >= self.fail_windows_from:
+ return ScriptedResponse(status=500)
+ matching = [
+ order
+ for order in _ORDERS
+ if (low is None or datetime.fromisoformat(order["updatedAt"]) >= low)
+ and (high is None or datetime.fromisoformat(order["updatedAt"]) < high)
+ ]
+ offset = int(variables.get("cursor") or 0)
+ page = matching[offset : offset + variables["pageSize"]]
+ end = offset + len(page)
+ return ScriptedResponse(
+ json={
+ "data": {
+ "orders": {
+ "nodes": page,
+ "pageInfo": {"hasNextPage": end < len(matching), "endCursor": str(end)},
+ }
+ }
+ }
+ )
+
+
+def _run(store: _Store, *, flag_enabled: bool = True, incremental: bool = True, **inputs: Any) -> DriveResult:
+ config = ShopifySourceConfig(
+ shopify_store_id="my-store",
+ auth_method=ShopifyAuthMethodConfig(
+ selection="client_credentials", shopify_client_id="cid", shopify_client_secret="secret"
+ ),
+ )
+ with patch(f"{_SHOPIFY}.source._is_parallel_read_enabled", return_value=flag_enabled):
+ return SourceDriver(ShopifySource(), config).run(
+ "orders", store, should_use_incremental_field=incremental, **inputs
+ )
+
+
+def _searches(result: DriveResult) -> list[str]:
+ return [
+ request.json()["variables"].get("query") or ""
+ for request in result.requests
+ if request.path.endswith("/graphql.json")
+ ]
+
+
+def test_parallel_read_returns_every_order_once_and_splits_a_dense_range() -> None:
+ result = _run(_Store())
+
+ assert result.raised is None
+ ids = [row["id"] for row in result.rows]
+ assert sorted(ids) == sorted(_ORDER_IDS)
+ lower_bounds = {re.search(r"updated_at:>='([^']+)'", search).group(1) for search in _searches(result)} # type: ignore[union-attr]
+ # The dense minutes get windows of their own, so more than the first window runs.
+ assert sum(1 for bound in lower_bounds if datetime.fromisoformat(bound) >= _CLUSTER_START) >= 2
+ assert all("updated_at:<'" in search for search in _searches(result))
+
+
+def test_resumed_parallel_read_returns_exactly_the_orders_the_first_attempt_did_not_hand_over() -> None:
+ first = _run(_Store(fail_windows_from=_CLUSTER_START + timedelta(seconds=150)))
+
+ assert first.raised is not None
+ assert isinstance(first.final_state, ShopifyResumeConfig)
+ assert first.final_state.phase == PHASE_PARALLEL
+
+ second = _run(_Store(), resume_state=first.final_state, flag_enabled=False)
+
+ assert second.raised is None
+ first_ids = {row["id"] for row in first.rows}
+ second_ids = [row["id"] for row in second.rows]
+ assert len(second_ids) == len(set(second_ids))
+ assert first_ids.isdisjoint(second_ids)
+ assert first_ids | set(second_ids) == _ORDER_IDS
+
+
+ "kwargs",
+ [{"flag_enabled": False}, {"incremental": False}],
+ ids=["flag_off", "full_refresh"],
+)
+def test_reads_pages_in_one_chain_when_the_parallel_read_does_not_apply(kwargs: dict[str, Any]) -> None:
+ result = _run(_Store(), **kwargs)
+
+ assert sorted(row["id"] for row in result.rows) == sorted(_ORDER_IDS)
+ assert not any("updated_at" in search for search in _searches(result))
+
+
+def test_a_resumed_window_splits_above_the_rows_it_already_handed_over() -> None:
+ window = ReadWindow(
+ lo=_CLUSTER_START, query_hi=_CLUSTER_START + timedelta(hours=4), stop_hi=_CLUSTER_START + timedelta(hours=4)
+ )
+ window.progress = _CLUSTER_START + timedelta(hours=3)
+ window.cursor = "40"
+
+ resumed = ReadWindow.from_state(window.to_state())
+ new_window = _split([resumed], [resumed])
+
+ assert new_window is not None
+ # Splitting from `lo` would give the new window a range the resumed window already read.
+ assert new_window.lo > window.progress
+ assert resumed.stop_hi == new_window.lo
+
+
+def test_a_stopped_read_asks_shopify_for_no_more_pages() -> None:
+ stop = threading.Event()
+ response = MagicMock()
+ response.status_code = 200
+ response.json.return_value = {
+ "data": {ABANDONED_CHECKOUTS: {"nodes": [{"id": "1"}], "pageInfo": {"hasNextPage": True, "endCursor": "2"}}}
+ }
+ sess = MagicMock()
+ sess.post.return_value = response
+
+ pages = iter_shopify_pages(
+ "https://example.invalid/graphql",
+ sess,
+ SHOPIFY_GRAPHQL_OBJECTS[ABANDONED_CHECKOUTS],
+ MagicMock(),
+ stop=stop,
+ )
+
+ assert next(pages).next_cursor == "2"
+ stop.set()
+ with pytest.raises(ShopifyReadStopped):
+ next(pages)
+ assert sess.post.call_count == 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
index 79f764eb4bce..d54ef5382d38 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/tests/test_resumable.py
@@ -164,7 +164,7 @@ class TestMakePaginatedShopifyRequest:
captured: list[tuple[int, str | None]] = []
sess = MagicMock()
- def post(url: str, json: dict[str, Any]) -> MagicMock:
+ def post(url: str, json: dict[str, Any], **kwargs: Any) -> MagicMock:
captured.append((json["variables"]["pageSize"], json["variables"].get("cursor")))
outcome = outcomes[len(captured) - 1]
if isinstance(outcome, Exception):