Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): sync auth0 users past the 1000-result search cap (#122120)

Luke Beltonb7abefea09ba
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py
index 1060dd845c82..d275fce87f1b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/auth0.py
@@ -314,8 +314,8 @@ def _build_params(
# watermark to checkpoint per batch and for the window to slide forward.
params["sort"] = f"{window_field}:1"
if window_start is not None:
- # Inclusive lower bound: the boundary row is re-read and deduped by the merge on the
- # primary key, which is cheaper than risking a skipped row on an exclusive bound.
+ # Inclusive lower bound: the boundary rows are re-read and dropped in `_iter_pages`,
+ # which is cheaper than risking a skipped row on an exclusive bound.
params["q"] = f'{window_field}:["{window_start}" TO *]'
return params
@@ -428,6 +428,10 @@ def _max_window_value(rows: list[dict[str, Any]], window_field: str) -> Optional
return max(values) if values else None
+def _row_key(row: dict[str, Any], config: Auth0EndpointConfig) -> tuple[Any, ...]:
+ return tuple(row.get(key) for key in config.primary_keys)
+
+
def _probe(session: requests.Session, token: str, url: str, read_body: bool) -> tuple[int, Any]:
"""Request one probe URL and return its status, plus the parsed body when asked for it.
@@ -553,6 +557,13 @@ def _iter_pages(
page = position.page
window_start = position.window_start
pages_fetched = 0
+ # Keys of the rows seen at the newest window value so far, and of the rows the previous window
+ # yielded at the current lower bound. The inclusive bound re-reads those rows, and a full
+ # refresh appends batches without a merge, so they are dropped rather than written twice. They
+ # are not saved in the resume state: a restart at a window boundary can write a few rows twice.
+ newest_seen: Optional[str] = None
+ newest_seen_keys: set[tuple[Any, ...]] = set()
+ already_yielded: set[tuple[Any, ...]] = set()
while True:
if pages_fetched >= MAX_PAGES:
raise Auth0PaginationLimitError(
@@ -563,21 +574,39 @@ def _iter_pages(
data = fetch(_build_url(domain, config, api_version, params))
pages_fetched += 1
- rows = _extract_rows(data, config)
- if not rows:
+ fetched_rows = _extract_rows(data, config)
+ if not fetched_rows:
return
+ # A re-read row whose window value moved past the boundary is a newer version, so it stays.
+ rows = [
+ row
+ for row in fetched_rows
+ if not (
+ window_field and str(row.get(window_field)) == window_start and _row_key(row, config) in already_yielded
+ )
+ ]
next_page = page + 1
total = data.get("total") if isinstance(data, dict) else None
- is_last_page = (isinstance(total, int) and next_page * config.page_size >= total) or (
- len(rows) < config.page_size
- )
+ # Auth0 reports `total` as the result cap whenever more rows match, so on a windowed
+ # collection only a short page ends the walk.
+ is_last_page = (
+ window_field is None and isinstance(total, int) and next_page * config.page_size >= total
+ ) or len(fetched_rows) < config.page_size
+
+ if window_field:
+ page_newest = _max_window_value(fetched_rows, window_field)
+ if page_newest != newest_seen:
+ newest_seen, newest_seen_keys = page_newest, set()
+ newest_seen_keys.update(
+ _row_key(row, config) for row in fetched_rows if str(row.get(window_field)) == page_newest
+ )
next_window_start = window_start
if not is_last_page and window_field and next_page * config.page_size >= SEARCH_RESULT_CAP:
# Auth0 refuses to page past the first 1000 matches, so restart at page 0 with the
# window's lower bound moved up to the newest row seen.
- newest = _max_window_value(rows, window_field)
+ newest = newest_seen
if newest is None:
raise Auth0PaginationStalledError(
f"{PAGINATION_STALLED_ERROR}: {config.name} rows carry no `{window_field}` value to slide on"
@@ -599,6 +628,8 @@ def _iter_pages(
if is_last_page:
return
+ if next_window_start != window_start:
+ already_yielded = set(newest_seen_keys)
page = next_page
window_start = next_window_start
@@ -828,7 +859,8 @@ def get_rows(
# of silently re-reading (and, for a full refresh, duplicating) this one.
resumable_source_manager.save_state(page.next_position)
- yield _strip_credential_fields(page.rows, endpoint)
+ if page.rows:
+ yield _strip_credential_fields(page.rows, endpoint)
def auth0_source(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py
index 583a552b8f26..b5d20dc42262 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/auth0/tests/test_auth0.py
@@ -269,9 +269,15 @@ class TestGetRows:
def test_window_slides_past_the_search_result_cap(self) -> None:
newest = "2024-06-02T00:00:00.000Z"
- page_0 = {"users": _user_rows(100, "2024-06-01T00:00:00.000Z"), "total": 5000}
- page_1 = {"users": _user_rows(100, newest, start=100), "total": 5000}
- page_2 = {"users": _user_rows(3, newest, start=200), "total": 3}
+ page_0 = {"users": _user_rows(100, "2024-06-01T00:00:00.000Z"), "total": 200}
+ page_1 = {
+ "users": _user_rows(50, "2024-06-01T00:00:00.000Z", start=100) + _user_rows(50, newest, start=150),
+ "total": 200,
+ }
+ # The inclusive lower bound re-reads 49 users already synced at `newest`. `auth0|150` comes
+ # back too, updated after the slide, so its newer version must not be dropped.
+ updated = {"user_id": "auth0|150", "updated_at": "2024-06-03T00:00:00.000Z"}
+ page_2 = {"users": [*_user_rows(52, newest, start=151), updated], "total": 53}
with mock.patch.object(auth0_module, "SEARCH_RESULT_CAP", 200):
result = self._run(
@@ -281,7 +287,8 @@ class TestGetRows:
)
assert result.raised is None
- assert [len(batch) for batch in result.items] == [100, 100, 3]
+ assert [len(batch) for batch in result.items] == [100, 100, 4]
+ assert result.items[2][-1] == updated
# The third request restarts at page 0 with the window's lower bound moved to the newest
# row seen — that is how the collection is walked past Auth0's 1000-result ceiling.
assert result.requests[3].param("page") == "0"
@@ -294,7 +301,7 @@ class TestGetRows:
def test_window_that_cannot_advance_fails_loudly(self) -> None:
stuck = "2024-06-01T00:00:00.000Z"
- page = {"users": _user_rows(100, stuck), "total": 5000}
+ page = {"users": _user_rows(100, stuck), "total": 200}
with mock.patch.object(auth0_module, "SEARCH_RESULT_CAP", 200):
result = self._run(