Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): resume sentry tag values after the last finished tag (#126526)

Kyle Swank4e8485288fe5
diff --git a/products/warehouse_sources/sources/sentry/sentry.py b/products/warehouse_sources/sources/sentry/sentry.py
index 3ed352510991..79a5a6936e1b 100644
--- a/products/warehouse_sources/sources/sentry/sentry.py
+++ b/products/warehouse_sources/sources/sentry/sentry.py
@@ -1,4 +1,5 @@
import re
+import itertools
import dataclasses
from collections.abc import Callable, Iterable, Iterator
from datetime import UTC, datetime, timedelta
@@ -56,16 +57,16 @@ _RETRYABLE_STATUS_CODES = (429, 500, 502, 503, 504, *CLOUDFLARE_TRANSIENT_STATUS
# Upper bound on a server-provided 429 wait, so a misreported Retry-After or reset header cannot
# park a worker. Matches the default limit of the shared REST client.
_MAX_RETRY_AFTER_SECONDS = 300.0
-# Safety bound for how many issues the issue_tag_values fan-out will skip while
-# fast-forwarding to a saved checkpoint issue. If the checkpoint issue was
+# Safety bound for how many issues past its saved position the issue_tag_values fan-out
+# will skip while fast-forwarding to a saved checkpoint issue. If the checkpoint issue was
# deleted between runs, we'd otherwise skip every remaining issue and yield
-# nothing. Once this bound is exceeded we treat the checkpoint as stale and
-# fall through to fresh processing of the current and remaining issues.
+# nothing. Once this bound is exceeded we treat the checkpoint as stale, clear it and
+# fail the attempt, so the next attempt reads every issue without a cursor.
_RESUME_ISSUE_SKIP_LIMIT = 5000
logger = structlog.get_logger(__name__)
[email protected](frozen=True)
class SentryResumeConfig:
"""Resume state for Sentry endpoints.
@@ -73,9 +74,15 @@ class SentryResumeConfig:
``next_url`` returned by ``SentryPaginator``.
``issue_tag_values`` is a three-level hand-rolled fan-out
- (issues -> tags-per-issue -> values-per-tag); its checkpoint is the
- ``(issue_id, tag_key, values_next_url)`` triple pointing at the next
- values page to fetch for that specific (issue, tag) combination.
+ (issues -> tags-per-issue -> values-per-tag). Its checkpoint is one of:
+
+ - ``issue_id`` with ``completed_tag_keys``, the tags of that issue already read in full,
+ plus ``tag_key`` and ``values_next_url`` when a tag stopped between two values pages.
+ - ``completed_issue_id``, the last issue whose tags were all read.
+
+ The finished tags are kept by key and not by position, because the tags listing order is not
+ guaranteed between attempts. ``issue_position`` is where the checkpoint issue sits in the
+ issue order, so a resume can tell a deleted checkpoint issue from one that is far down.
Parent/child fan-out endpoints driven by ``build_dependent_resource``
don't currently checkpoint — the framework does not expose a resume
@@ -86,6 +93,10 @@ class SentryResumeConfig:
issue_id: Optional[str] = None
tag_key: Optional[str] = None
values_next_url: Optional[str] = None
+ # None on a checkpoint from code that predates it, which skips every tag listed before tag_key.
+ completed_tag_keys: Optional[list[str]] = None
+ completed_issue_id: Optional[str] = None
+ issue_position: Optional[int] = None
# Which issue ordering the fan-out checkpoint above is a position in: None for the API's
# `sort=date` listing, or the pinned Delta version when the parent came from the warehouse.
# Resuming across a change here would fast-forward past issues the new order never reached.
@@ -286,6 +297,10 @@ class SentryRateLimitedError(Exception):
"""Sentry kept returning 429 until the request-level retry budget ran out."""
+class SentryStaleResumeCheckpointError(Exception):
+ """The issue_tag_values checkpoint issue is not in the listing, and this attempt cannot read the issues again without duplicates."""
+
+
def _is_retryable_response(response: Response) -> bool:
return response.status_code in _RETRYABLE_STATUS_CODES
@@ -496,9 +511,9 @@ def _usable_resume_state(
A checkpoint is a position in an iteration order, so it only means anything to a run
walking the same order: the API's `sort=date` listing, or one pinned Delta version.
Applying one across that boundary fast-forwards past issues the new order never reached
- while the watermark still advances, so the rows are lost until a reset. The full triple
- has to be present — anything partial is treated as absent rather than applied to the
- wrong (issue, tag) pair.
+ while the watermark still advances, so the rows are lost until a reset. The checkpoint
+ has to name a values page, a finished tag or a completed issue. Anything partial is
+ treated as absent rather than applied to the wrong (issue, tag) pair.
Callers must also `clear_state()` when this returns None with state still stored: the
pipeline reads `can_resume()` itself to pick replace-vs-append for chunk 0, so a source
@@ -507,13 +522,96 @@ def _usable_resume_state(
if manager is None or not manager.can_resume():
return None
loaded = manager.load_state()
- if loaded is None or not (loaded.issue_id and loaded.tag_key and loaded.values_next_url):
+ if loaded is None:
+ return None
+ in_values_page = bool(loaded.issue_id and loaded.tag_key and loaded.values_next_url)
+ after_finished_tags = bool(loaded.issue_id and loaded.completed_tag_keys)
+ if not (in_values_page or after_finished_tags or loaded.completed_issue_id):
return None
if loaded.parent_version != parent_version:
return None
return loaded
[email protected](frozen=True)
+class _ListedIssue:
+ position: int
+ issue: dict[str, Any]
+ is_checkpoint: bool
+
+
+def _resume_issue_order(
+ list_issues: Callable[[], Iterable[dict[str, Any]]],
+ checkpoint_issue_id: str | None,
+ checkpoint_issue_done: bool,
+ skip_limit: int,
+ listing_floor: datetime | None,
+ clear_checkpoint: Callable[[], None],
+) -> Iterator[_ListedIssue]:
+ """Yield each issue from the checkpoint on, with its position and whether it is the checkpoint issue.
+
+ This assumes the issue order of the earlier attempt, so the issues listed before the checkpoint
+ issue were read and are skipped. Sentry orders `sort=date` by last seen, so an issue with a new
+ event can move ahead of the checkpoint and get skipped. The checkpoint issue can also be deleted or
+ merged between attempts. `sentry_source` clears the checkpoint of a deleted or merged issue before
+ the attempt starts. When the checkpoint issue is still not found within `skip_limit` positions, or
+ before the listing ends or reaches `listing_floor`, the checkpoint is cleared and the attempt fails.
+ A resume that skips every issue would complete with no rows and report success. Reading the issues
+ again in this attempt would append the rows of the earlier attempt a second time, because a
+ resumed attempt appends whatever the sync type is. The next attempt has no cursor, so the
+ pipeline loads it the way it loads a new attempt.
+ """
+ positioned = enumerate(list_issues(), start=1)
+ if checkpoint_issue_id is not None:
+ found = False
+ for position, issue in positioned:
+ if str(issue["id"]) == checkpoint_issue_id:
+ found = True
+ if not checkpoint_issue_done:
+ yield _ListedIssue(position=position, issue=issue, is_checkpoint=True)
+ break
+ if position > skip_limit:
+ break
+ if listing_floor is not None:
+ issue_last_seen = _parse_datetime_value(issue.get("lastSeen"))
+ if issue_last_seen is not None and issue_last_seen <= listing_floor:
+ break
+ if not found:
+ logger.info("sentry_source.stale_resume_checkpoint", resume_issue_id=checkpoint_issue_id)
+ clear_checkpoint()
+ raise SentryStaleResumeCheckpointError(
+ "The resume checkpoint issue is no longer listed, so the sync starts again from the first issue."
+ )
+ for position, issue in positioned:
+ yield _ListedIssue(position=position, issue=issue, is_checkpoint=False)
+
+
+def _checkpoint_issue_is_gone(
+ base_api_url: str, headers: dict[str, str], organization_slug: str, issue_id: str
+) -> bool:
+ """Whether Sentry no longer has the checkpoint issue under its id.
+
+ An issue merged into another one can answer with the other issue, so a different id counts as
+ gone. Any other answer keeps the checkpoint, because the iterator still bounds its search. One
+ request with no retries, because it runs before the pipeline starts and a rate-limit wait would
+ hold the attempt there.
+ """
+ url = urljoin(f"{base_api_url}/", f"organizations/{organization_slug}/issues/{quote(issue_id, safe='')}/")
+ try:
+ response = make_tracked_session(retry=Retry(total=0)).get(url, headers=headers, timeout=_REQUEST_TIMEOUT)
+ except RequestException:
+ return False
+ if response.status_code == 404:
+ return True
+ if not response.ok:
+ return False
+ try:
+ payload = response.json()
+ except ValueError:
+ return False
+ return isinstance(payload, dict) and payload.get("id") is not None and str(payload["id"]) != issue_id
+
+
def _iter_issue_tag_values_rows(
base_api_url: str,
headers: dict[str, str],
@@ -526,26 +624,25 @@ def _iter_issue_tag_values_rows(
cutoff_last_seen = _parse_datetime_value(incremental_last_seen_max)
use_warehouse_parent = issues_table is not None
- issues: Iterator[dict[str, Any]]
- if issues_table is not None:
- # Streamed scan — the reader must never materialize the table. The incremental
- # early-break below becomes a per-row filter in this mode (same issue set, no
- # ordering requirement). Duplicate rows can't occur: append-mode parents take the
- # API path instead of this one.
- issues = (
- row
- for page in iter_parent_pages_from_warehouse(
- table=issues_table,
- parent_name="issues",
- columns=_ISSUES_PARENT_COLUMNS,
- page_size=100,
- schema_name="issue_tag_values",
- row_filter=_issues_parent_row_filter(cutoff_last_seen),
+ def list_issues() -> Iterator[dict[str, Any]]:
+ if issues_table is not None:
+ # Streamed scan — the reader must never materialize the table. The incremental
+ # early-break below becomes a per-row filter in this mode (same issue set, no
+ # ordering requirement). Duplicate rows can't occur: append-mode parents take the
+ # API path instead of this one.
+ return (
+ row
+ for page in iter_parent_pages_from_warehouse(
+ table=issues_table,
+ parent_name="issues",
+ columns=_ISSUES_PARENT_COLUMNS,
+ page_size=100,
+ schema_name="issue_tag_values",
+ row_filter=_issues_parent_row_filter(cutoff_last_seen),
+ )
+ for row in page
)
- for row in page
- )
- else:
- issues = _iter_endpoint_rows(
+ return _iter_endpoint_rows(
base_api_url=base_api_url,
path=f"/organizations/{organization_slug}/issues/",
headers=headers,
@@ -555,19 +652,43 @@ def _iter_issue_tag_values_rows(
# A pinned Delta version enumerates its files in a fixed order, so a warehouse run
# resumes like an API run — but only against a checkpoint written over the same pin.
parent_version = issues_table.version if issues_table is not None else None
- resume_issue_id: str | None = None
+ checkpoint_issue_id: str | None = None
+ checkpoint_issue_done = False
resume_tag_key: str | None = None
resume_values_next_url: str | None = None
+ resume_done_tag_keys: list[str] = []
+ resume_skips_to_tag = False
+ resume_skip_limit = _RESUME_ISSUE_SKIP_LIMIT
loaded = _usable_resume_state(resumable_source_manager, parent_version)
if loaded is not None:
- resume_issue_id = loaded.issue_id
- resume_tag_key = loaded.tag_key
- resume_values_next_url = loaded.values_next_url
+ resume_skip_limit += loaded.issue_position or 0
+ if loaded.completed_issue_id:
+ checkpoint_issue_id = loaded.completed_issue_id
+ checkpoint_issue_done = True
+ else:
+ checkpoint_issue_id = loaded.issue_id
+ resume_tag_key = loaded.tag_key
+ resume_values_next_url = loaded.values_next_url
+ resume_done_tag_keys = list(loaded.completed_tag_keys or [])
+ resume_skips_to_tag = loaded.completed_tag_keys is None
+
+ def clear_checkpoint() -> None:
+ if resumable_source_manager is not None:
+ resumable_source_manager.clear_state()
stale_issues: set[str] = set()
- skipped_for_resume = 0
- for issue in issues:
+ for listed in _resume_issue_order(
+ list_issues,
+ checkpoint_issue_id,
+ checkpoint_issue_done,
+ resume_skip_limit,
+ None if use_warehouse_parent else cutoff_last_seen,
+ clear_checkpoint=clear_checkpoint,
+ ):
+ issue = listed.issue
+ issue_position = listed.position
+ is_checkpoint_issue = listed.is_checkpoint
if cutoff_last_seen is not None:
issue_last_seen = _parse_datetime_value(issue.get("lastSeen"))
if issue_last_seen is not None and issue_last_seen <= cutoff_last_seen:
@@ -580,30 +701,10 @@ def _iter_issue_tag_values_rows(
issue_id = str(issue["id"])
- # Fast-forward until we reach the saved checkpoint issue. We rely on
- # the deterministic sort=date ordering to land back on the same issue.
- # If the checkpoint issue has been deleted we could skip forever, so
- # bound the skip count and fall through to a fresh run when exceeded.
- if resume_issue_id is not None and issue_id != resume_issue_id:
- skipped_for_resume += 1
- if skipped_for_resume > _RESUME_ISSUE_SKIP_LIMIT:
- logger.info(
- "sentry_source.stale_resume_checkpoint",
- resume_issue_id=resume_issue_id,
- skipped=skipped_for_resume,
- )
- resume_issue_id = None
- resume_tag_key = None
- resume_values_next_url = None
- # Fall through: process the current issue and subsequent ones fresh.
- else:
- continue
-
- # Mark that we've found the checkpoint issue. If the checkpoint tag has
- # since disappeared, we still exit the middle loop with no match — clear
- # outer fast-forward state at the end of this iteration so subsequent
- # issues run fresh instead of being skipped forever.
- matched_checkpoint_issue = resume_issue_id is not None
+ done_tag_keys = list(resume_done_tag_keys) if is_checkpoint_issue else []
+ pending_tag_key = resume_tag_key if is_checkpoint_issue else None
+ pending_values_url = resume_values_next_url if is_checkpoint_issue else None
+ skips_to_tag = resume_skips_to_tag and is_checkpoint_issue
tags = _iter_endpoint_rows(
base_api_url=base_api_url,
@@ -617,27 +718,33 @@ def _iter_issue_tag_values_rows(
# schema last synced; their tags endpoint 404s. A fresh API parent pull would
# simply not list them, so skip instead of failing the sync.
tags = _skip_rows_on_stale_issue_404(tags, organization_slug, issue_id, stale_issues)
- for tag in _skip_issue_on_tags_server_error(tags, organization_slug, issue_id):
+ listed_tags = _skip_issue_on_tags_server_error(tags, organization_slug, issue_id)
+ # The saved values page goes first, so a checkpoint for another tag cannot replace its cursor.
+ first_tag_key = pending_tag_key if pending_values_url and not skips_to_tag else None
+ tag_stream = itertools.chain([{"key": first_tag_key}], listed_tags) if first_tag_key else listed_tags
+ for tag in tag_stream:
tag_key = tag.get("key") or tag.get("id")
if not isinstance(tag_key, str) or not tag_key:
continue
- if resume_issue_id is not None and resume_tag_key is not None and tag_key != resume_tag_key:
+ if tag_key in done_tag_keys:
+ continue
+ if skips_to_tag and pending_tag_key is not None and tag_key != pending_tag_key:
+ done_tag_keys.append(tag_key)
continue
values_path = f"/organizations/{organization_slug}/issues/{issue_id}/tags/{quote(tag_key, safe='')}/values/"
- if resume_issue_id is not None and resume_values_next_url:
- values_url: str = resume_values_next_url
+ if pending_values_url and tag_key == pending_tag_key:
+ values_url: str = pending_values_url
values_params: dict[str, Any] | None = None
else:
values_url = urljoin(f"{base_api_url}/", values_path.lstrip("/"))
values_params = {"limit": 100, "sort": "-date"}
pages_read = 0
- # Clear resume markers so the NEXT (issue, tag) pair runs fresh.
- resume_issue_id = None
- resume_tag_key = None
- resume_values_next_url = None
+ if tag_key == pending_tag_key:
+ pending_tag_key = None
+ pending_values_url = None
while values_url:
if pages_read >= _MAX_PAGES_PER_PARENT:
@@ -694,6 +801,15 @@ def _iter_issue_tag_values_rows(
tag_key=tag_key,
)
break
+ # The saved page is read before the tags listing, so it 404s for a dropped tag.
+ if response.status_code == 404 and tag_key == first_tag_key:
+ logger.info(
+ "sentry_source.resumed_tag_not_found",
+ organization_slug=organization_slug,
+ issue_id=issue_id,
+ tag_key=tag_key,
+ )
+ break
# Other client errors (401, etc.) still propagate to the job-level handler.
raise
@@ -748,19 +864,38 @@ def _iter_issue_tag_values_rows(
issue_id=issue_id,
tag_key=tag_key,
values_next_url=urljoin(f"{base_api_url}/", next_url),
+ completed_tag_keys=list(done_tag_keys),
+ issue_position=issue_position,
parent_version=parent_version,
)
)
+ resumable_source_manager.safe_point()
if not next_url:
break
values_url = urljoin(f"{base_api_url}/", next_url)
values_params = None
- if matched_checkpoint_issue:
- resume_issue_id = None
- resume_tag_key = None
- resume_values_next_url = None
+ done_tag_keys.append(tag_key)
+ if resumable_source_manager is not None:
+ resumable_source_manager.save_state(
+ SentryResumeConfig(
+ issue_id=issue_id,
+ completed_tag_keys=list(done_tag_keys),
+ issue_position=issue_position,
+ parent_version=parent_version,
+ )
+ )
+ resumable_source_manager.safe_point()
+
+ if resumable_source_manager is not None:
+ # Lets a resume skip this issue without listing its tags again.
+ resumable_source_manager.save_state(
+ SentryResumeConfig(
+ completed_issue_id=issue_id, issue_position=issue_position, parent_version=parent_version
+ )
+ )
+ resumable_source_manager.safe_point()
if use_warehouse_parent:
# Stale issues are ones the snapshot still lists but Sentry has dropped, each costing
@@ -1469,7 +1604,17 @@ def sentry_source(
# The pipeline reads this same Redis state to pick replace-vs-append for chunk 0,
# so state the iterator will refuse has to go now, before it decides. Same
# predicate as the iterator's, so the two can't disagree.
- if _usable_resume_state(resumable_source_manager, issues_table.version if issues_table else None) is None:
+ loaded = _usable_resume_state(resumable_source_manager, issues_table.version if issues_table else None)
+ checkpoint_issue_id = (loaded.completed_issue_id or loaded.issue_id) if loaded is not None else None
+ if loaded is None:
+ resumable_source_manager.clear_state()
+ # A pinned Delta version cannot lose an issue, so only the live listing needs the check.
+ elif (
+ issues_table is None
+ and checkpoint_issue_id
+ and _checkpoint_issue_is_gone(base_api_url, headers, organization_slug, checkpoint_issue_id)
+ ):
+ logger.info("sentry_source.resume_checkpoint_issue_gone", resume_issue_id=checkpoint_issue_id)
resumable_source_manager.clear_state()
return _make_source_response(
endpoint_config,
diff --git a/products/warehouse_sources/sources/sentry/tests/test_sentry.py b/products/warehouse_sources/sources/sentry/tests/test_sentry.py
index f95220c0a248..c3e44880bb62 100644
--- a/products/warehouse_sources/sources/sentry/tests/test_sentry.py
+++ b/products/warehouse_sources/sources/sentry/tests/test_sentry.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.sources.sentry.sentry import (
SentryRateLimitedError,
SentryResumeConfig,
SentrySessionsRejectedError,
+ SentryStaleResumeCheckpointError,
SentryStatsSummaryRejectedError,
_custom_endpoint_rows,
_normalize_organization_slug,
@@ -848,9 +849,38 @@ class TestIssueTagValuesResumable:
issue_id="100",
tag_key="browser",
values_next_url="https://sentry.io/api/0/organizations/acme/issues/100/tags/browser/values/?cursor=0:100:2",
- )
+ completed_tag_keys=[],
+ issue_position=1,
+ ),
+ SentryResumeConfig(issue_id="100", completed_tag_keys=["browser"], issue_position=1),
+ SentryResumeConfig(completed_issue_id="100", issue_position=1),
]
+ def test_failed_run_resumes_after_the_last_finished_tag(self) -> None:
+ failing_values = "/organizations/acme/issues/101/tags/browser/values/"
+
+ def script(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/organizations/acme/issues/"):
+ return _json([{"id": "100"}, {"id": "101"}])
+ if request.path.endswith(failing_values):
+ return _json({"detail": "Invalid token"}, status=401)
+ if request.path.endswith("/tags/"):
+ return _json([{"key": "os"}, {"key": "browser"}])
+ return _json([{"value": "x"}])
+
+ failed = _driver().run("issue_tag_values", script)
+
+ assert failed.raised is not None
+ assert failed.final_state == SentryResumeConfig(issue_id="101", completed_tag_keys=["os"], issue_position=2)
+
+ failing_values = "none"
+ resumed = _driver().run("issue_tag_values", script, resume_state=failed.final_state)
+
+ assert resumed.raised is None
+ assert resumed.rows == [{"value": "x", "issue_id": "101", "tag_key": "browser"}]
+ assert "/api/0/organizations/acme/issues/100/tags/" not in resumed.paths
+ assert "/api/0/organizations/acme/issues/101/tags/os/values/" not in resumed.paths
+
@parameterized.expand(
[
# A watermark inside the window is the tighter floor, so the scan stops there.
@@ -973,35 +1003,144 @@ class TestIssueTagValuesResumable:
values_path = "/api/0/organizations/acme/issues/100/tags/browser/values/"
values_requests = [request for request in result.requests if request.path == values_path]
assert [request.param("cursor") for request in values_requests] == ["0:100:2"]
+ assert SentryResumeConfig(issue_id="100", completed_tag_keys=["os", "browser"], issue_position=2) in (
+ result.saved_states
+ )
- @patch("products.warehouse_sources.sources.sentry.sentry._RESUME_ISSUE_SKIP_LIMIT", 2)
- def test_stale_checkpoint_falls_through_after_skip_limit(self) -> None:
- """If the checkpoint issue was deleted between runs, bounded skipping
- falls through so subsequent issues still get processed."""
+ @parameterized.expand(
+ [
+ (
+ "other_tag_fails",
+ False,
+ SentryResumeConfig(
+ issue_id="100",
+ tag_key="os",
+ values_next_url="https://sentry.io/api/0/organizations/acme/issues/100/tags/os/values/?cursor=0:100:2",
+ completed_tag_keys=["browser"],
+ issue_position=1,
+ ),
+ ),
+ ("saved_tag_dropped", True, SentryResumeConfig(completed_issue_id="100", issue_position=1)),
+ ]
+ )
+ def test_resume_reads_the_saved_values_page_before_other_tags(
+ self, _name: str, saved_tag_dropped: bool, expected_final_state: SentryResumeConfig
+ ) -> None:
+ resume_url = "https://sentry.io/api/0/organizations/acme/issues/100/tags/browser/values/?cursor=0:100:2"
+ os_next_link = (
+ "<https://sentry.io/api/0/organizations/acme/issues/100/tags/os/values/?cursor=0:100:2>; "
+ 'rel="next"; results="true"'
+ )
def script(request: RecordedRequest) -> ScriptedResponse:
if request.path.endswith("/organizations/acme/issues/"):
- # None of these match the checkpoint issue_id=999.
- return _json([{"id": "100"}, {"id": "101"}, {"id": "102"}])
- if request.path.endswith("/organizations/acme/issues/102/tags/"):
- return _json([{"key": "browser"}])
- if request.path.endswith("/organizations/acme/issues/102/tags/browser/values/"):
- return _json([{"value": "Chrome"}])
- return _json([])
+ return _json([{"id": "100"}])
+ if request.path.endswith("/organizations/acme/issues/100/tags/"):
+ return _json([{"key": "os"}] if saved_tag_dropped else [{"key": "os"}, {"key": "browser"}])
+ if request.url == resume_url or request.path.endswith("/tags/browser/values/"):
+ if saved_tag_dropped:
+ return _json({"detail": "Not found"}, status=404)
+ return _json([{"value": "Firefox"}])
+ if request.param("cursor") == "0:100:2":
+ return _json([]) if saved_tag_dropped else _json({"detail": "Invalid token"}, status=401)
+ return _json([{"value": "Linux"}], link_header=os_next_link)
result = _driver().run(
"issue_tag_values",
script,
resume_state=SentryResumeConfig(
- issue_id="999",
+ issue_id="100",
tag_key="browser",
- values_next_url="https://sentry.io/api/0/organizations/acme/issues/999/tags/browser/values/?cursor=0:100:2",
+ values_next_url=resume_url,
+ completed_tag_keys=[],
+ issue_position=1,
),
)
- # With skip limit 2, issues 100 and 101 are skipped; on 102 we exceed
- # the limit, clear the markers, and process it fresh.
- assert result.rows == [{"value": "Chrome", "issue_id": "102", "tag_key": "browser"}]
+ assert (result.raised is None) == saved_tag_dropped
+ assert result.final_state == expected_final_state
+
+ @parameterized.expand(
+ [
+ (
+ "deleted_checkpoint_issue",
+ SentryResumeConfig(
+ issue_id="999",
+ tag_key="browser",
+ values_next_url="https://sentry.io/api/0/organizations/acme/issues/999/tags/browser/values/?cursor=0:100:2",
+ ),
+ _json({"detail": "Not found"}, status=404),
+ False,
+ ["100", "101", "102", "103"],
+ ),
+ (
+ "merged_checkpoint_issue",
+ SentryResumeConfig(completed_issue_id="999", issue_position=10),
+ _json({"id": "100"}),
+ False,
+ ["100", "101", "102", "103"],
+ ),
+ (
+ "checkpoint_past_the_limit",
+ SentryResumeConfig(completed_issue_id="102", issue_position=3),
+ _json({"id": "102"}),
+ False,
+ ["103"],
+ ),
+ (
+ "checkpoint_issue_not_listed",
+ SentryResumeConfig(completed_issue_id="999", issue_position=10),
+ _json({"id": "999"}),
+ False,
+ None,
+ ),
+ (
+ "checkpoint_issue_not_listed_on_a_watermark_run",
+ SentryResumeConfig(completed_issue_id="999", issue_position=10),
+ _json({"id": "999"}),
+ True,
+ None,
+ ),
+ ]
+ )
+ @patch("products.warehouse_sources.sources.sentry.sentry._RESUME_ISSUE_SKIP_LIMIT", 1)
+ def test_checkpoint_issue_search_is_bounded(
+ self,
+ _name: str,
+ resume_state: SentryResumeConfig,
+ checkpoint_issue_response: ScriptedResponse,
+ incremental: bool,
+ expected_issue_ids: list[str] | None,
+ ) -> None:
+ def script(request: RecordedRequest) -> ScriptedResponse:
+ if request.path.endswith("/organizations/acme/issues/"):
+ return _json([{"id": "100"}, {"id": "101"}, {"id": "102"}, {"id": "103"}])
+ if request.path.endswith(
+ f"/organizations/acme/issues/{resume_state.completed_issue_id or resume_state.issue_id}/"
+ ):
+ return checkpoint_issue_response
+ if request.path.endswith("/tags/"):
+ return _json([{"key": "browser"}])
+ return _json([{"value": "Chrome"}])
+
+ incremental_inputs: dict[str, Any] = (
+ {
+ "should_use_incremental_field": True,
+ "db_incremental_field_last_value": datetime(2026, 1, 1, tzinfo=UTC),
+ "incremental_field": "lastSeen",
+ }
+ if incremental
+ else {}
+ )
+ result = _driver().run("issue_tag_values", script, resume_state=resume_state, **incremental_inputs)
+
+ if expected_issue_ids is None:
+ assert isinstance(result.raised, SentryStaleResumeCheckpointError)
+ assert result.rows == []
+ assert result.final_state is None
+ else:
+ assert result.raised is None
+ assert [row["issue_id"] for row in result.rows] == expected_issue_ids
def test_partial_resume_state_falls_through_to_fresh_run(self) -> None:
"""Only activate resume when the full (issue_id, tag_key, values_next_url)