Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(spaces): crash gate reads Sentry with SENTRY_READ_TOKEN on de.sentry.io (CUA-2343) (#6026)

cua-agents[bot]c5ede63fd7d5
diff --git a/.github/scripts/release_rollout_auto.py b/.github/scripts/release_rollout_auto.py
index 874dbbb0eb..adc01650ea 100644
--- a/.github/scripts/release_rollout_auto.py
+++ b/.github/scripts/release_rollout_auto.py
@@ -37,11 +37,13 @@ previous stable (the most active non-prerelease version in the week before the s
below the previous version's over the same window (needs --min-sessions on the
new version). A previous version with fewer than --min-sessions sessions (it
shipped before crash reporting) is no baseline: the new version must then be at
- least --min-crash-free percent. Without SENTRY_AUTH_TOKEN the gate is reported as
- skipped.
+ least --min-crash-free percent. Without SENTRY_READ_TOKEN the gate is reported as skipped;
+ if Sentry cannot be read (unreachable, 401/403, bad config) it waits and says why.
Standard library only. Keys come from the environment: POSTHOG_API_KEY (query read),
-INCIDENT_IO_API_KEY (optional), SENTRY_AUTH_TOKEN (optional, org:read and project:read).
+INCIDENT_IO_API_KEY (optional), SENTRY_READ_TOKEN (optional, an Internal Integration token with
+Project, Issue & Event and Organization read; the SENTRY_AUTH_TOKEN org:ci token cannot read stats).
+SENTRY_API_HOST overrides the Sentry host (default https://de.sentry.io, the EU region of org cua-ai).
"""
from __future__ import annotations
@@ -62,7 +64,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
import release_rollout as rr # noqa: E402
POSTHOG = "https://eu.posthog.com/api/projects/56041/query/"
-SENTRY = "https://sentry.io/api/0"
+SENTRY_DEFAULT_HOST = "https://de.sentry.io"
SENTRY_CONFIG = Path(__file__).resolve().parents[2] / "config" / "spaces-crash-reporting.json"
NEXT = {"10": "50", "50": "100"}
@@ -224,8 +226,12 @@ def gate_incident(key: str | None, fetch=None) -> dict:
return {"gate": "incident", "status": "pass", "detail": "no active incident naming Spaces"}
+def sentry_base() -> str:
+ return (os.environ.get("SENTRY_API_HOST") or SENTRY_DEFAULT_HOST).rstrip("/") + "/api/0"
+
+
def sentry_get(path: str, token: str, params: list[tuple[str, str]] | None = None) -> dict:
- url = SENTRY + path + ("?" + urllib.parse.urlencode(params) if params else "")
+ url = sentry_base() + path + ("?" + urllib.parse.urlencode(params) if params else "")
req = urllib.request.Request(url, headers={"Authorization": "Bearer " + token})
with urllib.request.urlopen(req, timeout=60) as r:
return json.load(r)
@@ -233,13 +239,19 @@ def sentry_get(path: str, token: str, params: list[tuple[str, str]] | None = Non
def sentry_sessions(token: str, cfg: dict, project_id: str, version: str, since: datetime, now: datetime) -> tuple[int, float | None]:
"""(sessions, crash-free fraction or None) of one release on the stable environment in a window."""
- data = sentry_get(
- "/organizations/%s/sessions/" % cfg["org"], token,
- [("project", project_id), ("environment", "stable"),
- ("field", "sum(session)"), ("field", "crash_free_rate(session)"),
- ("query", 'release:"cua-spaces-desktop@%s"' % version),
- ("start", since.strftime("%Y-%m-%dT%H:%M:%S")), ("end", now.strftime("%Y-%m-%dT%H:%M:%S")),
- ("interval", "1d")])
+ try:
+ data = sentry_get(
+ "/organizations/%s/sessions/" % cfg["org"], token,
+ [("project", project_id), ("environment", "stable"),
+ ("field", "sum(session)"), ("field", "crash_free_rate(session)"),
+ ("query", 'release:"cua-spaces-desktop@%s"' % version),
+ ("start", since.strftime("%Y-%m-%dT%H:%M:%S")), ("end", now.strftime("%Y-%m-%dT%H:%M:%S")),
+ ("interval", "1d")])
+ except urllib.error.HTTPError as e:
+ # Sentry answers 400/404 "Unknown environments selected" until a stable session exists: no data yet.
+ if e.code in (400, 404):
+ return 0, None
+ raise
total, rate = 0, None
for g in data.get("groups", []):
t = g.get("totals", {})
@@ -248,10 +260,16 @@ def sentry_sessions(token: str, cfg: dict, project_id: str, version: str, since:
return total, rate
+def _why(e: Exception) -> str:
+ if isinstance(e, urllib.error.HTTPError):
+ return f"HTTP {e.code} from {(e.filename or '').split('/api/')[0]}"
+ return f"{type(e).__name__}: {e}"
+
+
def gate_crash(token: str | None, new: str, prev: str | None, since: datetime, now: datetime, a, cfg: dict | None = None, fetch=None) -> dict:
"""Crash-free sessions of the new version against the previous one (Sentry release health)."""
if not token:
- return {"gate": "crash", "status": "skip", "detail": "SENTRY_AUTH_TOKEN not set, gate skipped"}
+ return {"gate": "crash", "status": "skip", "detail": "SENTRY_READ_TOKEN not set, gate skipped"}
try:
cfg = cfg or json.loads(SENTRY_CONFIG.read_text())
if fetch is None:
@@ -260,7 +278,9 @@ def gate_crash(token: str | None, new: str, prev: str | None, since: datetime, n
n_new, r_new = fetch(new)
n_prev, r_prev = fetch(prev) if prev else (0, None)
except (urllib.error.URLError, OSError, ValueError, KeyError) as e:
- return {"gate": "crash", "status": "wait", "detail": f"Sentry unreachable or misconfigured ({e}); not advancing blind"}
+ return {"gate": "crash", "status": "wait",
+ "detail": f"cannot read crash stats from Sentry ({_why(e)}); keeping the stage, not advancing blind. "
+ "Check SENTRY_READ_TOKEN and SENTRY_API_HOST"}
if n_new < a.min_sessions or r_new is None:
return {"gate": "crash", "status": "wait",
"detail": f"{n_new} sessions of {new} reported to Sentry since the stage began (need {a.min_sessions})"}
@@ -298,7 +318,7 @@ def decide(state: dict, now: datetime, a, q=None, incident=None, hold: str = "",
d["previous"] = prev
d["gates"] = [gate_adoption(q, new, since, now, a), gate_space_create(q, new, prev, since, now, a),
gate_update(q, prev, since, now, a), incident or gate_incident(os.environ.get("INCIDENT_IO_API_KEY")),
- crash or gate_crash(os.environ.get("SENTRY_AUTH_TOKEN"), new, prev, since, now, a)]
+ crash or gate_crash(os.environ.get("SENTRY_READ_TOKEN"), new, prev, since, now, a)]
failed = [g for g in d["gates"] if g["status"] == "fail"]
waiting = [g for g in d["gates"] if g["status"] == "wait"]
if failed:
diff --git a/.github/scripts/tests/test_release_rollout_auto.py b/.github/scripts/tests/test_release_rollout_auto.py
index 6ce56d7895..d4c4ac079b 100644
--- a/.github/scripts/tests/test_release_rollout_auto.py
+++ b/.github/scripts/tests/test_release_rollout_auto.py
@@ -1,6 +1,7 @@
"""Run: python3 .github/scripts/tests/test_release_rollout_auto.py"""
import argparse
+import os
import sys
import tempfile
import unittest
@@ -79,7 +80,7 @@ def sessions(**by_version):
NOW = PUB + timedelta(hours=25)
-CFG = {"org": "trycua", "project": "cua-spaces-desktop"}
+CFG = {"org": "cua-ai", "project": "cua-spaces-desktop"}
def crash(new, prev, token="t", **kw):
@@ -114,6 +115,31 @@ class CrashGate(unittest.TestCase):
self.assertEqual(g["status"], "wait")
+class SentryHostAndFailures(unittest.TestCase):
+ def test_host_defaults_to_the_eu_region_and_is_configurable(self):
+ os.environ.pop("SENTRY_API_HOST", None)
+ self.assertEqual(ra.sentry_base(), "https://de.sentry.io/api/0")
+ os.environ["SENTRY_API_HOST"] = "https://example.invalid/"
+ try:
+ self.assertEqual(ra.sentry_base(), "https://example.invalid/api/0")
+ finally:
+ os.environ.pop("SENTRY_API_HOST")
+
+ def test_unreadable_sentry_waits_with_a_clear_message(self):
+ import urllib.error
+ denied = urllib.error.HTTPError("https://de.sentry.io/api/0/projects/cua-ai/x/", 403, "Forbidden", {}, None)
+
+ def deny(v):
+ raise denied
+ g = ra.gate_crash("t", "0.16.0", "0.15.0", PUB, NOW, args(), cfg=CFG, fetch=deny)
+ self.assertEqual(g["status"], "wait")
+ self.assertIn("HTTP 403 from https://de.sentry.io", g["detail"])
+ self.assertIn("not advancing", g["detail"])
+ d = Decide().go(crash=g)
+ self.assertEqual(d["action"], "wait")
+ self.assertNotEqual(d["action"], "advance")
+
+
class Decide(unittest.TestCase):
def go(self, hours=25, hold="", q=None, stage="10", crash=None):
s = ra.read_state(feeds("10", ra.rr.HOLD_SECONDS)) if stage == "10" else ra.read_state(feeds(stage, None))
diff --git a/.github/workflows/release-rollout-auto.yml b/.github/workflows/release-rollout-auto.yml
index af6e97ff0f..25eb2b7e19 100644
--- a/.github/workflows/release-rollout-auto.yml
+++ b/.github/workflows/release-rollout-auto.yml
@@ -46,7 +46,8 @@ jobs:
TAG: cua-spaces-latest
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
INCIDENT_IO_API_KEY: ${{ secrets.INCIDENT_IO_API_KEY }}
- SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+ SENTRY_READ_TOKEN: ${{ secrets.SENTRY_READ_TOKEN }}
+ SENTRY_API_HOST: ${{ vars.SENTRY_API_HOST }}
ALERTMANAGER_INTAKE_TOKEN: ${{ secrets.ALERTMANAGER_INTAKE_TOKEN }}
HOLD_VAR: ${{ vars.SPACES_ROLLOUT_HOLD }}
DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || 'false' }}