Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

ci(spaces): health-gated rollout soak, 2h at 10% and 4h at 50% for patch releases (#6591)

Francesco Bonacci1422680af2d4
diff --git a/.github/scripts/release_rollout_auto.py b/.github/scripts/release_rollout_auto.py
index adc01650ea..4d943d6275 100644
--- a/.github/scripts/release_rollout_auto.py
+++ b/.github/scripts/release_rollout_auto.py
@@ -18,6 +18,11 @@ Decision (the Play Store / Firebase / Sentry release-health pattern):
pause a gate is past its threshold: the workflow pauses and alerts
advance old enough, every gate has enough data and passes: 10 -> 50 -> 100
+Soak (how old "old enough" is), thresholds in config/spaces-rollout-soak.json:
+ patch release soak_hours_10 (2h) at 10%, soak_hours_50 (4h) at 50%, but only once the crash
+ gate has passed on min_sessions sessions; until then it waits, up to soak_hours_full
+ x.y.0 release soak_hours_full (24h) at every step, as before
+
Gates, PostHog project 56041 (EU), spaces_app events of the new version versus the
previous stable (the most active non-prerelease version in the week before the stage):
adoption at least --min-installs installs of the new version sent cua_app_active
@@ -66,9 +71,36 @@ import release_rollout as rr # noqa: E402
POSTHOG = "https://eu.posthog.com/api/projects/56041/query/"
SENTRY_DEFAULT_HOST = "https://de.sentry.io"
SENTRY_CONFIG = Path(__file__).resolve().parents[2] / "config" / "spaces-crash-reporting.json"
+SOAK_CONFIG = Path(__file__).resolve().parents[2] / "config" / "spaces-rollout-soak.json"
NEXT = {"10": "50", "50": "100"}
+def load_soak(path: Path = SOAK_CONFIG) -> dict:
+ """The named soak thresholds (config/spaces-rollout-soak.json); every one is also a flag."""
+ return json.loads(path.read_text())
+
+
+def is_minor_or_major(version: str | None) -> bool:
+ """x.y.0 (a minor or major release). Anything else, and anything unparsable, is a patch."""
+ m = re.match(r"^v?(\d+)\.(\d+)\.(\d+)", version or "")
+ return bool(m) and m.group(3) == "0"
+
+
+def soak_hours(version: str | None, stage: str, a, crash_ready: bool) -> tuple[float, str]:
+ """(hours the stage must have run before it may advance, why that figure).
+
+ x.y.0 releases keep the full soak at every step. A patch release uses the short soak of its
+ stage, but only once the crash gate has passed on enough sessions; until then it waits, up to
+ the full soak (the old fixed 24h)."""
+ full = a.soak_hours_full
+ if is_minor_or_major(version):
+ return full, f"x.y.0 release keeps the {full:g}h soak"
+ short = min(a.soak_hours_10 if stage == "10" else a.soak_hours_50, full)
+ if not crash_ready:
+ return full, f"patch soak of {short:g}h needs the crash gate to pass on enough sessions; until then up to {full:g}h"
+ return short, f"patch soak of {short:g}h at {stage}%"
+
+
def _num(text: str | None) -> int | None:
return int(text) if text and text.strip().isdigit() else None
@@ -320,6 +352,9 @@ def decide(state: dict, now: datetime, a, q=None, incident=None, hold: str = "",
gate_update(q, prev, since, now, a), incident or gate_incident(os.environ.get("INCIDENT_IO_API_KEY")),
crash or gate_crash(os.environ.get("SENTRY_READ_TOKEN"), new, prev, since, now, a)]
failed = [g for g in d["gates"] if g["status"] == "fail"]
+ crash_ready = any(g["gate"] == "crash" and g["status"] == "pass" for g in d["gates"])
+ need, soak_why = soak_hours(new, state["stage"], a, crash_ready)
+ d["soak_hours"] = need
waiting = [g for g in d["gates"] if g["status"] == "wait"]
if failed:
d["action"] = "pause"
@@ -327,16 +362,16 @@ def decide(state: dict, now: datetime, a, q=None, incident=None, hold: str = "",
elif hold:
d["action"] = "hold"
d["why"] = "hold is set (" + hold + "): not advancing"
- elif age < a.min_age_hours:
+ elif age < need:
d["action"] = "wait"
- d["why"] = f"stage is {age:.1f}h old, needs {a.min_age_hours}h"
+ d["why"] = f"stage is {age:.1f}h old, needs {need:g}h ({soak_why})"
elif waiting:
d["action"] = "wait"
d["why"] = "not enough data: " + "; ".join(g["detail"] for g in waiting)
else:
d["action"] = "advance"
d["next"] = NEXT[state["stage"]]
- d["why"] = f"{age:.1f}h at {state['stage']}%, every gate passes"
+ d["why"] = f"{age:.1f}h at {state['stage']}%, every gate passes ({soak_why})"
return d
@@ -345,15 +380,20 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument("--feed-dir", required=True, type=Path)
ap.add_argument("--now", help="ISO time (tests)")
ap.add_argument("--hold", default=os.environ.get("ROLLOUT_HOLD", ""), help="non-empty: do not advance")
- ap.add_argument("--min-age-hours", type=float, default=24)
- ap.add_argument("--min-installs", type=int, default=15)
- ap.add_argument("--min-create-events", type=int, default=30)
- ap.add_argument("--max-create-error-delta", type=float, default=10.0)
- ap.add_argument("--min-update-events", type=int, default=20)
- ap.add_argument("--max-update-failure-delta", type=float, default=15.0)
- ap.add_argument("--min-sessions", type=int, default=100)
- ap.add_argument("--max-crash-free-drop", type=float, default=0.5)
- ap.add_argument("--min-crash-free", type=float, default=99.0)
+ ap.add_argument("--soak-config", type=Path, default=SOAK_CONFIG, help="JSON with the soak and gate thresholds")
+ cfg = load_soak(ap.parse_known_args(argv)[0].soak_config)
+ # Every threshold is a named value in the config file; a flag overrides it for one run.
+ ap.add_argument("--soak-hours-10", type=float, default=cfg["soak_hours_10"])
+ ap.add_argument("--soak-hours-50", type=float, default=cfg["soak_hours_50"])
+ ap.add_argument("--soak-hours-full", type=float, default=cfg["soak_hours_full"])
+ ap.add_argument("--min-installs", type=int, default=cfg["min_installs"])
+ ap.add_argument("--min-create-events", type=int, default=cfg["min_create_events"])
+ ap.add_argument("--max-create-error-delta", type=float, default=cfg["max_create_error_delta"])
+ ap.add_argument("--min-update-events", type=int, default=cfg["min_update_events"])
+ ap.add_argument("--max-update-failure-delta", type=float, default=cfg["max_update_failure_delta"])
+ ap.add_argument("--min-sessions", type=int, default=cfg["min_sessions"])
+ ap.add_argument("--max-crash-free-drop", type=float, default=cfg["max_crash_free_drop"])
+ ap.add_argument("--min-crash-free", type=float, default=cfg["min_crash_free"])
a = ap.parse_args(argv)
now = datetime.fromisoformat(a.now) if a.now else datetime.now(timezone.utc)
key = os.environ.get("POSTHOG_API_KEY", "")
diff --git a/.github/scripts/tests/test_release_rollout_auto.py b/.github/scripts/tests/test_release_rollout_auto.py
index d4c4ac079b..3fe9af2211 100644
--- a/.github/scripts/tests/test_release_rollout_auto.py
+++ b/.github/scripts/tests/test_release_rollout_auto.py
@@ -29,7 +29,7 @@ def feeds(staging, interval, pub=PUB, title="0.16.0"):
def args(**kw):
- a = argparse.Namespace(min_age_hours=24, min_installs=15, min_create_events=30, max_create_error_delta=10.0,
+ a = argparse.Namespace(soak_hours_10=2, soak_hours_50=4, soak_hours_full=24, min_installs=15, min_create_events=30, max_create_error_delta=10.0,
min_update_events=20, max_update_failure_delta=15.0,
min_sessions=100, max_crash_free_drop=0.5, min_crash_free=99.0)
a.__dict__.update(kw)
@@ -141,10 +141,49 @@ class SentryHostAndFailures(unittest.TestCase):
class Decide(unittest.TestCase):
- def go(self, hours=25, hold="", q=None, stage="10", crash=None):
- s = ra.read_state(feeds("10", ra.rr.HOLD_SECONDS)) if stage == "10" else ra.read_state(feeds(stage, None))
+ def go(self, hours=25, hold="", q=None, stage="10", crash=None, title="0.16.0"):
+ s = ra.read_state(feeds("10", ra.rr.HOLD_SECONDS, title=title)) if stage == "10" else ra.read_state(feeds(stage, None, title=title))
return ra.decide(s, PUB + timedelta(hours=hours), args(), q=q or fake(), incident=OK(), hold=hold, crash=crash or CR())
+ def test_patch_advances_after_the_short_soak_at_10(self):
+ self.assertEqual(self.go(hours=1.5, title="0.16.1")["action"], "wait")
+ d = self.go(hours=2.5, title="0.16.1")
+ self.assertEqual((d["action"], d["next"], d["soak_hours"]), ("advance", "50", 2))
+
+ def test_patch_advances_after_the_short_soak_at_50(self):
+ s = ra.read_state(feeds("50", ra.rr.WEEK, title="0.16.1"))
+ three_weeks = timedelta(weeks=3)
+ self.assertEqual(ra.decide(s, PUB + three_weeks + timedelta(hours=3), args(), q=fake(), incident=OK(), crash=CR())["action"], "wait")
+ d = ra.decide(s, PUB + three_weeks + timedelta(hours=4.5), args(), q=fake(), incident=OK(), crash=CR())
+ self.assertEqual((d["action"], d["next"], d["soak_hours"]), ("advance", "100", 4))
+
+ def test_minor_and_major_keep_24h_at_every_step(self):
+ for title in ("0.16.0", "1.0.0"):
+ self.assertEqual(self.go(hours=5, title=title)["action"], "wait")
+ self.assertEqual(self.go(hours=25, title=title)["soak_hours"], 24)
+ self.assertEqual(self.go(hours=25, title=title)["action"], "advance")
+
+ def test_short_soak_needs_enough_crash_sessions(self):
+ few = {"gate": "crash", "status": "wait", "detail": "30 sessions (need 100)"}
+ d = self.go(hours=5, title="0.16.1", crash=few)
+ self.assertEqual(d["action"], "wait")
+ skipped = {"gate": "crash", "status": "skip", "detail": "no token"}
+ d = self.go(hours=5, title="0.16.1", crash=skipped)
+ self.assertEqual((d["action"], d["soak_hours"]), ("wait", 24))
+ self.assertEqual(self.go(hours=25, title="0.16.1", crash=skipped)["action"], "advance")
+
+ def test_patch_still_waits_for_data_after_the_short_soak(self):
+ self.assertEqual(self.go(hours=5, title="0.16.1", q=fake(installs=3))["action"], "wait")
+
+ def test_is_minor_or_major(self):
+ for v, want in (("0.16.0", True), ("1.0.0", True), ("0.16.1", False), ("2.3.10", False), ("0.16.0-beta.1", True), (None, False), ("junk", False)):
+ self.assertEqual(ra.is_minor_or_major(v), want, v)
+
+ def test_config_names_every_threshold(self):
+ cfg = ra.load_soak()
+ self.assertEqual((cfg["soak_hours_10"], cfg["soak_hours_50"], cfg["soak_hours_full"]), (2, 4, 24))
+ self.assertTrue(set(args().__dict__) <= set(cfg))
+
def test_advance(self):
d = self.go()
self.assertEqual((d["action"], d["next"]), ("advance", "50"))
diff --git a/.github/workflows/release-rollout-auto.yml b/.github/workflows/release-rollout-auto.yml
index 25eb2b7e19..54d5a85efd 100644
--- a/.github/workflows/release-rollout-auto.yml
+++ b/.github/workflows/release-rollout-auto.yml
@@ -2,10 +2,11 @@ name: "Release: auto rollout (Cua Spaces)"
# Advances the Cua Spaces staged rollout by itself (health-gated, with auto-pause),
# the pattern of Play Store, Firebase and Sentry release health. Hourly it reads the
-# stage from the live feeds, and when a stage has run 24h and every health gate
-# passes it dispatches release-rollout.yml for the next one (10 -> 50 -> 100). If a
+# stage from the live feeds, and when the stage has soaked long enough (patch release:
+# about 2h at 10% and 4h at 50% once the crash gate has enough sessions; x.y.0 release:
+# 24h) and every health gate passes it dispatches release-rollout.yml for the next one (10 -> 50 -> 100). If a
# gate fails it dispatches `pause` and posts an alert. Gates and thresholds:
-# .github/scripts/release_rollout_auto.py. Manual dispatch of release-rollout.yml
+# .github/scripts/release_rollout_auto.py and config/spaces-rollout-soak.json. Manual dispatch of release-rollout.yml
# keeps working and is what this workflow calls; a person can still pause or advance.
#
# Hold: set the repository variable SPACES_ROLLOUT_HOLD to any text (the reason), or
diff --git a/apps/cua-spaces-desktop/docs/staged-rollout.md b/apps/cua-spaces-desktop/docs/staged-rollout.md
index 4061cbc11e..d3b68c28b4 100644
--- a/apps/cua-spaces-desktop/docs/staged-rollout.md
+++ b/apps/cua-spaces-desktop/docs/staged-rollout.md
@@ -31,12 +31,47 @@ job summary as usual. Manual dispatch is unchanged.
| Decision | When |
|---|---|
-| advance (10 to 50, 50 to 100) | stage at least 24h old, every gate has enough data and passes |
-| wait | younger than 24h, or a gate has too little data (no data is not healthy) |
+| advance (10 to 50, 50 to 100) | the stage has soaked long enough (see Soak below), every gate has enough data and passes |
+| wait | younger than the soak, or a gate has too little data (no data is not healthy) |
| pause | a gate is past its threshold, at any stage age: dispatches `pause` and posts an Alertmanager alert (`SpacesRolloutAutoPaused`, source `github-actions`, to #alerts-staging; it does not page) |
| hold | the variable `SPACES_ROLLOUT_HOLD` is set (the text is the reason) or an open issue has the label `spaces-rollout-hold`: never advances; a failing gate still pauses |
| none | 100%, paused (resuming is a person's call), or the feeds disagree |
+### Soak
+
+How long a stage must have run before it may advance depends on the release:
+
+| Release | At 10% | At 50% |
+|---|---|---|
+| Patch (x.y.z, z not 0) | 2h | 4h |
+| x.y.0 (minor or major) | 24h | 24h |
+
+The short soak only applies once the crash gate has passed on enough sessions
+(`min_sessions`, 100, on the new version). Until then a patch release waits for
+more sessions, up to the full 24h. After that it advances as before as soon as
+every gate has data and passes; a gate with too little data still waits. A
+failing gate pauses at any age, and the hold still stops advancing. A Spaces
+release reaches 100% in about 8h instead of 48h.
+
+Every threshold is a named value in `config/spaces-rollout-soak.json`, and each is also a flag
+of the script that overrides it for one run:
+
+| Config value | Flag | Default | Meaning |
+|---|---|---|---|
+| `soak_hours_10` | `--soak-hours-10` | 2 | patch release: hours at 10% before advancing |
+| `soak_hours_50` | `--soak-hours-50` | 4 | patch release: hours at 50% before advancing |
+| `soak_hours_full` | `--soak-hours-full` | 24 | x.y.0 releases at every step, and the cap a patch waits for enough sessions |
+| `min_sessions` | `--min-sessions` | 100 | crash gate: sessions of the new version needed |
+| `min_installs` | `--min-installs` | 15 | adoption gate |
+| `min_create_events` | `--min-create-events` | 30 | space create gate: events needed on each side |
+| `max_create_error_delta` | `--max-create-error-delta` | 10.0 | space create gate: points above the previous version |
+| `min_update_events` | `--min-update-events` | 20 | update gate: attempts needed |
+| `max_update_failure_delta` | `--max-update-failure-delta` | 15.0 | update gate: points above the week before |
+| `max_crash_free_drop` | `--max-crash-free-drop` | 0.5 | crash gate: points below the previous version |
+| `min_crash_free` | `--min-crash-free` | 99.0 | crash gate: floor when there is no baseline |
+
+To change a threshold, edit the config file in a PR; to try one, run the script locally with the flag.
+
Gates (PostHog project 56041, `spaces_app` events of the new version against the
previous stable, the most active non-prerelease version the week before the stage):
@@ -46,7 +81,7 @@ previous stable, the most active non-prerelease version the week before the stag
- incident: no active incident.io incident with "spaces" in its name (needs the `INCIDENT_IO_API_KEY` secret; skipped and said so in the summary until it is set).
- crash: Sentry release health (org and project in `config/spaces-crash-reporting.json`, release `cua-spaces-desktop@<version>`, environment `stable`). Crash-free sessions of the new version since the stage began must be at most 0.5 points below the previous version's over the same window, with at least 100 sessions on the new version (fewer: wait). A previous version with fewer than 100 sessions (it shipped before crash reporting) is no baseline, so the new version must reach 99% crash-free. Needs the `SENTRY_AUTH_TOKEN` secret (`org:read`, `project:read`); skipped and said so in the summary until it is set. Crash reports and sessions only come from people who left "Share usage data" on, so the numbers are a sample.
-Thresholds are flags of that script. Secrets: `POSTHOG_API_KEY` (query read),
+Thresholds are in the config file and flags of that script. Secrets: `POSTHOG_API_KEY` (query read),
optional `INCIDENT_IO_API_KEY`, `SENTRY_AUTH_TOKEN`, and the existing
`ALERTMANAGER_INTAKE_TOKEN`.
diff --git a/config/spaces-rollout-soak.json b/config/spaces-rollout-soak.json
new file mode 100644
index 0000000000..00c61fdadd
--- /dev/null
+++ b/config/spaces-rollout-soak.json
@@ -0,0 +1,13 @@
+{
+ "soak_hours_10": 2,
+ "soak_hours_50": 4,
+ "soak_hours_full": 24,
+ "min_installs": 15,
+ "min_create_events": 30,
+ "max_create_error_delta": 10.0,
+ "min_update_events": 20,
+ "max_update_failure_delta": 15.0,
+ "min_sessions": 100,
+ "max_crash_free_drop": 0.5,
+ "min_crash_free": 99.0
+}