Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): measure stateless send-before-ready baseline
Alfredo Lopez52a0049999b6
diff --git a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.tsindex b2bd8a8b7..823336fed 100644--- a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts+++ b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts@@ -68,6 +68,11 @@ type JourneyMetrics = {draft_preserved_across_restart: booleansecond_suspend_ms: numbercold_send_to_waking_visible_ms: number | null+ cold_send_pre_click_ready_replicas: number+ cold_send_first_outcome: 'response' | 'waking' | null+ cold_send_to_user_message_visible_ms: number | null+ cold_send_to_task_acknowledged_ms: number | null+ cold_send_to_ready_observed_ms: number | nullcold_send_to_response_ms: numbercold_send_to_composer_idle_ms: numbercold_send_retries: number@@ -274,6 +279,7 @@ async function resolveColdSend(): Promise<{wakingVisibleMs: number | nullretries: number+ firstOutcome: 'response' | 'waking'}> {const started = Date.now()const response = page.getByTestId('agent-response').filter({ hasText: responseMarker })@@ -291,6 +297,9 @@ async function resolveColdSend(.not.toBeNull()let wakingVisibleMs: number | null = nulllet retries = 0+ if (firstOutcome === null) {+ throw new Error('Cold send ended without an observable first outcome')+ }if (firstOutcome === 'waking') {wakingVisibleMs = Date.now() - started// The retry is a user action. Wait for the observed Host to become Ready@@ -306,6 +315,7 @@ async function resolveColdSend(return {wakingVisibleMs,retries,+ firstOutcome,}}@@ -384,6 +394,11 @@ test('human journey — two Hosts, restart, cache, and two verified stateless wadraft_preserved_across_restart: false,second_suspend_ms: 0,cold_send_to_waking_visible_ms: null,+ cold_send_pre_click_ready_replicas: 0,+ cold_send_first_outcome: null,+ cold_send_to_user_message_visible_ms: null,+ cold_send_to_task_acknowledged_ms: null,+ cold_send_to_ready_observed_ms: null,cold_send_to_response_ms: 0,cold_send_to_composer_idle_ms: 0,cold_send_retries: 0,@@ -564,10 +579,55 @@ test('human journey — two Hosts, restart, cache, and two verified stateless waconst followPrompt = `Earlier I gave you a token. Reply with exactly that token followed by ${statelessFollowMarker}.`await humanType(composer, followPrompt)await expect(page.getByTestId('send-button')).toBeEnabled()+ // The catalog prewarm after reopen is measured separately by+ // second_launch_to_stateless_ready_ms. This fresh snapshot proves that+ // the second send is itself a cold wake intent, not a continuation of+ // that earlier prewarm window.+ const preClick = await readHostSnapshot(STATELESS_DEPLOYMENT, STATELESS_HOST_RESOURCE)+ metrics.cold_send_pre_click_ready_replicas = preClick.readyReplicas+ expect(preClick.lifecycleState).toBe('suspended')+ expect(preClick.replicas).toBe(0)+ expect(preClick.readyReplicas).toBe(0)+ expect(preClick.podNames).toHaveLength(0)+const sendStarted = Date.now()await page.getByTestId('send-button').click()+ await expect(page.getByTestId('message-list')).toContainText(statelessFollowMarker, {+ timeout: 30_000,+ })+ metrics.cold_send_to_user_message_visible_ms = Date.now() - sendStarted++ const followResponse = page+ .getByTestId('agent-response')+ .filter({ hasText: statelessFollowMarker })+ const taskStepper = page.getByTestId('progress-stepper')+ const stepperWatch = expect+ .poll(+ async () => {+ if (+ metrics.cold_send_to_task_acknowledged_ms === null &&+ (await taskStepper.isVisible().catch(() => false))+ ) {+ metrics.cold_send_to_task_acknowledged_ms = Date.now() - sendStarted+ }+ return (+ metrics.cold_send_to_task_acknowledged_ms !== null ||+ (await followResponse.isVisible().catch(() => false))+ )+ },+ {+ timeout: 270_000,+ message: 'Cold send must expose either a task acknowledgement or a final response',+ }+ )+ .toBe(true)+ const readyObservation = waitForStatelessReady(270_000).then(result => ({+ result,+ observedAt: Date.now(),+ }))const outcome = await resolveColdSend(page, statelessFollowMarker, statelessMarker)metrics.cold_send_to_waking_visible_ms = outcome.wakingVisibleMs+ metrics.cold_send_first_outcome = outcome.firstOutcomemetrics.cold_send_to_response_ms = Date.now() - sendStartedmetrics.cold_send_retries = outcome.retriesawait expect(composer).toHaveValue('', { timeout: 150_000 })@@ -576,12 +636,14 @@ test('human journey — two Hosts, restart, cache, and two verified stateless wa})await expect(page.getByTestId('send-button')).toBeDisabled({ timeout: 150_000 })metrics.cold_send_to_composer_idle_ms = Date.now() - sendStarted-- const ready = await waitForStatelessReady(270_000)- metrics.stateless_pod_after_cold_send = ready.snapshot.podNames- metrics.stateless_image_ids_after_cold_send = ready.snapshot.imageIds- expect(ready.snapshot.readyReplicas).toBeGreaterThan(0)- expect(ready.snapshot.imageIds).toEqual(metrics.stateless_image_ids_before)+ await stepperWatch++ const readyAfterSend = await readyObservation+ metrics.cold_send_to_ready_observed_ms = readyAfterSend.observedAt - sendStarted+ metrics.stateless_pod_after_cold_send = readyAfterSend.result.snapshot.podNames+ metrics.stateless_image_ids_after_cold_send = readyAfterSend.result.snapshot.imageIds+ expect(readyAfterSend.result.snapshot.readyReplicas).toBeGreaterThan(0)+ expect(readyAfterSend.result.snapshot.imageIds).toEqual(metrics.stateless_image_ids_before)await expect(page.getByTestId('agent-response').filter({ hasText: statelessFollowMarker })).toContainText(statelessFollowMarker, { timeout: 30_000 })diff --git a/scripts/e2e/e2e-stateless-wake-recovery.sh b/scripts/e2e/e2e-stateless-wake-recovery.shindex 924e04193..7df60f6bc 100755--- a/scripts/e2e/e2e-stateless-wake-recovery.sh+++ b/scripts/e2e/e2e-stateless-wake-recovery.sh@@ -81,6 +81,11 @@# measured_total_ms, infra_overhead_ms}]}}# R4 additionally emits prewarm_to_ready_ms:[ms] -- the per-cycle overlap# series (informational, no budget).+# R2 additionally emits cold_resume_baseline:[{cycle, first_post_status,+# admission_attempts, first_200_ms, status_flipped_to_replicas_patched_ms}]+# -- characterization-only admission/scale data for later before/after+# comparison. It is informational, has no budget, and never replaces the+# existing ms gate metric.# Each measured turn (baseline, R1, R2, R4) is attributed with the serving# pod's [TurnTiming] phase line (queue_wait/session_load/prompt_assembly/# llm_wall/llm_calls/tool_loop/tools_called/input_chars_approx) and@@ -337,9 +342,13 @@ send_turn_raw() {# hard failure at the caller — an unreachable assertion never passes.RECOVERY_MS=""TURN_T0_RFC3339=""+ADMISSION_FIRST_STATUS=""+ADMISSION_ATTEMPTS=""measure_recovery_turn() {local content=$1 t0 t1 deadline attempt=0RECOVERY_MS=""+ ADMISSION_FIRST_STATUS=""+ ADMISSION_ATTEMPTS=""mint_rpc_token || { echo "could not mint RPC token before t0" >&2; return 1; }TURN_T0_RFC3339="$(now_rfc3339)"t0="$(now_ms)"@@ -347,9 +356,11 @@ measure_recovery_turn() {while [ "$SECONDS" -lt "$deadline" ]; doattempt=$((attempt + 1))send_turn_raw "$content" || return 1+ if [ "$attempt" -eq 1 ]; then ADMISSION_FIRST_STATUS="$TURN_STATUS"; fiif [ "$TURN_STATUS" = "200" ]; thent1="$(now_ms)"RECOVERY_MS=$((t1 - t0))+ ADMISSION_ATTEMPTS="$attempt"return 0fiif [ "$TURN_STATUS" = "503" ] && echo "$TURN_BODY" | grep -qE 'host_waking|host_draining'; then@@ -557,11 +568,11 @@ drain_window_signal() {}record_cycle() {- local scenario=$1 cycle=$2 ms=$3 resolution=$4 wake_lines=$5 timing_json=${6:-null} prewarm_ms=${7:-null}+ local scenario=$1 cycle=$2 ms=$3 resolution=$4 wake_lines=$5 timing_json=${6:-null} prewarm_ms=${7:-null} baseline_json=${8:-null}jq -cn --arg s "$scenario" --argjson c "$cycle" --argjson ms "$ms" \--arg r "$resolution" --arg wl "$wake_lines" --argjson tt "$timing_json" \- --argjson pw "$prewarm_ms" \- '{scenario:$s, cycle:$c, ms:$ms, resolution:$r, wake_phases:($wl | split("\n") | map(select(length>0))), turn_timing:$tt, prewarm_to_ready_ms:$pw}' \+ --argjson pw "$prewarm_ms" --argjson ab "$baseline_json" \+ '{scenario:$s, cycle:$c, ms:$ms, resolution:$r, wake_phases:($wl | split("\n") | map(select(length>0))), turn_timing:$tt, prewarm_to_ready_ms:$pw, admission_baseline:$ab}' \>> "$CYCLES_FILE"}@@ -873,6 +884,16 @@ while [ "$cycle" -le "$RECOVERY_CYCLES" ]; dofail "R2 cycle ${cycle}: turn to suspended host did not reach a definitive 200"pod_diagnostics; print_results; exit 1fi+ # Characterization baseline for a later, separately approved proxy+ # admission change: first POST HTTP status, POST count through the first+ # 200, and the first-200 send->200 latency. first_200_ms is deliberately+ # the same first-200 measurement as RECOVERY_MS (LLM-empty re-sends never+ # touch it); it is repeated here so the baseline record is self-contained.+ r2_baseline_json="$(jq -cn \+ --arg first_status "$ADMISSION_FIRST_STATUS" \+ --argjson attempts "$ADMISSION_ATTEMPTS" \+ --argjson first_200_ms "$RECOVERY_MS" \+ '{first_post_status:$first_status, admission_attempts:$attempts, first_200_ms:$first_200_ms}')"assert_success_response "R2 cycle ${cycle} turn" "Reply with exactly: ${r2_marker}" || { print_results; exit 1; }if wait_for_state "active" "$ACTIVE_WAIT"; thenok "R2 cycle ${cycle}: state=active after wake"@@ -910,7 +931,7 @@ while [ "$cycle" -le "$RECOVERY_CYCLES" ]; dofail "R2 cycle ${cycle}: no [StatelessWake] phase=wake_observed line since ${r2_since} -- the wake path is unproven for this cycle"pod_diagnostics; print_results; exit 1fi- record_cycle "R2_cold_resume_from_suspended" "$cycle" "$RECOVERY_MS" "cold" "$r2_wake_lines" "$r2_attr"+ record_cycle "R2_cold_resume_from_suspended" "$cycle" "$RECOVERY_MS" "cold" "$r2_wake_lines" "$r2_attr" null "$r2_baseline_json"cycle=$((cycle + 1))done@@ -1203,12 +1224,56 @@ for name, budget in budgets.items():continueattribution.append({"cycle": r["cycle"], **tt})breakdown = []+ baseline = []for r in recs:parsed = []for line in r.get("wake_phases", []):m = wake_re.search(line)if m:parsed.append({"generation": m.group(1), "phase": m.group(2), "ts_ms": int(m.group(3))})+ if name == "R2_cold_resume_from_suspended":+ raw = r.get("admission_baseline") or {}+ for key in ("first_post_status", "admission_attempts", "first_200_ms"):+ if key not in raw:+ print(+ f"scenario {name} cycle {r['cycle']}: admission baseline field '{key}' is missing -- refusing to emit partial baseline data",+ file=sys.stderr,+ )+ raise SystemExit(1)+ # Pair the two HCC phases by wake generation. A missing phase or+ # multiple candidate generations stays null with a reason: the+ # baseline must characterize the observed signal, never invent it.+ generations = {}+ for phase in parsed:+ generations.setdefault(phase["generation"], {})[phase["phase"]] = phase["ts_ms"]+ paired = [+ phases["replicas_patched"] - phases["status_flipped"]+ for phases in generations.values()+ if "status_flipped" in phases and "replicas_patched" in phases+ ]+ baseline_entry = {+ "cycle": r["cycle"],+ "first_post_status": raw["first_post_status"],+ "admission_attempts": raw["admission_attempts"],+ "first_200_ms": raw["first_200_ms"],+ "status_flipped_to_replicas_patched_ms": None,+ }+ if len(paired) == 1:+ if paired[0] >= 0:+ baseline_entry["status_flipped_to_replicas_patched_ms"] = paired[0]+ else:+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (+ f"negative timestamp delta ({paired[0]}ms) in [StatelessWake] lines"+ )+ elif len(paired) == 0:+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (+ "no [StatelessWake] generation in this cycle has both phase=status_flipped and phase=replicas_patched"+ )+ else:+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (+ f"{len(paired)} [StatelessWake] generations have both phases -- attribution is ambiguous"+ )+ baseline.append(baseline_entry)if parsed:breakdown.append({"cycle": r["cycle"], "wake_phases": parsed})# R4 overlap series (prewarm POST -> pod Ready). Informational, no@@ -1257,6 +1322,10 @@ for name, budget in budgets.items():entry["turn_attribution"] = attributionif prewarm:entry["prewarm_to_ready_ms"] = prewarm+ if baseline:+ # Informational characterization only: no budget and no effect on+ # the existing R2 recovery verdict.+ entry["cold_resume_baseline"] = baselinescenarios[name] = entryartifact = {