Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): measure stateless send-before-ready baseline

Alfredo Lopez52a0049999b6
diff --git a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
index b2bd8a8b7..823336fed 100644
--- a/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
+++ b/desktop-app/test/e2e-playwright/human-stateless-continuity.spec.ts
@@ -68,6 +68,11 @@ type JourneyMetrics = {
draft_preserved_across_restart: boolean
second_suspend_ms: number
cold_send_to_waking_visible_ms: number | null
+ cold_send_pre_click_ready_replicas: number
+ cold_send_first_outcome: 'response' | 'waking' | null
+ cold_send_to_user_message_visible_ms: number | null
+ cold_send_to_task_acknowledged_ms: number | null
+ cold_send_to_ready_observed_ms: number | null
cold_send_to_response_ms: number
cold_send_to_composer_idle_ms: number
cold_send_retries: number
@@ -274,6 +279,7 @@ async function resolveColdSend(
): Promise<{
wakingVisibleMs: number | null
retries: number
+ firstOutcome: 'response' | 'waking'
}> {
const started = Date.now()
const response = page.getByTestId('agent-response').filter({ hasText: responseMarker })
@@ -291,6 +297,9 @@ async function resolveColdSend(
.not.toBeNull()
let wakingVisibleMs: number | null = null
let retries = 0
+ if (firstOutcome === null) {
+ throw new Error('Cold send ended without an observable first outcome')
+ }
if (firstOutcome === 'waking') {
wakingVisibleMs = Date.now() - started
// The retry is a user action. Wait for the observed Host to become Ready
@@ -306,6 +315,7 @@ async function resolveColdSend(
return {
wakingVisibleMs,
retries,
+ firstOutcome,
}
}
@@ -384,6 +394,11 @@ test('human journey — two Hosts, restart, cache, and two verified stateless wa
draft_preserved_across_restart: false,
second_suspend_ms: 0,
cold_send_to_waking_visible_ms: null,
+ cold_send_pre_click_ready_replicas: 0,
+ cold_send_first_outcome: null,
+ cold_send_to_user_message_visible_ms: null,
+ cold_send_to_task_acknowledged_ms: null,
+ cold_send_to_ready_observed_ms: null,
cold_send_to_response_ms: 0,
cold_send_to_composer_idle_ms: 0,
cold_send_retries: 0,
@@ -564,10 +579,55 @@ test('human journey — two Hosts, restart, cache, and two verified stateless wa
const followPrompt = `Earlier I gave you a token. Reply with exactly that token followed by ${statelessFollowMarker}.`
await humanType(composer, followPrompt)
await expect(page.getByTestId('send-button')).toBeEnabled()
+ // The catalog prewarm after reopen is measured separately by
+ // second_launch_to_stateless_ready_ms. This fresh snapshot proves that
+ // the second send is itself a cold wake intent, not a continuation of
+ // that earlier prewarm window.
+ const preClick = await readHostSnapshot(STATELESS_DEPLOYMENT, STATELESS_HOST_RESOURCE)
+ metrics.cold_send_pre_click_ready_replicas = preClick.readyReplicas
+ expect(preClick.lifecycleState).toBe('suspended')
+ expect(preClick.replicas).toBe(0)
+ expect(preClick.readyReplicas).toBe(0)
+ expect(preClick.podNames).toHaveLength(0)
+
const sendStarted = Date.now()
await page.getByTestId('send-button').click()
+ await expect(page.getByTestId('message-list')).toContainText(statelessFollowMarker, {
+ timeout: 30_000,
+ })
+ metrics.cold_send_to_user_message_visible_ms = Date.now() - sendStarted
+
+ const followResponse = page
+ .getByTestId('agent-response')
+ .filter({ hasText: statelessFollowMarker })
+ const taskStepper = page.getByTestId('progress-stepper')
+ const stepperWatch = expect
+ .poll(
+ async () => {
+ if (
+ metrics.cold_send_to_task_acknowledged_ms === null &&
+ (await taskStepper.isVisible().catch(() => false))
+ ) {
+ metrics.cold_send_to_task_acknowledged_ms = Date.now() - sendStarted
+ }
+ return (
+ metrics.cold_send_to_task_acknowledged_ms !== null ||
+ (await followResponse.isVisible().catch(() => false))
+ )
+ },
+ {
+ timeout: 270_000,
+ message: 'Cold send must expose either a task acknowledgement or a final response',
+ }
+ )
+ .toBe(true)
+ const readyObservation = waitForStatelessReady(270_000).then(result => ({
+ result,
+ observedAt: Date.now(),
+ }))
const outcome = await resolveColdSend(page, statelessFollowMarker, statelessMarker)
metrics.cold_send_to_waking_visible_ms = outcome.wakingVisibleMs
+ metrics.cold_send_first_outcome = outcome.firstOutcome
metrics.cold_send_to_response_ms = Date.now() - sendStarted
metrics.cold_send_retries = outcome.retries
await expect(composer).toHaveValue('', { timeout: 150_000 })
@@ -576,12 +636,14 @@ test('human journey — two Hosts, restart, cache, and two verified stateless wa
})
await expect(page.getByTestId('send-button')).toBeDisabled({ timeout: 150_000 })
metrics.cold_send_to_composer_idle_ms = Date.now() - sendStarted
-
- const ready = await waitForStatelessReady(270_000)
- metrics.stateless_pod_after_cold_send = ready.snapshot.podNames
- metrics.stateless_image_ids_after_cold_send = ready.snapshot.imageIds
- expect(ready.snapshot.readyReplicas).toBeGreaterThan(0)
- expect(ready.snapshot.imageIds).toEqual(metrics.stateless_image_ids_before)
+ await stepperWatch
+
+ const readyAfterSend = await readyObservation
+ metrics.cold_send_to_ready_observed_ms = readyAfterSend.observedAt - sendStarted
+ metrics.stateless_pod_after_cold_send = readyAfterSend.result.snapshot.podNames
+ metrics.stateless_image_ids_after_cold_send = readyAfterSend.result.snapshot.imageIds
+ expect(readyAfterSend.result.snapshot.readyReplicas).toBeGreaterThan(0)
+ expect(readyAfterSend.result.snapshot.imageIds).toEqual(metrics.stateless_image_ids_before)
await expect(
page.getByTestId('agent-response').filter({ hasText: statelessFollowMarker })
).toContainText(statelessFollowMarker, { timeout: 30_000 })
diff --git a/scripts/e2e/e2e-stateless-wake-recovery.sh b/scripts/e2e/e2e-stateless-wake-recovery.sh
index 924e04193..7df60f6bc 100755
--- a/scripts/e2e/e2e-stateless-wake-recovery.sh
+++ b/scripts/e2e/e2e-stateless-wake-recovery.sh
@@ -81,6 +81,11 @@
# measured_total_ms, infra_overhead_ms}]}}
# R4 additionally emits prewarm_to_ready_ms:[ms] -- the per-cycle overlap
# series (informational, no budget).
+# R2 additionally emits cold_resume_baseline:[{cycle, first_post_status,
+# admission_attempts, first_200_ms, status_flipped_to_replicas_patched_ms}]
+# -- characterization-only admission/scale data for later before/after
+# comparison. It is informational, has no budget, and never replaces the
+# existing ms gate metric.
# Each measured turn (baseline, R1, R2, R4) is attributed with the serving
# pod's [TurnTiming] phase line (queue_wait/session_load/prompt_assembly/
# llm_wall/llm_calls/tool_loop/tools_called/input_chars_approx) and
@@ -337,9 +342,13 @@ send_turn_raw() {
# hard failure at the caller — an unreachable assertion never passes.
RECOVERY_MS=""
TURN_T0_RFC3339=""
+ADMISSION_FIRST_STATUS=""
+ADMISSION_ATTEMPTS=""
measure_recovery_turn() {
local content=$1 t0 t1 deadline attempt=0
RECOVERY_MS=""
+ ADMISSION_FIRST_STATUS=""
+ ADMISSION_ATTEMPTS=""
mint_rpc_token || { echo "could not mint RPC token before t0" >&2; return 1; }
TURN_T0_RFC3339="$(now_rfc3339)"
t0="$(now_ms)"
@@ -347,9 +356,11 @@ measure_recovery_turn() {
while [ "$SECONDS" -lt "$deadline" ]; do
attempt=$((attempt + 1))
send_turn_raw "$content" || return 1
+ if [ "$attempt" -eq 1 ]; then ADMISSION_FIRST_STATUS="$TURN_STATUS"; fi
if [ "$TURN_STATUS" = "200" ]; then
t1="$(now_ms)"
RECOVERY_MS=$((t1 - t0))
+ ADMISSION_ATTEMPTS="$attempt"
return 0
fi
if [ "$TURN_STATUS" = "503" ] && echo "$TURN_BODY" | grep -qE 'host_waking|host_draining'; then
@@ -557,11 +568,11 @@ drain_window_signal() {
}
record_cycle() {
- local scenario=$1 cycle=$2 ms=$3 resolution=$4 wake_lines=$5 timing_json=${6:-null} prewarm_ms=${7:-null}
+ local scenario=$1 cycle=$2 ms=$3 resolution=$4 wake_lines=$5 timing_json=${6:-null} prewarm_ms=${7:-null} baseline_json=${8:-null}
jq -cn --arg s "$scenario" --argjson c "$cycle" --argjson ms "$ms" \
--arg r "$resolution" --arg wl "$wake_lines" --argjson tt "$timing_json" \
- --argjson pw "$prewarm_ms" \
- '{scenario:$s, cycle:$c, ms:$ms, resolution:$r, wake_phases:($wl | split("\n") | map(select(length>0))), turn_timing:$tt, prewarm_to_ready_ms:$pw}' \
+ --argjson pw "$prewarm_ms" --argjson ab "$baseline_json" \
+ '{scenario:$s, cycle:$c, ms:$ms, resolution:$r, wake_phases:($wl | split("\n") | map(select(length>0))), turn_timing:$tt, prewarm_to_ready_ms:$pw, admission_baseline:$ab}' \
>> "$CYCLES_FILE"
}
@@ -873,6 +884,16 @@ while [ "$cycle" -le "$RECOVERY_CYCLES" ]; do
fail "R2 cycle ${cycle}: turn to suspended host did not reach a definitive 200"
pod_diagnostics; print_results; exit 1
fi
+ # Characterization baseline for a later, separately approved proxy
+ # admission change: first POST HTTP status, POST count through the first
+ # 200, and the first-200 send->200 latency. first_200_ms is deliberately
+ # the same first-200 measurement as RECOVERY_MS (LLM-empty re-sends never
+ # touch it); it is repeated here so the baseline record is self-contained.
+ r2_baseline_json="$(jq -cn \
+ --arg first_status "$ADMISSION_FIRST_STATUS" \
+ --argjson attempts "$ADMISSION_ATTEMPTS" \
+ --argjson first_200_ms "$RECOVERY_MS" \
+ '{first_post_status:$first_status, admission_attempts:$attempts, first_200_ms:$first_200_ms}')"
assert_success_response "R2 cycle ${cycle} turn" "Reply with exactly: ${r2_marker}" || { print_results; exit 1; }
if wait_for_state "active" "$ACTIVE_WAIT"; then
ok "R2 cycle ${cycle}: state=active after wake"
@@ -910,7 +931,7 @@ while [ "$cycle" -le "$RECOVERY_CYCLES" ]; do
fail "R2 cycle ${cycle}: no [StatelessWake] phase=wake_observed line since ${r2_since} -- the wake path is unproven for this cycle"
pod_diagnostics; print_results; exit 1
fi
- record_cycle "R2_cold_resume_from_suspended" "$cycle" "$RECOVERY_MS" "cold" "$r2_wake_lines" "$r2_attr"
+ record_cycle "R2_cold_resume_from_suspended" "$cycle" "$RECOVERY_MS" "cold" "$r2_wake_lines" "$r2_attr" null "$r2_baseline_json"
cycle=$((cycle + 1))
done
@@ -1203,12 +1224,56 @@ for name, budget in budgets.items():
continue
attribution.append({"cycle": r["cycle"], **tt})
breakdown = []
+ baseline = []
for r in recs:
parsed = []
for line in r.get("wake_phases", []):
m = wake_re.search(line)
if m:
parsed.append({"generation": m.group(1), "phase": m.group(2), "ts_ms": int(m.group(3))})
+ if name == "R2_cold_resume_from_suspended":
+ raw = r.get("admission_baseline") or {}
+ for key in ("first_post_status", "admission_attempts", "first_200_ms"):
+ if key not in raw:
+ print(
+ f"scenario {name} cycle {r['cycle']}: admission baseline field '{key}' is missing -- refusing to emit partial baseline data",
+ file=sys.stderr,
+ )
+ raise SystemExit(1)
+ # Pair the two HCC phases by wake generation. A missing phase or
+ # multiple candidate generations stays null with a reason: the
+ # baseline must characterize the observed signal, never invent it.
+ generations = {}
+ for phase in parsed:
+ generations.setdefault(phase["generation"], {})[phase["phase"]] = phase["ts_ms"]
+ paired = [
+ phases["replicas_patched"] - phases["status_flipped"]
+ for phases in generations.values()
+ if "status_flipped" in phases and "replicas_patched" in phases
+ ]
+ baseline_entry = {
+ "cycle": r["cycle"],
+ "first_post_status": raw["first_post_status"],
+ "admission_attempts": raw["admission_attempts"],
+ "first_200_ms": raw["first_200_ms"],
+ "status_flipped_to_replicas_patched_ms": None,
+ }
+ if len(paired) == 1:
+ if paired[0] >= 0:
+ baseline_entry["status_flipped_to_replicas_patched_ms"] = paired[0]
+ else:
+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (
+ f"negative timestamp delta ({paired[0]}ms) in [StatelessWake] lines"
+ )
+ elif len(paired) == 0:
+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (
+ "no [StatelessWake] generation in this cycle has both phase=status_flipped and phase=replicas_patched"
+ )
+ else:
+ baseline_entry["status_flipped_to_replicas_patched_unavailable_reason"] = (
+ f"{len(paired)} [StatelessWake] generations have both phases -- attribution is ambiguous"
+ )
+ baseline.append(baseline_entry)
if parsed:
breakdown.append({"cycle": r["cycle"], "wake_phases": parsed})
# R4 overlap series (prewarm POST -> pod Ready). Informational, no
@@ -1257,6 +1322,10 @@ for name, budget in budgets.items():
entry["turn_attribution"] = attribution
if prewarm:
entry["prewarm_to_ready_ms"] = prewarm
+ if baseline:
+ # Informational characterization only: no budget and no effect on
+ # the existing R2 recovery verdict.
+ entry["cold_resume_baseline"] = baseline
scenarios[name] = entry
artifact = {