The Open Framework for Building Agents
test(e2e): wait for Vercel eval targets (#2972)
OwenKephartf451457210e1
diff --git a/.github/workflows/e2e-vercel.yml b/.github/workflows/e2e-vercel.ymlindex 7c7caeb7a..5ae3cfc21 100644--- a/.github/workflows/e2e-vercel.yml+++ b/.github/workflows/e2e-vercel.yml@@ -157,6 +157,7 @@ jobs:pnpm exec vc link --yes --project "$VERCEL_PROJECT_ID" "${team_args[@]}" "${token_args[@]}"pnpm exec vc env pull --yes --environment=preview "${token_args[@]}"+ VERCEL_OIDC_TOKEN="$(node --env-file=.env.local -p 'process.env.VERCEL_OIDC_TOKEN ?? ""')"# Sandbox templates key on VERCEL_PROJECT_ID (present at build and# runtime). Do not add VERCEL_TEAM_ID: it does not exist at runtime,# so the deployed function could not resolve a team-scoped template.@@ -170,6 +171,26 @@ jobs:--env "EVE_SANDBOX_IMAGE_TAG=$EVE_SANDBOX_IMAGE_TAG" \"${token_args[@]}" | tail -n 1)"+ wait_for_target() {+ local target="$1"+ local attempt status+ for attempt in $(seq 1 15); do+ status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \+ --header "Authorization: Bearer $VERCEL_OIDC_TOKEN" \+ --header "x-vercel-trusted-oidc-idp-token: $VERCEL_OIDC_TOKEN" \+ --max-time 3 "$target/eve/v1/health" || true)"+ if [ "$status" = "200" ]; then+ echo "Eval target ready: $target (attempt $attempt)"+ return 0+ fi+ echo "Eval target not ready: $target (attempt $attempt/15, health status ${status:-transport-error})"+ sleep 1+ done+ echo "Timed out waiting for eval target health: $target (last health status ${status:-transport-error})" >&2+ return 1+ }++ wait_for_target "$DEPLOYMENT_URL"npx eve eval --strict --verbose --exclude-tag real-model \--url "$DEPLOYMENT_URL" \--junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"@@ -184,6 +205,7 @@ jobs:# vc alias does not infer the team from the project link the way# link/deploy do, so pass the scope explicitly.pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}"+ wait_for_target "https://$ALIAS"EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \npx eve eval --tag redeploy --strict --verbose --exclude-tag real-model \--url "https://$ALIAS" \diff --git a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.tsindex d2a7fc731..1a4bddd89 100644--- a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts+++ b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts@@ -227,12 +227,21 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P/** Waits until the alias exposes the marker from the expected deployment. */async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {const deadline = Date.now() + 120_000;+ let lastStatus = "transport error";+ let lastMarkerMatch = false;while (Date.now() < deadline) {- const response = await t.target.fetch("/eve/v1/info");- if (response.ok && JSON.stringify(await response.json()).includes(marker)) {- return;+ try {+ const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });+ lastStatus = String(response.status);+ lastMarkerMatch = response.ok && JSON.stringify(await response.json()).includes(marker);+ if (lastMarkerMatch) return;+ } catch {+ // The alias may briefly be unavailable while its deployment propagates.}await t.sleep(1_000);}- throw new Error(`Timed out waiting for the alias to serve a deployment containing ${marker}.`);+ throw new Error(+ `Timed out waiting for alias ${new URL(t.target.url).host} to serve marker ${marker}; ` ++ `last status=${lastStatus}, marker matched=${lastMarkerMatch}.`,+ );}diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.tsindex 648820191..7ce8a935c 100644--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts@@ -202,17 +202,28 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): Pasync function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {const deadline = Date.now() + 120_000;let consecutiveMatches = 0;+ let lastStatus = "transport error";+ let lastMarkerMatch = false;while (Date.now() < deadline) {- const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });- if (response.ok && JSON.stringify(await response.json()).includes(marker)) {- consecutiveMatches += 1;- if (consecutiveMatches >= ALIAS_SETTLE_MATCHES) {- return;+ try {+ const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });+ lastStatus = String(response.status);+ lastMarkerMatch = response.ok && JSON.stringify(await response.json()).includes(marker);+ if (lastMarkerMatch) {+ consecutiveMatches += 1;+ if (consecutiveMatches >= ALIAS_SETTLE_MATCHES) {+ return;+ }+ } else {+ consecutiveMatches = 0;}- } else {+ } catch {consecutiveMatches = 0;}await t.sleep(1_000);}- throw new Error(`Timed out waiting for the alias to serve a deployment containing ${marker}.`);+ throw new Error(+ `Timed out waiting for alias ${new URL(t.target.url).host} to serve marker ${marker}; ` ++ `last status=${lastStatus}, marker matched=${lastMarkerMatch}, consecutive matches=${consecutiveMatches}.`,+ );}