Skip to content

armgit / eve

Public Imported from GitHub
0

The Open Framework for Building Agents

test(e2e): wait for Vercel eval targets (#2972)

OwenKephartf451457210e1
diff --git a/.github/workflows/e2e-vercel.yml b/.github/workflows/e2e-vercel.yml
index 7c7caeb7a..5ae3cfc21 100644
--- a/.github/workflows/e2e-vercel.yml
+++ b/.github/workflows/e2e-vercel.yml
@@ -157,6 +157,7 @@ jobs:
pnpm exec vc link --yes --project "$VERCEL_PROJECT_ID" "${team_args[@]}" "${token_args[@]}"
pnpm exec vc env pull --yes --environment=preview "${token_args[@]}"
+ VERCEL_OIDC_TOKEN="$(node --env-file=.env.local -p 'process.env.VERCEL_OIDC_TOKEN ?? ""')"
# Sandbox templates key on VERCEL_PROJECT_ID (present at build and
# runtime). Do not add VERCEL_TEAM_ID: it does not exist at runtime,
# so the deployed function could not resolve a team-scoped template.
@@ -170,6 +171,26 @@ jobs:
--env "EVE_SANDBOX_IMAGE_TAG=$EVE_SANDBOX_IMAGE_TAG" \
"${token_args[@]}" | tail -n 1)"
+ wait_for_target() {
+ local target="$1"
+ local attempt status
+ for attempt in $(seq 1 15); do
+ status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
+ --header "Authorization: Bearer $VERCEL_OIDC_TOKEN" \
+ --header "x-vercel-trusted-oidc-idp-token: $VERCEL_OIDC_TOKEN" \
+ --max-time 3 "$target/eve/v1/health" || true)"
+ if [ "$status" = "200" ]; then
+ echo "Eval target ready: $target (attempt $attempt)"
+ return 0
+ fi
+ echo "Eval target not ready: $target (attempt $attempt/15, health status ${status:-transport-error})"
+ sleep 1
+ done
+ echo "Timed out waiting for eval target health: $target (last health status ${status:-transport-error})" >&2
+ return 1
+ }
+
+ wait_for_target "$DEPLOYMENT_URL"
npx eve eval --strict --verbose --exclude-tag real-model \
--url "$DEPLOYMENT_URL" \
--junit "$EVE_EVAL_JUNIT_DIR/${{ matrix.name }}.xml"
@@ -184,6 +205,7 @@ jobs:
# vc alias does not infer the team from the project link the way
# link/deploy do, so pass the scope explicitly.
pnpm exec vc alias set "$DEPLOYMENT_URL" "$ALIAS" "${token_args[@]}" "${scope_args[@]}"
+ wait_for_target "https://$ALIAS"
EVE_E2E_REDEPLOY_ALIAS="$ALIAS" \
npx eve eval --tag redeploy --strict --verbose --exclude-tag real-model \
--url "https://$ALIAS" \
diff --git a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
index d2a7fc731..1a4bddd89 100644
--- a/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
+++ b/e2e/fixtures/agent-channels/evals/custom-channels/cross-version-session-inbox.eval.ts
@@ -227,12 +227,21 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
/** Waits until the alias exposes the marker from the expected deployment. */
async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {
const deadline = Date.now() + 120_000;
+ let lastStatus = "transport error";
+ let lastMarkerMatch = false;
while (Date.now() < deadline) {
- const response = await t.target.fetch("/eve/v1/info");
- if (response.ok && JSON.stringify(await response.json()).includes(marker)) {
- return;
+ try {
+ const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });
+ lastStatus = String(response.status);
+ lastMarkerMatch = response.ok && JSON.stringify(await response.json()).includes(marker);
+ if (lastMarkerMatch) return;
+ } catch {
+ // The alias may briefly be unavailable while its deployment propagates.
}
await t.sleep(1_000);
}
- throw new Error(`Timed out waiting for the alias to serve a deployment containing ${marker}.`);
+ throw new Error(
+ `Timed out waiting for alias ${new URL(t.target.url).host} to serve marker ${marker}; ` +
+ `last status=${lastStatus}, marker matched=${lastMarkerMatch}.`,
+ );
}
diff --git a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts
index 648820191..7ce8a935c 100644
--- a/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts
+++ b/e2e/fixtures/agent-tools-sandbox/evals/sandbox/redeploy.eval.ts
@@ -202,17 +202,28 @@ async function deployToAlias(t: EveEvalContext, alias: string, phase: string): P
async function waitForAliasToServe(t: EveEvalContext, marker: string): Promise<void> {
const deadline = Date.now() + 120_000;
let consecutiveMatches = 0;
+ let lastStatus = "transport error";
+ let lastMarkerMatch = false;
while (Date.now() < deadline) {
- const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });
- if (response.ok && JSON.stringify(await response.json()).includes(marker)) {
- consecutiveMatches += 1;
- if (consecutiveMatches >= ALIAS_SETTLE_MATCHES) {
- return;
+ try {
+ const response = await t.target.fetch("/eve/v1/info", { cache: "no-store" });
+ lastStatus = String(response.status);
+ lastMarkerMatch = response.ok && JSON.stringify(await response.json()).includes(marker);
+ if (lastMarkerMatch) {
+ consecutiveMatches += 1;
+ if (consecutiveMatches >= ALIAS_SETTLE_MATCHES) {
+ return;
+ }
+ } else {
+ consecutiveMatches = 0;
}
- } else {
+ } catch {
consecutiveMatches = 0;
}
await t.sleep(1_000);
}
- throw new Error(`Timed out waiting for the alias to serve a deployment containing ${marker}.`);
+ throw new Error(
+ `Timed out waiting for alias ${new URL(t.target.url).host} to serve marker ${marker}; ` +
+ `last status=${lastStatus}, marker matched=${lastMarkerMatch}, consecutive matches=${consecutiveMatches}.`,
+ );
}