🪢 Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
ci(preview): close the legacy-record gaps left by the shared environment (#16834)
Tobias Wochingerd179469bedee
diff --git a/.github/workflows/preview-build.yml b/.github/workflows/preview-build.ymlindex a47e70cfaf..9f149461e5 100644--- a/.github/workflows/preview-build.yml+++ b/.github/workflows/preview-build.yml@@ -350,6 +350,8 @@ jobs:PREVIEW_SHA: ${{ needs.meta.outputs.commit_sha }}PREVIEW_HOST: ${{ needs.meta.outputs.preview_host }}PREVIEW_TASK: preview-pr-${{ github.event.pull_request.number }}+ PR_NUMBER: ${{ github.event.pull_request.number }}+ LEGACY_ENVIRONMENT: pr-${{ github.event.pull_request.number }}with:script: |const { owner, repo } = context.repo;@@ -360,10 +362,40 @@ jobs:const environment = "PR Preview";const task = process.env.PREVIEW_TASK;- const previousDeployments = await github.paginate(- github.rest.repos.listDeployments,- { owner, repo, environment, task, per_page: 100 },- );+ // The job `if` gates on the EVENT payload's state, which is a+ // snapshot from when the build started — a PR closed during the+ // build still reads `open` there. Re-check live, or this records a+ // deployment Argo will never serve and nothing will ever reap+ // (preview-deactivate already fired on close; the stale sweep only+ // scans open PRs).+ const { data: livePr } = await github.rest.pulls.get({+ owner,+ repo,+ pull_number: Number(process.env.PR_NUMBER),+ });+ if (livePr.state !== "open") {+ core.info(`PR closed during the build; not recording a deployment.`);+ return;+ }++ // TRANSITIONAL (drop with the sibling query in+ // preview-deactivate.yml): also retire the pre-migration `pr-<n>`+ // record, else a rebuild leaves it active alongside the new one.+ const previousDeployments = [+ ...(await github.paginate(github.rest.repos.listDeployments, {+ owner,+ repo,+ environment,+ task,+ per_page: 100,+ })),+ ...(await github.paginate(github.rest.repos.listDeployments, {+ owner,+ repo,+ environment: process.env.LEGACY_ENVIRONMENT,+ per_page: 100,+ })),+ ];const { data: deployment } = await github.rest.repos.createDeployment({owner,diff --git a/.github/workflows/preview-stale-cleanup.yml b/.github/workflows/preview-stale-cleanup.ymlindex 17b829dbff..d752927365 100644--- a/.github/workflows/preview-stale-cleanup.yml+++ b/.github/workflows/preview-stale-cleanup.yml@@ -81,6 +81,7 @@ jobs:// (5xx, secondary rate limit, PR locked/closed since pulls.list)// would skip every remaining stale PR and the summary.let failures = 0;+ let deploymentFailures = 0;for (const pr of stale) {const ageDays = ((Date.now() - new Date(pr.updated_at).getTime()) / 86400000).toFixed(1);core.info(`PR #${pr.number} stale ${ageDays}d (updated ${pr.updated_at}) — ${pr.html_url}`);@@ -115,15 +116,34 @@ jobs:// preview-build.yml and preview-deactivate.yml, or// listDeployments matches nothing and dead records pile up.const task = `preview-pr-${pr.number}`;- const deployments = await github.paginate(- github.rest.repos.listDeployments,- { owner, repo, environment: "PR Preview", task, per_page: 100 },- );+ // TRANSITIONAL (drop with the sibling queries in+ // preview-build.yml / preview-deactivate.yml): a reaped PR+ // leaves the labeled set, so no later sweep revisits it — if+ // its pre-migration `pr-<n>` record is not retired here it+ // stays active until the PR eventually closes.+ const deployments = [+ ...(await github.paginate(github.rest.repos.listDeployments, {+ owner,+ repo,+ environment: "PR Preview",+ task,+ per_page: 100,+ })),+ ...(await github.paginate(github.rest.repos.listDeployments, {+ owner,+ repo,+ environment: `pr-${pr.number}`,+ per_page: 100,+ })),+ ];// Isolate per deployment, like the sibling loops in// preview-build.yml / preview-deactivate.yml: one transient- // 5xx must not skip this PR's remaining records. Not counted- // as a reap failure — the label removal above is the teardown,- // and it already succeeded.+ // 5xx must not skip this PR's remaining records. Counted+ // separately from the reap counter — the label removal above+ // is the teardown and it succeeded, but a leftover record has+ // no retry path (this PR is now unlabeled, and a+ // GITHUB_TOKEN label removal cannot fire the `unlabeled`+ // workflow), so it must not pass silently.for (const deployment of deployments) {try {await github.rest.repos.createDeploymentStatus({@@ -138,6 +158,7 @@ jobs:deployment_id: deployment.id,});} catch (e) {+ deploymentFailures++;core.warning(`PR #${pr.number} deployment ${deployment.id}: ${e.message} — continuing`,);@@ -153,9 +174,15 @@ jobs:core.notice(`${dryRun ? "[dry-run] " : ""}${stale.length} stale preview(s) `+ `(threshold ${staleDays}d) out of ${labeled.length} labeled`- + `${failures ? `; ${failures} failed (see warnings)` : ""}.`+ + `${failures ? `; ${failures} failed (see warnings)` : ""}`+ + `${deploymentFailures ? `; ${deploymentFailures} deployment record(s) left behind` : ""}.`);// Surface partial failure as a failed run so it isn't silently green.- if (failures) {- core.setFailed(`${failures} of ${stale.length} stale preview(s) could not be reaped.`);+ // Deployment-record failures count too: the reap itself succeeded,+ // but nothing will retry the leftover record before PR close.+ if (failures || deploymentFailures) {+ core.setFailed(+ `${failures} of ${stale.length} stale preview(s) could not be reaped; `+ + `${deploymentFailures} deployment record(s) could not be deleted.`+ );}