πͺ’ Open source agent evals & observability: Trace, evaluate, and improve LLM applications with one open platform.
TypeScript analyticsautogenevaluationlangchainlarge-language-modelsllama-indexllmllm-evaluationllm-observabilityllmopsmonitoringobservabilityopen-sourceopenaiplaygroundprompt-engineeringprompt-managementself-hostedycombinator
ci: drive flake tracking from the merge queue's own failures (#18759)
Der GerΓ€ta8f9a7a5ed48
diff --git a/.github/workflows/ci-runtime-analyst.lock.yml b/.github/workflows/ci-runtime-analyst.lock.ymlindex d6c67c69a7..ef01753650 100644--- a/.github/workflows/ci-runtime-analyst.lock.yml+++ b/.github/workflows/ci-runtime-analyst.lock.yml@@ -1,4 +1,4 @@-# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"43b54f1482a75b574f9930a7bdcc3398d164ce4b9649dd8282d9163c4ccd3a8b","body_hash":"ab51dd8e1cd3aed3e810fb70e94e99ecbd1f6a39c6279c388615a80bd7ed294e","compiler_version":"v0.86.2","agent_id":"claude","agent_model":"claude-opus-5","engine_versions":{"claude":"2.1.277"}}+# gh-aw-metadata: {"schema_version":"v4","frontmatter_hash":"43b54f1482a75b574f9930a7bdcc3398d164ce4b9649dd8282d9163c4ccd3a8b","body_hash":"6c08d3666942a25a12a598c57f98d188a5d4ea999bdf88623955ce0afdf73abe","compiler_version":"v0.86.2","agent_id":"claude","agent_model":"claude-opus-5","engine_versions":{"claude":"2.1.277"}}# gh-aw-manifest: {"version":1,"secrets":["CLAUDE_API_KEY","COPILOT_GITHUB_TOKEN","GH_AW_CI_ANALYST_TOKEN","GH_AW_GITHUB_MCP_SERVER_TOKEN","GH_AW_GITHUB_TOKEN","GITHUB_TOKEN"],"actions":[{"repo":"actions/cache/restore","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/cache/save","sha":"55cc8345863c7cc4c66a329aec7e433d2d1c52a9","version":"v6.1.0"},{"repo":"actions/checkout","sha":"3d3c42e5aac5ba805825da76410c181273ba90b1","version":"v7.0.1"},{"repo":"actions/download-artifact","sha":"3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c","version":"v8.0.1"},{"repo":"actions/github-script","sha":"3a2844b7e9c422d3c10d287c895573f7108da1b3","version":"v9.0.0"},{"repo":"actions/setup-node","sha":"820762786026740c76f36085b0efc47a31fe5020","version":"v7.0.0"},{"repo":"actions/upload-artifact","sha":"043fb46d1a93c77aae656e7c1c64a875d1fc6a0a","version":"v7.0.1"},{"repo":"github/gh-aw-actions/setup","sha":"6aab9e5b5c91c615506061f09bedd81a23babe3c","version":"v0.86.2"}],"containers":[{"image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44","digest":"sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4","pinned_image":"ghcr.io/github/gh-aw-firewall/agent:0.27.44@sha256:0d727725c737b58c7bdf51f640cffb928385ec46517e0917c7f1a02f1bada8b4"},{"image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44","digest":"sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7","pinned_image":"ghcr.io/github/gh-aw-firewall/api-proxy:0.27.44@sha256:b50fbadba138f6e9aba94aca09711335c489bb3b15861220cb66f6092e042dc7"},{"image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44","digest":"sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627","pinned_image":"ghcr.io/github/gh-aw-firewall/squid:0.27.44@sha256:83e48bbe12c634be8c228a576832fe45f66c529ac3659db92bddbcf2eeb6d627"},{"image":"ghcr.io/github/gh-aw-mcpg:v0.4.9","digest":"sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f","pinned_image":"ghcr.io/github/gh-aw-mcpg:v0.4.9@sha256:e5a1569aeaf41820fa7bdee3e94468cae448133cdbf00119ad24f5b74db1ab9f"},{"image":"ghcr.io/github/gh-aw-node","digest":"sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196","pinned_image":"ghcr.io/github/gh-aw-node@sha256:0d9f1fb5fd6610c0ac1f5194a38e45a8a1e81f8a390d5142d8e4e6f26a4b3196"},{"image":"ghcr.io/github/github-mcp-server:v1.9.0","digest":"sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e","pinned_image":"ghcr.io/github/github-mcp-server:v1.9.0@sha256:881b53d6f75f69bdbc1b5b10fc2f1361717c19054143b3a8529fb5c32061a50e"}]}# This file was automatically generated by gh-aw (v0.86.2). DO NOT EDIT. To debug this workflow, load the skill at https://github.com/github/gh-aw/blob/main/debug.md#diff --git a/.github/workflows/ci-runtime-analyst.md b/.github/workflows/ci-runtime-analyst.mdindex 4555a64398..0721d863e7 100644--- a/.github/workflows/ci-runtime-analyst.md+++ b/.github/workflows/ci-runtime-analyst.md@@ -153,6 +153,10 @@ checklist is only the spine. Every run ends the same way: exactly one issue.day for daily medians) and compare against history; investigate anysustained intra-week shift in this same run ("Metric definitions","Judging and acting").+- [ ] Analyse **every** failed merge-queue run in the window β attribute+ each to the job that failed and, where that job ran tests, to the test+ that failed ("Merge-queue failure analysis"). This is the primary flake+ signal; do not skip it because the timing numbers look quiet.- [ ] Parse vitest logs; update week-over-week flaky-test tracking, andmine the slowest tests for optimization candidates even when nothingregressed ("Vitest output analysis", "Judging and acting").@@ -166,9 +170,10 @@ checklist is only the spine. Every run ends the same way: exactly one issue.- [ ] Update all memory files, including `charts/<week>.svg`, `issues.json`,and pruned `notes.md`.- [ ] Write the FULL report β both filled-in `mermaid` charts (Chart 1 and- Chart 2, never the bare template), tables, `## Outcome` section β as the- issue body. Before calling `create_issue`, confirm the message you're- about to submit literally contains two ` ```mermaid ` blocks ("Report+ Chart 2, never the bare template), tables, the merge-queue failures+ section, `## Outcome` section β as the issue body. Before calling+ `create_issue`, confirm the message you're about to submit literally+ contains two ` ```mermaid ` blocks ("Reportand graph" β final gate). This holds even when you skip a fresh analysis(reuse the latest `history/*.json` numbers and say so); never file anissue with a one-line body ("Report and graph").@@ -250,26 +255,47 @@ Population rules:- **Timing statistics** (perceived/wall, execution, runner wait, segmentmedians) come exclusively from successful `merge_group` runs β they carrythe code changes into main and are directly comparable. Do not mix- `pull_request` or `push` timings into these aggregates.-- **Everything else** (vitest output analysis, slowest tests,- retried/flaky tests) draws on all successful runs of the week regardless- of event, EXCEPT runs on `main` (`push` events) β i.e. `merge_group` plus- `pull_request` runs.-- Exclude failed and cancelled runs from every analysis; count them- separately as context only.+ `pull_request` or `push` timings into these aggregates, and do not mix in a+ failed run: it stopped early, so its durations measure nothing.+- **Slowest tests and files** draw on all successful runs of the week+ regardless of event, EXCEPT runs on `main` (`push` events) β i.e.+ `merge_group` plus `pull_request` runs.+- **Failure and flake tracking takes the opposite population**: the week's+ **failed `merge_group` runs**, analysed in full rather than counted+ ("Merge-queue failure analysis"). Each one ejected a PR from the queue and+ names both the job and the test that did it, which makes it a sharper flake+ signal than a sample of green runs that happened to survive their retries.+ Retried-test data from the successful sample is a secondary input, not the+ source of truth.+- Cancelled runs are excluded from every analysis β a cancellation carries no+ verdict about the code. Count them separately as context only.## Vitest output analysis+This section describes the reporter output. It is read from two populations:+the successful sample below (for slow tests), and every failed merge-queue run+("Merge-queue failure analysis", for failures and flakes).+For a sample of successful runs spread across the week β `merge_group` and`pull_request` events, never `push`/main runs (at least 5 runs, or all runs-if fewer), for each sampled run read the log of the `run tests` step of the-`tests-web (β¦)` matrix jobs and of the `tests-worker (β¦)` matrix jobs-directly ("Extracting bulk data without bloating your own context"). The-blocks below sit at the very end of the step, so pass `get_job_logs` a-`tail_lines` just large enough to cover them (~150; the default is 500) rather-than pulling the whole log. Our-CI reporter (`scripts/vitest/ci-reporter.ts`) prints up to three blocks at-the end of every run:+if fewer), for each sampled run read the log of the test step of the+`tests-web (β¦)` matrix jobs, the `tests-worker (β¦)` matrix jobs and+`tests-storybook` directly β `run tests` for the first two, `run Storybook+tests` for the third; the table in "Merge-queue failure analysis" has the+rest ("Extracting bulk data without bloating your own context"). The blocks+below sit at the very end of the step, so pass+`get_job_logs` a `tail_lines` just large enough to cover them (~250; the+default is 500) rather than pulling the whole log.++Every job that runs the `web` or `worker` vitest config emits these blocks β+`tests-web (β¦)`, `tests-web-client`, `tests-worker (β¦)`, `tests-storybook`,+`e2e-server-tests`, `test-worker-llm-connections`, `test-ai-gateway-e2e`, and+`tests-fips (β¦)` (whose step also runs the `shared` suite, which emits+nothing). Jobs with their own config do NOT (`tests-shared`,+`tests-eslint-plugin`, `tests-in-app-agent-sandbox-runtime`), and neither does+`e2e-tests`, which is Playwright. Our CI reporter+(`scripts/vitest/ci-reporter.ts`) prints up to four blocks at the end of every+run:- `Slowest tests (top 10):` β ranked list with durations; a test thatneeded vitest retries additionally carries ` [retries=N]` and possibly@@ -281,30 +307,196 @@ the end of every run:`1. retries=2 <file> > <name> [flaky]` β note: no brackets around`retries=` here). This block is printed ONLY when at least one testretried, so its absence means zero retries in that run. Use this block,- not the slowest-tests markers, as the source of truth for flaky- tracking β a flaky test that isn't among the 10 slowest appears only- here.+ not the slowest-tests markers, as the source of truth for *retries* β a+ flaky test that isn't among the 10 slowest appears only here.+- `Failed tests (N):` β every test that ended the run in a failed state,+ same line shape minus the suffixes (`1. retries=0 <file> > <name>`).+ Printed only when something failed, so it is absent from every green run.+ β οΈ **This, not `Retried tests`, is the source of truth for what broke a+ failed run.** A suite can opt out of retries with+ `describe(..., { retry: 0 })`, and such a test fails with `retries=0` β so+ it appears in no retried block at all, and in the slowest-tests block only+ if it happens to rank in the top ten by duration.+ **`N` in the header is the true total; the list below it is capped at 50**+ and ends with a `β¦ and K more` line when it bites. So take the count from+ the header, never from the number of lines you parsed.+ β οΈ `retries` says how many attempts the test was **given**, not why it+ failed. A count equal to the config's CI retry setting (3 for both `web`+ and `worker`) means it had retries and used them all β which a+ deterministic regression does exactly as readily as a flake. `retries=0`+ means it was given none: a suite with `{ retry: 0 }`, or a failure outside+ the retry loop. **So never infer flakiness from the retry count.** A test+ earns the word *flaky* only when something shows it also passes β a+ `[flaky]` marker in another run, the same test green in a sibling matrix+ leg of the same run, or a green run on the same commit. Absent that, call+ it a failure and say the flakiness is unestablished.Extract and keep only the parsed blocks for each run, never the raw log text:```json-{"runId": 123, "event": "merge_group",- "slowest": [{"file": "web/src/x.test.ts", "test": "name", "ms": 4210}],- "slowestFiles": [{"file": "web/src/x.test.ts", "ms": 9100}],- "retried": [{"file": "web/src/y.test.ts", "test": "name", "retries": 2,- "flaky": true}]}+{"runId": 123, "event": "merge_group", "conclusion": "success",+ "job": "tests-web (node24, pg15, mode, otel-worker-shadow=true)",+ "slowest": [{"file": "src/x.test.ts", "test": "name", "ms": 4210}],+ "slowestFiles": [{"file": "src/x.test.ts", "ms": 9100}],+ "retried": [{"file": "src/y.test.ts", "test": "name", "retries": 2,+ "flaky": true}],+ "failed": [{"file": "src/z.test.ts", "test": "name", "retries": 3}]}```-An empty `retried` array means the `Retried tests` block was absent, i.e. zero-retries β not that you failed to find it.+An empty `retried` or `failed` array means that block was absent, i.e. zero+retries / zero failures β not that you failed to find it. **Two exceptions,+both recorded as `unknown` rather than zero:**++- **A run that predates the `Failed tests` block.** It landed in October+ 2026, so the first windows straddling it contain runs that could not have+ printed it. A *failed* run reporting zero failed tests is a contradiction β+ treat that combination as the tell, not as a quiet zero.+- **A truncated tail.** Confirm you actually saw the `Failed tests (N):`+ header, not just a run of numbered lines β one entry is not always one+ line (a `test.each` name holding pretty-printed JSON has spanned 19), so a+ tail can start mid-block. No header means widen the tail once, then record+ `unknown`. A truncated block must never read as a short one.+- And a third case where the block is **absent from a genuinely failed run**:+ vitest can fail outside any test case β a collection or import error, a+ `globalSetup` throw, a worker crash. The reporter only sees completed test+ cases, so it prints nothing at all. `Test Files N failed` in the summary+ above with no blocks beneath it is that shape: record `unknown` and read+ the error from the log rather than filing the job as zero failures.Aggregate across the sampled runs:- Recurring slowest tests and files (name, file, median duration, how manysampled runs they appeared in).-- **Retried/flaky tests**: every test that shows `[retries=N]` or `[flaky]`,- with occurrence counts. Track these week over week in memory β a test that- is flaky two weeks in a row deserves a callout.+- **Retried/flaky tests**: every test in the `Retried tests` block, with+ occurrence counts and whether it carried `[flaky]` (the slowest-tests+ markers are a convenience, not the membership rule). Track these week over+ week in memory β a test that is flaky two weeks in a row deserves a+ callout.++## Merge-queue failure analysis++A failed `merge_group` run is the sharpest CI signal this repository+produces: it ejected a PR from the merge queue, the jobs API names the job+that did it, and that job's log names the test. Unlike the timing work, this+is **not sampled** β analyse every failed merge-queue run in the window.++Steps 1β3 are pure API work and cost almost nothing, so they cover **every**+failed run in the window. Only step 4 reads logs, and only that step is+capped.++1. From the same `created=<from>..<to>` run list, take every run with+ `event == "merge_group"` and a `conclusion` of `failure` or `timed_out`.+2. For each, `GET /repos/{owner}/{repo}/actions/runs/{id}/jobs?per_page=100`+ and keep the jobs whose `conclusion` is `failure` **or `timed_out`**. A+ job that hits its `timeout-minutes` (10 for `tests-storybook`, 15 for+ `tests-shared`) concludes `timed_out`, not `failure`, so filtering on+ `failure` alone silently drops the job that ejected the PR.+ **Discard `all-ci-passed`.** It is the aggregating gate, so it fails+ whenever anything upstream of it fails and attributes nothing; left in, it+ would top every table in this section with no information in it.+3. Classify each remaining job from **the failing step's name**, which is+ already in the `steps` array of the payload you just fetched β no log read+ needed. Match the step name against this table rather than guessing from+ the word "test": several test steps do not contain it, and one step that+ does is not a test step at all.++ | Job | Its test step(s) |+ | --- | --- |+ | `tests-web (β¦)` | `run tests` |+ | `tests-worker (β¦)` | `run tests` **and** `run native codec tests against ClickHouse` β two of them |+ | `tests-web-client` | `run test-client` |+ | `tests-storybook` | `run Storybook tests` |+ | `e2e-server-tests` | `Run e2e tests` β οΈ same step name as `e2e-tests` below; classify by **job name**, not step name |+ | `e2e-tests` | `Run e2e tests` |+ | `test-worker-llm-connections` | `run llm connection tests` |+ | `test-ai-gateway-e2e` | `Run AI Gateway end-to-end tests` |++ - **test failure** β the failing step is one of the above. Only these feed+ flake tracking.+ - **infrastructure failure** β the job died before reaching a verdict+ about the code: `install dependencies`, a checkout, a docker login or+ pull, `Run + migrate`, a compose health check, a playwright or+ golang-migrate download, `Build`. Registry and supply-chain errors+ (`ERR_PNPM_META_FETCH_FAIL`, a failed lockfile policy check) and apt or+ mirror outages land here, as does a lost runner. Count and report these+ **separately**: a registry having a bad day is not a flaky week, and+ folding the two together is how a week gets misdiagnosed.+ - β οΈ **`tests-fips (web|worker)` cannot be classified by step name.** Its+ single step `Run <suite> tests under forced FIPS` shells out to+ `scripts/ci/fips-tests.sh`, which runs `pnpm install`, `db:generate` and+ the build *inside* that step before it reaches a test. On run+ 38435646285 both fips legs failed there with `ERR_PNPM_META_FETCH_FAIL`+ β infrastructure wearing a test step's name. Always read its tail.+ - **timed out** β the job hit its `timeout-minutes`. Record the time+ limit as the cause, and classify it by its running step where that is+ legible. Its test-level detail is `unknown`, never zero: a job killed+ mid-run never reached the end of the step, so it printed none of the+ reporter's blocks.+ - Any other step name is **ambiguous** β read the tail and look. Budget+ one of step 4's log tails for it. But note+ `ERR_PNPM_RECURSIVE_RUN_FIRST_FAIL` classifies nothing by itself, it is+ pnpm relaying a child's exit code, so read the lines above it.++ The step names come from the payload, so if this table drifts out of date+ the symptom is a job falling to "ambiguous" and costing a log read β never+ a confident wrong answer.+4. For test failures, read the tail of the failing step and parse it+ ("Extracting bulk data without bloating your own context"):+ - vitest-backed jobs (`tests-web`, `tests-worker`, `tests-web-client`,+ `tests-storybook`, `e2e-server-tests`) β the blocks in+ "Vitest output analysis". `Failed tests (N):` lists every non-passed+ *test case*; suite-level failures (`beforeAll`/`afterAll` hooks or+ unhandled errors) do not appear in it β read the tail before it for+ those. When `e2e-server-tests` fails, use the job name β not the step+ name β to pick this parser over the Playwright one below.+ - `e2e-tests` β Playwright run with `--reporter=line`, which prints none+ of those blocks. Its failures are the numbered+ `N) [project] βΊ <file>:<line> βΊ <test>` entries and the `N failed`+ summary line near the end of the step.+ Budget: **one log tail per failed run** (sibling matrix legs almost always+ carry the same failing test, so take the first failing test job and skip+ the rest), and **at most 25 log tails for this section**. If the window+ holds more failed runs than that, spend the budget on the most recent ones+ and on jobs not already represented, and say in the report how many runs+ went un-inspected. Those runs still count in the rate and in the+ by-job table from step 3 β only their test-level detail is missing.++Keep one small record per failed run, never the raw log:++```json+{"runId": 123, "conclusion": "failure", "createdAt": "2026-10-13T15:57:55Z",+ "failedJobs": [{"name": "tests-worker (node24, pg15, mode-redis-cluster)",+ "kind": "test",+ "failed": [{"file": "src/__tests__/x.test.ts",+ "test": "name", "retries": 0}]}]}+```++Aggregate across the week:++- **Merge-queue failure rate** β (failed + timed-out) Γ· (failed + timed-out+ + successful) merge-group runs, with the infrastructure share broken out+ so the test-attributable rate is readable on its own. Record the numerator+ split, since this figure goes into memory and a later week comparing+ against it cannot ask what it included. This denominator is the only place+ a failed run touches a statistic about the pipeline; keep it out of the+ timing medians.+- **Failures by job**, most damaging first.+- **A test-level ledger** β for each test, the number of merge-queue runs it+ failed. Key it on **base job name + file + test**, not file + test: the+ reporter prints vitest's module id *relative to the package it ran from*,+ so a web test and a worker test both read as `src/β¦` and would silently+ merge. Base job name means the text before the first ` (` in a matrix leg+ β `tests-worker`, not `tests-worker (node24, pg15, mode-redis-cluster)` β+ so a test that ejects two PRs on different legs counts as two failures of+ one test, not one failure each of two tests. Keep the legs seen as a+ separate field for diagnostics.+ This is a *failure* ranking; which entries are flakes rather than+ regressions is a separate judgement, made on the evidence named under+ `Failed tests (N):` and not on the retry count.++Carry all three into memory and into the report, and track them week over+week exactly like the flaky list: a test that ejects PRs from the merge queue+in two consecutive weeks is a callout and a suggested-diff candidate.## Memory (repo memory at `/tmp/gh-aw/repo-memory/default/`, branch `memory/ci-runtime-analysis`)@@ -315,7 +507,12 @@ this layout:merge-group timing aggregates (run count, p50/p90 perceived, p50/p90execution, p50/p90 runner wait, daily and weekly medians for the Buildstep, `run tests` step, and e2e-tests job), plus the week's slowest and- flaky tests (from merge-group + pull-request runs).+ flaky tests (from merge-group + pull-request runs), plus the three+ merge-queue failure aggregates from "Merge-queue failure analysis"+ (`failureRate` with its test and infrastructure shares, `failuresByJob`,+ and the `failuresByTest` ledger). The failure aggregates are what makes+ "this test broke the queue again this week" answerable next run, so never+ omit them, even on a week with no failures β record the zeroes.- `issues.json` β ledger of every issue this workflow has opened, oldestfirst, entries: `{number, url, openedAt, title, suggestedDiff: bool,expectedImpact: {metric, baseline, expected} | null, baselineStats: {..},@@ -352,7 +549,19 @@ this layout:trend, runner-wait share, Build / `run tests` step drift, new orpersistent flaky tests. Call out regressions larger than ~10% on medianswith links to the first run(s) exhibiting them.-2. **Sustained intra-week shifts are actionable on their own** β a step+2. **A test that ejects PRs from the merge queue outranks a slow one.** Work+ the `failuresByTest` ledger from "Merge-queue failure analysis" before the+ slowest-tests list: a test responsible for two or more merge-queue+ failures in the window, or one that appears in the ledger two weeks+ running, is this week's first suggested-diff candidate β a flake that+ survives its retries costs seconds, one that exhausts them costs a PR its+ place in the queue. Say in the report how many queue failures each+ candidate caused, and say whether it is a flake or a regression on the+ evidence rather than on the retry count: a regression wants a fix or a+ revert, a flake wants the test stabilised. If the week's failures are+ dominated by the infrastructure class instead, say that plainly and do+ not dress it up as a flaky week.+3. **Sustained intra-week shifts are actionable on their own** β a stepmedian moving β₯50% across three or more consecutive days (e.g.`run tests` doubling within the week) must be investigated in the samerun, not parked as "noisy" or deferred for lack of week-over-week@@ -361,14 +570,14 @@ this layout:slowest-tests output from runs before vs after, and name the suspecttests/PRs in the report. If the culprit is an in-surface test or config,that is a suggested-diff candidate for this week's issue.-3. **You are not only a regression watchdog.** Every week, also mine the+4. **You are not only a regression watchdog.** Every week, also mine thevitest slowest-tests/files output for optimization potential: serialawaits that could run concurrently, expensive setup repeated per-testthat could be hoisted, oversized fixtures, unnecessary sleeps/timeouts,redundant DB round-trips. A quiet week with no regressions is the besttime to land one such improvement. Missing baseline history blocksregression *claims* β it never blocks optimizing a measurably slow test.-4. Only when you have a concrete improvement whose expected effect you can+5. Only when you have a concrete improvement whose expected effect you canjustify from the measured data β and that passed the verificationdescribed below β include it in this run's issue as a suggested diff.You never edit repository files as part of this workflow's own change@@ -381,7 +590,7 @@ this layout:- `.github/workflows/pipeline.yml` itself β do NOT edit it, but asuggested diff against it is fine to include in the issue like anyother candidate.-5. Write every suggested diff as a fenced `diff` code block in the issue,+6. Write every suggested diff as a fenced `diff` code block in the issue,with the concrete `expectedImpact` metric and baseline it should bejudged against (record the same in `issues.json`) and the verificationyou ran ("Verify changes before recommending a fix"). If nothing passed@@ -506,15 +715,26 @@ the history does not cover are computed fresh from the API in this run.The report always contains, in order:1. The **weekly chart** with its values table (template below).-2. A markdown table of the top slow tests and the retried/flaky tests.-3. Any suggested diff(s), each as a fenced `diff` code block with a short+2. A **merge-queue failures** section from "Merge-queue failure analysis":+ the failure rate with its test and infrastructure shares, a table of+ failures by job, and a table of the `failuresByTest` ledger (test, file,+ runs failed, retry counts seen), most damaging first, each row linking the+ failed run(s). Say how many failed runs were swept and how many went+ un-inspected because the log-tail budget bound. A week with no merge-queue+ failures still gets the section, saying so.++ | Test | File | Queue failures | Retries seen |+ |---|---|---|---|+ | β¦ | β¦ | β¦ | β¦ |+3. A markdown table of the top slow tests and the retried/flaky tests.+4. Any suggested diff(s), each as a fenced `diff` code block with a short"what changed and why" note, the `expectedImpact` metric and baseline,and the exact verification you ran (command + quoted summary line),separately noting what could not be verified in this sandbox.-4. An **`## Outcome` section β mandatory, always present**: whether a+5. An **`## Outcome` section β mandatory, always present**: whether asuggested diff was included, and if not, a numbered list of theconcrete reasons why not.-5. A "Previously opened issues" section from `issues.json`, oldest first:+6. A "Previously opened issues" section from `issues.json`, oldest first:whether each suggested diff was applied on `main`, and whether thechange moved the following week's numbers.@@ -582,11 +802,12 @@ three series in one chart, and never move the legend into the chart titlestring you are about to submit and confirm, mechanically, that it containsAT LEAST two ` ```mermaid ` fenced code blocks (Chart 1 and Chart 2, filledin β not the bare template) β FOUR once `history/*.json` holds 2+ weeks and-the weekly charts below are also required β plus the values table and an-`## Outcome` section. A report missing any of these is incomplete and must-not be submitted as-is β go back and add the missing piece(s) first.-"Nothing was actionable" changes the Outcome section's content, never-whether the charts are present or whether the issue gets filed.+the weekly charts below are also required β plus the values table, a+merge-queue failures section, and an `## Outcome` section. A report missing+any of these is incomplete and must not be submitted as-is β go back and add+the missing piece(s) first. "Nothing was actionable" and "no merge-queue+failures this week" change those sections' content, never whether they are+present or whether the issue gets filed.Additionally, render the same weekly data as a standalone SVG chart(hand-write the SVG: time on x, seconds on y, one polyline per series withdiff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.ymlindex c6be13cf1b..22a8696f7c 100644--- a/.github/workflows/pipeline.yml+++ b/.github/workflows/pipeline.yml@@ -270,6 +270,8 @@ jobs:run: node --test scripts/github/label-cursor-prs.test.mjs- name: merge-queue retry helper testsrun: bash scripts/ci/retry-on-merge-queue.test.sh+ - name: vitest CI reporter tests+ run: node --test scripts/vitest/ci-reporter.test.mjs- name: Load default envrun: |cp .env.dev.example .envdiff --git a/scripts/vitest/ci-reporter.test.mjs b/scripts/vitest/ci-reporter.test.mjsnew file mode 100644index 0000000000..9952dae0bb--- /dev/null+++ b/scripts/vitest/ci-reporter.test.mjs@@ -0,0 +1,196 @@+import assert from "node:assert/strict";+import test from "node:test";++import { VitestCiReporter } from "./ci-reporter.ts";++const testCase = ({+ file,+ name,+ duration,+ state = "passed",+ retryCount = 0,+ flaky = false,+ diagnostic = true,+}) => ({+ fullName: name,+ module: { moduleId: `/abs/${file}`, relativeModuleId: file },+ diagnostic: () => (diagnostic ? { duration, flaky, retryCount } : undefined),+ result: () => ({ state }),+});++/** Runs the reporter over one module of test cases and captures its stdout. */+const report = (cases) => {+ const lines = [];+ const original = console.log;+ console.log = (line) => lines.push(String(line));++ try {+ new VitestCiReporter().onTestRunEnd([+ { children: { allTests: () => cases } },+ ]);+ } finally {+ console.log = original;+ }++ return lines.join("\n");+};++/** The lines of one `Header (N):`-style block, header excluded. */+const block = (output, header) => {+ const body = output.split(header)[1];+ if (body === undefined) return undefined;++ const lines = [];+ for (const line of body.split("\n").slice(1)) {+ // Blocks are separated by a blank line, so the first non-ranked line ends+ // this one β without it, every later block would leak into the result.+ if (!/^\s*\d+\./.test(line)) break;+ lines.push(line);+ }++ return lines;+};++test("lists a failed test that no other block would have shown", () => {+ // The langfuse#18741 case: a suite opting out with `describe(..., {retry: 0})`+ // fails with retryCount 0, so it reaches neither the retried block nor β+ // being fast β the slowest ten.+ const slowPassing = Array.from({ length: 12 }, (_, index) =>+ testCase({+ file: "src/slow.test.ts",+ name: `slow ${index}`,+ duration: 10_000 + index,+ }),+ );+ const output = report([+ ...slowPassing,+ testCase({+ // The reporter prints vitest's relativeModuleId, which is relative to+ // the package the suite runs from β never package-prefixed.+ file: "src/__tests__/nativeOtelMedia.unit.test.ts",+ name: "native media > compares generated media spellings",+ duration: 12,+ state: "failed",+ }),+ ]);++ assert.deepEqual(block(output, "\nFailed tests (1):"), [+ "1. retries=0 src/__tests__/nativeOtelMedia.unit.test.ts > native media > compares generated media spellings",+ ]);+ assert.ok(+ !output.includes("Retried tests"),+ "a failure that never retried must not fabricate a retried block",+ );+ assert.ok(+ !block(output, "\nSlowest tests (top 10):").some((line) =>+ line.includes("nativeOtelMedia"),+ ),+ "the fixture must keep the failure out of the slowest ten",+ );+});++test("separates an exhausted-retry flake from a deterministic failure", () => {+ const output = report([+ testCase({+ file: "src/a.servertest.ts",+ name: "flaked until it ran out of attempts",+ duration: 500,+ state: "failed",+ retryCount: 3,+ }),+ testCase({+ file: "src/b.servertest.ts",+ name: "broke on every attempt it was given",+ duration: 400,+ state: "failed",+ }),+ testCase({+ file: "src/c.servertest.ts",+ name: "recovered on its second attempt",+ duration: 300,+ retryCount: 1,+ flaky: true,+ }),+ ]);++ // Highest retry count first, so whatever was given the most attempts leads.+ assert.deepEqual(block(output, "\nFailed tests (2):"), [+ "1. retries=3 src/a.servertest.ts > flaked until it ran out of attempts",+ "2. retries=0 src/b.servertest.ts > broke on every attempt it was given",+ ]);+ // The retried block keeps its own membership rule: retried, pass or fail.+ assert.deepEqual(block(output, "\nRetried tests (2):"), [+ "1. retries=3 src/a.servertest.ts > flaked until it ran out of attempts [failed]",+ "2. retries=1 src/c.servertest.ts > recovered on its second attempt [flaky]",+ ]);+});++test("caps the list but keeps the true total in the header", () => {+ const output = report(+ Array.from({ length: 63 }, (_, index) =>+ testCase({+ file: "src/mass.servertest.ts",+ name: `case ${index}`,+ duration: 5,+ state: "failed",+ }),+ ),+ );++ // The header is the count the analyst reads, so it must stay honest even+ // though the list below it is bounded.+ const listed = block(output, "\nFailed tests (63):");+ assert.equal(listed.length, 50);+ assert.ok(+ output.includes("\nβ¦ and 13 more"),+ "truncation must be explicit, never silent",+ );+});++test("omits the block when nothing failed", () => {+ const output = report([+ testCase({ file: "src/a.servertest.ts", name: "passes", duration: 10 }),+ testCase({+ file: "src/b.servertest.ts",+ name: "passes after a retry",+ duration: 20,+ retryCount: 2,+ flaky: true,+ }),+ ]);++ assert.ok(output.includes("Slowest tests (top 10):"));+ assert.ok(output.includes("Retried tests (1):"));+ assert.ok(+ !output.includes("Failed tests"),+ "a green run's log must look exactly as it did before",+ );+});++test("ignores tests that never produced a result", () => {+ const output = report([+ testCase({+ file: "src/a.servertest.ts",+ name: "passes",+ duration: 10,+ }),+ testCase({+ file: "src/b.servertest.ts",+ name: "skipped",+ duration: 0,+ state: "skipped",+ }),+ testCase({+ file: "src/c.servertest.ts",+ name: "never ran",+ duration: 0,+ state: "failed",+ diagnostic: false,+ }),+ ]);++ assert.ok(+ !output.includes("Failed tests"),+ "skipped tests and tests without a diagnostic are not failures",+ );+});diff --git a/scripts/vitest/ci-reporter.ts b/scripts/vitest/ci-reporter.tsindex bdbb8ca569..7262f1e9ab 100644--- a/scripts/vitest/ci-reporter.ts+++ b/scripts/vitest/ci-reporter.ts@@ -1,5 +1,7 @@const SLOW_TEST_LIMIT = 10;const SLOW_FILE_LIMIT = 10;+/** Caps the Failed tests list so the earlier blocks survive a log tail */+const FAILED_TEST_LIMIT = 50;type TestCaseLike = {fullName: string;@@ -166,5 +168,34 @@ export class VitestCiReporter {);});}++ printFailedTests(completedTests);+ }+}++/** Lists every non-passed test case; absent on a green run */+function printFailedTests(completedTests: TestResult[]): void {+ const failedTests = completedTests.filter((test) => test.state !== "passed");++ if (failedTests.length === 0) {+ return;+ }++ const listed = [...failedTests]+ .sort((left, right) => right.retryCount - left.retryCount)+ .slice(0, FAILED_TEST_LIMIT);+ const failedRankWidth = String(listed.length).length;++ console.log(`\nFailed tests (${failedTests.length}):`);+ listed.forEach((test, index) => {+ const rank = String(index + 1).padStart(failedRankWidth, " ");++ console.log(+ `${rank}. retries=${test.retryCount} ${test.file} > ${test.name}`,+ );+ });++ if (failedTests.length > listed.length) {+ console.log(`β¦ and ${failedTests.length - listed.length} more`);}}