Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

The merge train starts a fresh batch when main moves (#4379) (#3503)

0xken1568e55cd70eb8
diff --git a/backend/src/superTabs/mergeTrain/fakeTrainGitHub.ts b/backend/src/superTabs/mergeTrain/fakeTrainGitHub.ts
index 7292d9f45..a8de88f7e 100644
--- a/backend/src/superTabs/mergeTrain/fakeTrainGitHub.ts
+++ b/backend/src/superTabs/mergeTrain/fakeTrainGitHub.ts
@@ -33,10 +33,13 @@ export function fakeTrainGitHub() {
nextNumber: 900,
/** Runs before each merge into a batch branch: a test pushes onto the branch here. */
beforeMerge: null as ((branch: string) => void) | null,
+ /** Runs after each read, by name: a test moves main mid-pass here. */
+ afterRead: null as ((name: string) => void) | null,
}
const read = (name: string) => {
state.reads.push(name)
if (state.failing.has(name)) throw new TrainReadError(name, 500)
+ state.afterRead?.(name)
}
const write = (name: string) => {
state.writes.push(name)
diff --git a/backend/src/superTabs/mergeTrain/train.test.ts b/backend/src/superTabs/mergeTrain/train.test.ts
index d581d3041..584905f2c 100644
--- a/backend/src/superTabs/mergeTrain/train.test.ts
+++ b/backend/src/superTabs/mergeTrain/train.test.ts
@@ -247,6 +247,88 @@ describe('the merge train finishes batches', () => {
expect(second!.body).toContain('f'.repeat(12))
})
+ it('builds the fresh batch on the main it reads right before building, not the main the pass began with (#3496, 11-07)', async () => {
+ const { repo, swarm, pass } = setup()
+ swarm.approve(repo.addPull(7, ['backend/src/a.ts']))
+ const moved = 'e'.repeat(40)
+ // A hand merge lands while the pass reads its candidates, after it read main.
+ repo.state.afterRead = name => { if (name === 'files') { repo.state.main = moved; repo.state.afterRead = null } }
+ await pass()
+ const [batch] = repo.batches()
+ expect(batch!.body).toContain(moved.slice(0, 12))
+ repo.state.ci.set(batch!.head, 'success')
+ await pass()
+ expect(repo.state.squashes.map(s => s.number)).toEqual([batch!.number])
+ })
+
+ it('rebuilds a batch closed because main moved on the next pass, while GitHub still reads its members as not worked out', async () => {
+ const { repo, swarm, pass } = setup()
+ const a = repo.addPull(7, ['backend/src/a.ts']), b = repo.addPull(8, ['cli/src/b.ts'])
+ swarm.approve(a); swarm.approve(b)
+ await pass()
+ const first = repo.batches()[0]!
+ // A hand merge moves main mid-batch; GitHub answers mergeable null for every open pull request until it has worked it out.
+ repo.state.main = 'd'.repeat(40)
+ a.mergeable = null; b.mergeable = null
+ await pass()
+ expect(repo.state.pulls.get(first.number)!.state).toBe('closed')
+ const [second] = repo.batches()
+ expect(second!.body).toContain('d'.repeat(12))
+ expect(repo.state.contents.get(second!.head)).toEqual([a.head, b.head])
+ })
+
+ it('leaves out a member that now conflicts with the new main, with its reason, and rebuilds the rest', async () => {
+ const { repo, swarm, pass } = setup()
+ const a = repo.addPull(7, ['backend/src/a.ts']), b = repo.addPull(8, ['cli/src/b.ts'])
+ swarm.approve(a); swarm.approve(b)
+ await pass()
+ repo.state.main = 'd'.repeat(40)
+ a.mergeable = null; b.mergeable = null
+ repo.state.conflicts.add(b.head)
+ await pass()
+ expect(repo.batches().map(p => repo.state.contents.get(p.head))).toEqual([[a.head]])
+ expect(swarm.told.map(t => [t.taskId, t.key])).toEqual([['task-8', `train-conflict:${b.head}`]])
+ })
+
+ it('never merges a docs pull request GitHub has not worked out yet: the fast lane merges on GitHub, with no batch to test it', async () => {
+ const { repo, swarm, pass } = setup()
+ swarm.approve(repo.addPull(7, ['docs/a.md'], { mergeable: null }))
+ await pass()
+ expect(repo.state.squashes).toEqual([])
+ expect(repo.batches()).toEqual([])
+ })
+
+ it('tells a member left waiting after main moved why it waits and when the next batch starts, once', async () => {
+ const { repo, swarm, pass } = setup()
+ const rules: TrainRules = { ...ON, everyMinutes: 10 }
+ const a = repo.addPull(7, ['backend/src/a.ts']), b = repo.addPull(8, ['cli/src/b.ts'])
+ swarm.approve(a); swarm.approve(b)
+ await pass(rules)
+ const first = repo.batches()[0]!
+ repo.state.main = 'd'.repeat(40)
+ b.labels = ['hold']
+ await pass(rules)
+ expect(repo.batches().map(p => repo.state.contents.get(p.head))).toEqual([[a.head]])
+ expect(swarm.told.map(t => [t.taskId, t.key])).toEqual([['task-8', `train-moved:${first.number}:${b.head}`]])
+ expect(swarm.told[0]!.text).toMatch(/main moved/i)
+ expect(swarm.told[0]!.text).toContain('hold label')
+ expect(swarm.told[0]!.text).toContain('in about 10 minutes')
+ await pass(rules)
+ expect(swarm.told).toHaveLength(1)
+ })
+
+ it('says nothing on a member merged by hand meanwhile, often why main moved', async () => {
+ const { repo, swarm, pass } = setup()
+ const a = repo.addPull(7, ['backend/src/a.ts']), b = repo.addPull(8, ['cli/src/b.ts'])
+ swarm.approve(a); swarm.approve(b)
+ await pass()
+ repo.state.main = 'd'.repeat(40)
+ Object.assign(b, { state: 'closed', merged: true })
+ await pass()
+ expect(repo.batches().map(p => repo.state.contents.get(p.head))).toEqual([[a.head]])
+ expect(swarm.told).toEqual([])
+ })
+
it('never merges a member pushed after its batch went green: the heads are read again first', async () => {
const { repo, swarm, pass } = setup()
const a = repo.addPull(7, ['backend/src/a.ts']), b = repo.addPull(8, ['backend/src/b.ts'])
diff --git a/backend/src/superTabs/mergeTrain/train.ts b/backend/src/superTabs/mergeTrain/train.ts
index a13d4795e..1731971a2 100644
--- a/backend/src/superTabs/mergeTrain/train.ts
+++ b/backend/src/superTabs/mergeTrain/train.ts
@@ -38,8 +38,9 @@ interface Batch { number: number; url: string; branch: string; tip: string; base
export interface TrainMemory { batches: Map<number, Batch>; failed: Set<string>; splits: string[][] }
export const newTrainMemory = (): TrainMemory => ({ batches: new Map(), failed: new Set(), splits: [] })
-/** The project's rules record as this pass read it: its switch, and the review table its reads come from (policy.ts). */
-export interface TrainRules { enabled: boolean; rules: ProjectRules }
+/** The project's rules record as this pass read it: its switch, the review table its reads come from (policy.ts), and the
+ * minutes between passes, which the line to a waiting pull request names. */
+export interface TrainRules { enabled: boolean; rules: ProjectRules; everyMinutes?: number }
export type TrainPassLine = { off: true } | { candidates: number; unreadable: number; built: number; merged: number; split: number; closed: number; refused: number; unrecorded: number }
const short = (sha: string) => sha.slice(0, 12)
@@ -48,6 +49,14 @@ const batchLabel = (at: number) => { const d = new Date(at), two = (n: number) =
return `${two(d.getUTCMonth() + 1)}-${two(d.getUTCDate())} train-${two(d.getUTCHours())}${two(d.getUTCMinutes())}` }
const PULL = /\/pull\/([1-9][0-9]*)$/
+/** Why a pull request the train read waits, in plain words, for the line to a member of a batch main moved under. */
+const WAITS: Record<Ineligible, string> = {
+ closed: 'it is closed', draft: 'it is a draft', base: 'it no longer merges into main', hold: 'it has a hold label',
+ ci: 'ci/required is not green at its head', mergeable: 'GitHub has not worked out yet whether it merges cleanly',
+ conflict: 'it conflicts with main', verdict: 'a review or test does not count at its head, or a step is still open',
+ reads: 'it needs more passed reviews at its head',
+}
+
interface Read { member: Member; verdict: { ok: true } | { ok: false; why: Ineligible }; reads: number | null; required: number; lane: 'fast' | 'code' }
export async function trainPass(input: { projectId: string; github: TrainGitHub; swarm: TrainSwarm; memory: TrainMemory; now: number; rules: TrainRules }): Promise<TrainPassLine> {
@@ -68,17 +77,21 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
const taskOf = new Map<string, TrainTaskPull>()
for (const linked of tasks) if (!taskOf.has(linked.url)) taskOf.set(linked.url, linked)
- /** [url] as it is now: GitHub's answer and the Swarm's verdicts. Null when it is not an open pull request of this repository. */
- const read = async (linked: TrainTaskPull): Promise<Read | null> => {
+ /** [url] as it is now: GitHub's answer and the Swarm's verdicts. Null when it is not an open pull request of this repository.
+ * [building]: for a new batch, where the train's own merge into the batch branch tests for a conflict, a code pull request
+ * GitHub has not worked out yet (mergeable null, as it answers for every open one just after main moves) is read as
+ * mergeable; a conflict there leaves it out and tells its Engineer. The fast lane merges on GitHub alone and still waits. */
+ const read = async (linked: TrainTaskPull, building = false): Promise<Read | null> => {
const number = Number(PULL.exec(linked.url)?.[1])
if (!number) return null
const pull = await github.pull(number)
if (pull.state !== 'open' || pull.url.toLowerCase() !== linked.url.toLowerCase() || pull.headRef.startsWith(TRAIN_BRANCH_PREFIX)) return null
const files = await github.files(number), ciRequired = await github.ciRequired(pull.head)
const reads = headReads(linked.task, linked.url, pull.head, pull.base), required = requiredReads(rules.rules, files)
- const verdict = eligible({ url: linked.url, number, head: pull.head, state: pull.state, draft: pull.draft, base: pull.base, mergeable: pull.mergeable,
+ const lane = laneOf(files), mergeable = building && lane === 'code' && pull.mergeable === null ? true : pull.mergeable
+ const verdict = eligible({ url: linked.url, number, head: pull.head, state: pull.state, draft: pull.draft, base: pull.base, mergeable,
labels: pull.labels, files, ciRequired }, reads, required)
- return { member: { number, url: linked.url, head: pull.head, title: pull.title, files, taskId: linked.taskId }, verdict, reads, required, lane: laneOf(files) }
+ return { member: { number, url: linked.url, head: pull.head, title: pull.title, files, taskId: linked.taskId }, verdict, reads, required, lane }
}
const closeBatch = async (number: number, branch: string, why: string) => {
@@ -88,6 +101,9 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
await github.deleteBranch(branch)
line.closed++
}
+ /** The batches this pass closed because main moved under them: their members left waiting hear why, after the build. */
+ const moved: Batch[] = []
+ const closeMoved = async (batch: Batch) => { await closeBatch(batch.number, batch.branch, 'main moved'); moved.push(batch) }
const tellRed = (member: Member) => swarm.tell(member.taskId, member.url, `train-red:${member.head}`,
`CI failed for your pull request #${member.number} in the merge train, on its own with main (head ${member.head.slice(0, 7)}): fix it and push. The train takes it again at its new head. ${member.url}`)
@@ -110,7 +126,7 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
await closeBatch(pull.number, pull.headRef, batch ? 'its branch changed after the train built it' : 'the train did not build it on this server')
continue
}
- if (batch.base !== main) { await closeBatch(batch.number, batch.branch, 'main moved'); continue }
+ if (batch.base !== main) { await closeMoved(batch); continue }
const ci = await github.ciRequired(batch.tip)
if (ci === 'pending' || ci === null) {
if (now - batch.startedAt > TRAIN_CI_TIMEOUT_MS) await closeBatch(batch.number, batch.branch, 'its CI did not finish in time')
@@ -128,7 +144,7 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
continue
}
// Green: read main and every member again, right before merging.
- if (await github.mainHead() !== batch.base) { await closeBatch(batch.number, batch.branch, 'main moved'); continue }
+ if (await github.mainHead() !== batch.base) { await closeMoved(batch); continue }
let still = true
for (const member of batch.members) {
const linked = taskOf.get(member.url), current = linked ? await read(linked) : null
@@ -176,13 +192,17 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
if (memory.batches.size) return line
// Nothing open: read every candidate, merge the fast lane, and build the next batches.
- const reads: Read[] = []
+ // Why each pull request read waits: null where it had its own line this pass (a conflict, its reads, a workflow change), or
+ // where it went into a batch or main.
+ const reads: Read[] = [], waits = new Map<string, string | null>()
for (const linked of taskOf.values()) {
try {
- const found = await read(linked)
+ const found = await read(linked, true)
if (!found || memory.failed.has(`${found.member.url}@${found.member.head}`)) continue
line.candidates++
const { member, verdict } = found
+ waits.set(member.url, verdict.ok ? 'it did not fit the new batch (another pull request in it changes the same files, or it was full)'
+ : verdict.why === 'conflict' || verdict.why === 'reads' ? null : WAITS[verdict.why])
if (!verdict.ok && verdict.why === 'conflict') await tellConflict(swarm, member)
else if (!verdict.ok && verdict.why === 'reads') {
await swarm.tell(member.taskId, member.url, `train-reads:${member.head}`,
@@ -190,6 +210,7 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
} else if (verdict.ok && member.files.some(file => NOT_MERGEABLE_BY_TOKEN.test(file))) {
await swarm.tell(member.taskId, member.url, `train-workflows:${member.head}`,
`Your pull request #${member.number} changes CI workflows, which the merge train cannot merge: merge it by hand. ${member.url}`)
+ waits.set(member.url, null)
} else if (verdict.ok) reads.push(found)
} catch (error) {
if (!(error instanceof TrainReadError)) throw error
@@ -202,9 +223,12 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
const { member } = fast
await github.squash(member.number, member.head, `${member.title} (#${member.number})`, `Merged by the merge train (fast lane).`)
await swarm.merged(member.taskId, member.url, now)
+ waits.set(member.url, null)
line.merged++
}
- if (reads.some(r => r.lane === 'fast')) main = await github.mainHead()
+ // Main again, right before building: reading the candidates takes minutes, and a hand merge meanwhile would leave the batch
+ // on an older main, closed again at the next pass (#3496 on 11-07, built 46 s after main moved).
+ main = await github.mainHead()
} catch (error) {
if (!(error instanceof TrainReadError)) throw error
line.unreadable++
@@ -222,9 +246,9 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
const members: Member[] = []
for (const member of group) {
const merged = await github.mergeInto(branch, member.head, `Merge #${member.number} into the merge train's batch`, tip)
- if (merged === 'conflict') { await tellConflict(swarm, member); continue }
+ if (merged === 'conflict') { await tellConflict(swarm, member); waits.set(member.url, null); continue }
// In main already (merged by hand meanwhile): nothing of it to batch.
- if (merged === 'already') continue
+ if (merged === 'already') { waits.set(member.url, null); continue }
tip = merged.sha
members.push(member)
}
@@ -236,17 +260,33 @@ export async function trainPass(input: { projectId: string; github: TrainGitHub;
// Opened at another head than the tip built: someone pushed onto the branch. Never a batch (review 1 of #3226).
if (opened.head !== tip) { await closeBatch(opened.number, branch, 'its branch changed while the train built it'); continue }
memory.batches.set(opened.number, { number: opened.number, url: opened.url, branch, tip, base: main, startedAt: now, members })
+ for (const member of members) waits.set(member.url, null)
line.built++
} catch (error) {
if (!(error instanceof TrainReadError)) throw error
line.unreadable++
+ for (const member of group) if (waits.get(member.url)) waits.set(member.url, 'GitHub failed while the train built the new batch')
// Whatever it holds, a branch the train could not finish building is never left behind to be opened later.
await github.deleteBranch(branch).catch(() => {})
}
}
+ await tellMoved(swarm, moved, waits, rules.everyMinutes)
return line
}
+/** Tells each member of a batch closed because main moved (task #4379) that the train read again and that went into no new
+ * batch why it waits and when the next batch starts: once per closed batch and head. One merged by hand meanwhile, or one
+ * told by its own line this pass, hears nothing more. */
+async function tellMoved(swarm: TrainSwarm, moved: readonly Batch[], waits: ReadonlyMap<string, string | null>, everyMinutes?: number) {
+ const when = everyMinutes ? `, in about ${everyMinutes} minutes` : ''
+ for (const batch of moved) for (const member of batch.members) {
+ const why = waits.get(member.url)
+ if (!why) continue
+ await swarm.tell(member.taskId, member.url, `train-moved:${batch.number}:${member.head}`,
+ `Main moved under the merge train's batch #${batch.number}, so the train closed it to build again from the new main. Your pull request #${member.number} waits: ${why}. The train builds its next batch at its next pass${when}. ${member.url}`)
+ }
+}
+
/** The first members of [group], up to [TRAIN_MAX_BATCH], that share no file with an earlier one: two pull requests changing
* one file would need a read of their combination, so the later one waits for the next batch and is tested on top of the
* first once it merged (task #3401: a combination read only where files are shared). */
diff --git a/docs/plans/2026-11-07-train-rebuilds-when-main-moves-plan.md b/docs/plans/2026-11-07-train-rebuilds-when-main-moves-plan.md
new file mode 100644
index 000000000..e5104544f
--- /dev/null
+++ b/docs/plans/2026-11-07-train-rebuilds-when-main-moves-plan.md
@@ -0,0 +1,82 @@
+# The merge train starts a fresh batch when main moves: implementation plan
+
+> Written by the project's Engineer way of working (project instructions, Engineer): plan first, then each task test first,
+> the failing test committed before its code. No Execution Handoff.
+
+**Goal:** When main moves under a batch, the train builds a fresh batch from the main it reads right before building, takes
+the same pull requests while GitHub is still working out whether they merge, and tells each one left waiting why and when the
+next batch starts (task #4379).
+
+**Architecture:** `mergeTrain/train.ts` only. A pass reads main again right before it creates a batch branch, so it never
+builds on the main it read at the start of a pass that took minutes. A code-lane candidate whose only problem is GitHub's
+`mergeable: null` (not worked out yet, as GitHub answers for every open pull request just after main moves) goes into the
+batch build, where the train's own merge into the batch branch is the conflict test; a conflict there is left out and told,
+as now. After building, each member of a batch closed this pass because main moved that is in no new batch gets one line on
+its task: the reason it waits and the next pass time.
+
+**Tech Stack:** TypeScript backend, vitest, the fake GitHub of `mergeTrain/fakeTrainGitHub.ts`.
+
+**Spec:** task #4379's brief. Evidence (GitHub, 11-07 UTC): batch #3488 closed "main moved" at 01:56:22, a minute after
+#3484's hand merge (01:55:19), and that pass built nothing; batch #3496 was built at 02:29:42 on 3a393e7, 46 s after #3485 had
+moved main to a89280b (02:28:56), so the next pass closed it again. On 11-06, 20 batches closed "main moved".
+
+## Global Constraints
+
+- No requester (crew reliability, opened by the PM): the PR body says so in its first line.
+- Backend only; owns `backend/src/superTabs/mergeTrain/`.
+- Never push to a `harness-train/` branch by hand. `mergeTrain.enabled` stays the kill switch. One train run at a time per
+ project (unchanged: `startMergeTrain`).
+- Same rules: ready, every step passed at the head, ci/required green, no conflict, no hold. `mergeable: false` still waits;
+ the fast lane (merged straight on GitHub) still needs `mergeable: true`; the green re-read before a squash is unchanged.
+
+## Review Focus
+
+- `mergeable: null` that is a real conflict: the train's merge into the batch branch answers `conflict`; it is left out and
+ its Engineer told, the rest batch.
+- Main moving again while the pass reads candidates: the batch is built on the main read just before `createBranch`.
+- A closed batch's member merged by hand (often why main moved): no longer open, so no line.
+- A member that waits for a reason already told by its own line (conflict, reads, workflows): no second line.
+- Every open batch is built in one pass on one main, so main moving closes them all: none is left open to wait for.
+
+### Task 1: build on the main read just before building
+
+**Files:** Modify `backend/src/superTabs/mergeTrain/train.ts`, `fakeTrainGitHub.ts` (an `afterRead` hook); Test
+`train.test.ts`.
+
+- [ ] Test `builds the fresh batch on the main it reads right before building, not the main the pass began with`: main
+ moves (hook on the first `files` read) after the pass read it; the new batch's body names the new main, and the next pass
+ does not close it.
+- [ ] Run `npx vitest run src/superTabs/mergeTrain/train.test.ts`: FAIL (body names the old main).
+- [ ] Code: `main = await github.mainHead()` before the build loop, inside the fast-lane try, always.
+- [ ] Run again: PASS. Commit.
+
+### Task 2: rebuild while GitHub is still working out mergeability
+
+**Files:** Modify `train.ts`; Test `train.test.ts`.
+
+- [ ] Test `rebuilds a batch closed because main moved on the next pass, while GitHub still reads its members as not worked
+ out`: batch of #7 and #8 in CI, main moves, both `mergeable: null`; the next pass closes it and builds one batch of both
+ on the new main.
+- [ ] Test `leaves out a member that now conflicts, with its reason, and rebuilds the rest`: as above, #8's head conflicts
+ in the merge; the new batch holds #7; #8's task gets `train-conflict:<head>`.
+- [ ] Test: a fast-lane pull request with `mergeable: null` is not merged.
+- [ ] Run: FAIL (no batch built). Commit the tests.
+- [ ] Code: in the candidate loop, a code-lane read whose `why` is `mergeable` joins the batch build.
+- [ ] Run: PASS. Commit.
+
+### Task 3: tell the waiting members of a batch closed because main moved
+
+**Files:** Modify `train.ts` (`TrainRules.everyMinutes?: number`), `run.ts` passes it already (`set`); Test
+`train.test.ts`.
+
+**Interfaces:** `TrainRules { enabled: boolean; rules: ProjectRules; everyMinutes?: number }`. Tell key
+`train-moved:<closed batch number>:<head>`.
+
+- [ ] Test `tells a member left waiting after main moved why it waits and when the next batch starts, once`: batch of #7
+ and #8, main moves, #8 gets a hold label; #8's task gets one line naming the hold and "in about 10 minutes"; #7 (rebuilt)
+ gets none; a second pass adds nothing.
+- [ ] Test: a member merged by hand meanwhile gets no line; a conflicting one gets only its conflict line.
+- [ ] Run: FAIL. Commit the tests.
+- [ ] Code: collect the batches closed for "main moved" this pass; record each candidate's wait reason (none where its own
+ line went); after building, tell each closed member that is open, read, and in no new batch.
+- [ ] Run the mergeTrain tests and `npx tsc --noEmit -p .`: PASS. Commit.