The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
/**
* A verdict counts only at the pull request head it read (Dee, task #3394): a newer head stops it counting unless the pull
* request's own patch is the same (a plain rebase, or a main merge that leaves the PR's diff unchanged).
*/
import { describe, expect, it } from 'vitest'
import { HEADS_KEPT, checkedPullRequest, deriveCheck, keepHead, recordCheckVerdict, refreshChecks, verdictCounts, watchedLinks, type PullRequestHeads } from './checkVerdicts.js'
import type { Check, CheckVerdict } from './model.js'
const PR = 'https://github.com/acme/app/pull/7'
const A = 'aaaaaaa1111111111111111111111111111111aa', B = 'bbbbbbb2222222222222222222222222222222bb'
const check = (): Check => ({ id: 'c', description: 'Signs in offline', independent: false, evidence: null, passedBy: null })
const verdict = (by: string, passed: boolean, head: string | undefined, patch?: string): CheckVerdict =>
({ by, passed, evidence: `${by} read it`, at: 1, ...head ? { pullRequest: PR, head } : {}, ...patch ? { patch } : {} })
const at = (head: string, patch: string | null): PullRequestHeads => ({ [PR]: { head, patch, at: 1 } })
describe('a verdict at a head', () => {
it('counts at its head, in either case', () => {
expect(verdictCounts(verdict('r', true, A), at(A, 'p1'))).toBe(true)
expect(verdictCounts(verdict('r', true, A.toUpperCase()), at(A, 'p1'))).toBe(true)
})
// Security review of #3218 (Warden2), Important 2: a commit ground to share a short head's characters kept the verdict.
it('never counts a shortened head at a head that merely starts with it', () => {
const ground = `${A.slice(0, 7)}${'f'.repeat(33)}`
expect(verdictCounts(verdict('r', true, A.slice(0, 7), 'p1'), at(ground, 'p2'))).toBe(false)
expect(verdictCounts(verdict('r', true, A.slice(0, 7)), at(A, 'p1'))).toBe(false)
})
it('stops counting at a newer head with another patch, or a patch nobody could read', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, 'p2'))).toBe(false)
expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, null))).toBe(false)
expect(verdictCounts(verdict('r', true, A), at(B, 'p1'))).toBe(false)
})
it('keeps counting at a newer head with the same patch', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, 'p1'))).toBe(true)
})
it('counts while the task has no watched head', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), undefined)).toBe(true)
expect(verdictCounts(verdict('r', true, undefined), undefined)).toBe(true)
})
// Review of #3218 (Ledger, round 3), Important 2: a task whose watched heads all left the watch (its links moved past them)
// was watched: its verdicts stay unverified, never "as before".
it('does not count once the task was watched, though it keeps no head now', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), {})).toBe(false)
expect(verdictCounts(verdict('r', true, undefined), {})).toBe(false)
})
// Review of #3218 (Ledger), Important 2 and 6: once the task's pull requests are watched, a verdict whose own one has no head
// (not read yet, or dropped from the kept heads) is unverified, never valid for every head.
it('does not count on a pull request with no watched head while another of the task has one', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), { 'https://github.com/acme/app/pull/8': { head: A, patch: 'p1', at: 1 } })).toBe(false)
})
it('naming a head before the task linked a pull request, counts only at that head once one is watched', () => {
const early: CheckVerdict = { by: 'r', passed: true, evidence: '', at: 1, head: A }
expect(verdictCounts(early, undefined)).toBe(true)
expect(verdictCounts(early, at(A, 'p1'))).toBe(true)
expect(verdictCounts(early, at(B, 'p2'))).toBe(false)
})
// Security review of #3218 (Warden2), Important 3: a verdict naming no head was given the head watched last, which a push
// during the review made code nobody read; and one from before heads counted at every head.
it('naming no head, stops counting once a head is watched', () => {
expect(verdictCounts(verdict('r', true, undefined), at(B, 'p2'))).toBe(false)
expect(verdictCounts({ by: 'r', passed: true, evidence: '', at: 1, pullRequest: PR }, at(B, 'p2'))).toBe(false)
})
})
describe('a check of verdicts at heads', () => {
it('passes on a pass at the head, and reads not passed once that head is replaced', () => {
const c = check()
recordCheckVerdict(c, verdict('agent:r', true, A, 'p1'), undefined, at(A, 'p1'))
expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
deriveCheck(c, at(B, 'p2'))
expect(c).toMatchObject({ passedBy: null, evidence: null })
expect(c.verdicts![0]!.stale).toBe(B)
// Back at the head it read (a force-push back), it counts again.
deriveCheck(c, at(A, 'p1'))
expect(c.passedBy).toBe('agent:r')
expect(c.verdicts![0]!.stale).toBeUndefined()
})
it('a failure at an old head no longer fails the check; a pass at the new head passes it', () => {
const c = check()
recordCheckVerdict(c, verdict('agent:t', false, A, 'p1'), undefined, at(A, 'p1'))
expect(c).toMatchObject({ passedBy: null, evidence: 'agent:t read it' })
recordCheckVerdict(c, verdict('agent:r', true, B, 'p2'), undefined, at(B, 'p2'))
expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
expect(c.verdicts!.map(v => v.stale)).toEqual([B, undefined])
})
it('a failure at the head still wins over a pass', () => {
const c = check()
recordCheckVerdict(c, verdict('agent:t', false, B, 'p2'), undefined, at(B, 'p2'))
recordCheckVerdict(c, verdict('agent:r', true, B, 'p2'), undefined, at(B, 'p2'))
expect(c).toMatchObject({ passedBy: null, evidence: 'agent:t read it' })
})
it('verdicts with no head behave as they did while no head is watched, and are stale at the first', () => {
const c = check()
recordCheckVerdict(c, verdict('agent:r', true, undefined))
expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
expect(c.verdicts![0]).not.toHaveProperty('stale')
expect(refreshChecks([c], PR, undefined, at(A, 'p1'))).toEqual([c])
expect(c.verdicts![0]!.stale).toBe(A)
})
})
describe('a new head', () => {
it('learns the patch of verdicts given before it was read, then keeps those with the same patch', () => {
const c = check()
// Recorded before the watch had read A's patch.
recordCheckVerdict(c, verdict('agent:r', true, A), undefined, at(A, null))
const changed = refreshChecks([c], PR, { head: A, patch: 'p1', at: 1 }, at(B, 'p1'))
expect(c.verdicts![0]!.patch).toBe('p1')
expect(c.passedBy).toBe('agent:r')
expect(changed).toEqual([])
})
it('names the checks that stopped passing', () => {
const c = check(), other = { ...check(), id: 'd' }
recordCheckVerdict(c, verdict('agent:r', true, A, 'p1'), undefined, at(A, 'p1'))
const changed = refreshChecks([c, other], PR, { head: A, patch: 'p1', at: 1 }, at(B, 'p2'))
expect(changed).toEqual([c])
expect(c.passedBy).toBeNull()
})
})
// Review of #3218 (Ledger), Important 6: a task keeps the heads of the pull requests it links, at most HEADS_KEPT of them.
describe('the heads a task keeps', () => {
const head = (n: number) => ({ head: n.toString(16).padStart(40, '0'), patch: null, at: n })
const url = (n: number) => `https://github.com/acme/app/pull/${n}`
it('keeps only pull requests it still links', () => {
expect(Object.keys(keepHead({ [url(1)]: head(1), [url(2)]: head(2) }, url(3), head(3), [url(2), url(3)])).sort()).toEqual([url(2), url(3)])
})
it('keeps at most HEADS_KEPT, the most recently read', () => {
let heads: PullRequestHeads = {}
const links = Array.from({ length: HEADS_KEPT + 5 }, (_, i) => url(i + 1))
for (let n = 1; n <= HEADS_KEPT + 5; n++) heads = keepHead(heads, url(n), head(n), links)
expect(Object.keys(heads)).toHaveLength(HEADS_KEPT)
expect(heads[url(1)]).toBeUndefined()
expect(heads[url(HEADS_KEPT + 5)]).toBeDefined()
})
})
// Review of #3218 (Ledger, round 2), Important 2: the watch read every link and the task kept 50 heads, so past 50 open pull
// requests each pass dropped a head it read again next pass, with no patch: a cascade of unread patches.
describe('the links a task is watched for', () => {
const url = (n: number) => `https://github.com/acme/app/pull/${n}`
it('are its newest HEADS_KEPT links, and its kept heads are theirs', () => {
const links = Array.from({ length: HEADS_KEPT + 3 }, (_, i) => url(i + 1))
expect(watchedLinks(links)).toEqual(links.slice(3))
let heads: PullRequestHeads = {}
for (const link of watchedLinks(links)) heads = keepHead(heads, link, { head: 'a'.repeat(40), patch: 'p', at: 1 }, links)
expect(Object.keys(heads).sort()).toEqual(watchedLinks(links).sort())
// A pass reading the same links again keeps every one.
for (const link of watchedLinks(links)) heads = keepHead(heads, link, heads[link]!, links)
expect(Object.keys(heads)).toHaveLength(HEADS_KEPT)
})
})
// Security reviews of #3218 (Warden2, rounds 4 and 5) and review round 7 (Ledger): a reviewer reads the pull request's diff
// against a base branch, and no base the server infers (the one watched when the verdict came, or the one its head was first
// seen at) is known to be the one this reviewer read: a round trip of retargets defeats each. The checker names it, as it names
// the head; once a head with its base is watched, a verdict naming none counts at none.
describe('a verdict and the base branch it read against', () => {
const on = (head: string, patch: string | null, base: string): PullRequestHeads => ({ [PR]: { head, patch, at: 1, base } })
it('counts at its head only on the base it names', () => {
const c = check()
recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'evil' }, undefined, on(A, 'p-evil', 'evil'))
expect(c.verdicts![0]).toMatchObject({ head: A, patch: 'p-evil', base: 'evil' })
expect(c.passedBy).toBe('agent:r')
refreshChecks([c], PR, { head: A, patch: 'p-evil', at: 1, base: 'evil' }, on(A, 'p-main', 'main'))
expect(c.passedBy).toBeNull()
expect(c.verdicts![0]!.stale).toBe(A)
refreshChecks([c], PR, { head: A, patch: 'p-main', at: 1, base: 'main' }, on(A, 'p-evil', 'evil'))
expect(c.passedBy).toBe('agent:r')
})
it('naming no base, counts at no watched base, and learns none', () => {
const c = check()
recordCheckVerdict(c, verdict('agent:r', true, A), undefined, on(A, 'p-main', 'main'))
expect(c.verdicts![0]).not.toHaveProperty('base')
expect(c.verdicts![0]).not.toHaveProperty('patch')
expect(c.passedBy).toBeNull()
})
it('learns no patch read against another base', () => {
const c = check()
recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'evil' }, undefined, on(A, 'p-main', 'main'))
expect(c.verdicts![0]).not.toHaveProperty('patch')
expect(c.passedBy).toBeNull()
})
it('carries to a new head on its own base with the same patch', () => {
const c = check()
recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'main' }, undefined, on(A, 'p1', 'main'))
refreshChecks([c], PR, { head: A, patch: 'p1', at: 1, base: 'main' }, on(B, 'p1', 'main'))
expect(c.passedBy).toBe('agent:r')
})
})
// Task #3553: a head that is git's own merge of the reviewed head and main (mergeCarry.ts) carries its verdicts, on the same
// base only, and through a chain of such merges; any other head does not.
describe('a verdict at a head a clean main merge carries', () => {
const C = 'ccccccc3333333333333333333333333333333cc', D = 'ddddddd4444444444444444444444444444444dd'
const on = (head: string, carries: string[] | undefined, base = 'main'): PullRequestHeads => ({ [PR]: { head, patch: 'p9', at: 1, base, ...carries ? { carries } : {} } })
const read = (head: string, base = 'main'): CheckVerdict => ({ ...verdict('r', true, head, 'p1'), base })
it('counts at a head that carries its own', () => {
expect(verdictCounts(read(A), on(B, [A]))).toBe(true)
// Through a chain: D merged main into C, which merged main into A.
expect(verdictCounts(read(A), on(D, [C, A]))).toBe(true)
})
it('counts only on its own base, and never for a head that carries another', () => {
expect(verdictCounts(read(A, 'release'), on(B, [A]))).toBe(false)
expect(verdictCounts(read(A), on(B, [C]))).toBe(false)
expect(verdictCounts(read(A), on(B, undefined))).toBe(false)
// Never a shortened head, as at its own head.
expect(verdictCounts(read(A.slice(0, 7)), on(B, [A]))).toBe(false)
})
})
// Task #3953: the pull request a task's checks count at is its own, not simply its newest link.
describe('checkedPullRequest', () => {
const OWN = 'https://github.com/acme/app/pull/20', OTHER = 'https://github.com/acme/app/pull/19'
const read = (extra: Partial<PullRequestHeads[string]> = {}) => ({ head: A, patch: 'p', at: 1, base: 'main', ...extra })
const task = (over: Partial<Parameters<typeof checkedPullRequest>[0]> = {}) => ({ pullRequestUrls: [OWN, OTHER], pullRequestHeads: { [OWN]: read(), [OTHER]: read({ head: B }) }, ...over })
const repository = 'https://github.com/acme/app'
const said = (result: string) => ({ assignment: { result } as never })
it('prefers the pull request its Engineer\'s latest status names over a newer link', () => {
expect(checkedPullRequest(task(said(`Ready: ${OWN} at aaaaaaa`)), repository)).toBe(OWN)
})
it('then the code list, then the newest open link', () => {
expect(checkedPullRequest(task({ codePullRequestUrls: [OWN] }), repository)).toBe(OWN)
expect(checkedPullRequest(task(), repository)).toBe(OTHER)
})
it('never picks a closed pull request while another is open, even when its Engineer names it', () => {
const heads = { [OWN]: read(), [OTHER]: read({ closed: true as const }) }
expect(checkedPullRequest(task({ pullRequestHeads: heads, ...said(`Ready: ${OTHER}`) }), repository)).toBe(OWN)
expect(checkedPullRequest(task({ pullRequestHeads: heads }), repository)).toBe(OWN)
})
it('keeps the newest link when every link is closed or none is read', () => {
const closed = { [OWN]: read({ closed: true as const }), [OTHER]: read({ closed: true as const }) }
expect(checkedPullRequest(task({ pullRequestHeads: closed }), repository)).toBe(OTHER)
expect(checkedPullRequest(task({ pullRequestHeads: undefined }), repository)).toBe(OTHER)
expect(checkedPullRequest({}, repository)).toBeUndefined()
})
})
// Task #3953 (Hugo's review of #3435): a Yes recorded at a pull request later closed unmerged stops counting once another
// linked pull request is open; a merged one is never marked closed, and a task with nothing else open keeps its Yes.
describe('a verdict at a pull request closed unmerged', () => {
const NEXT = 'https://github.com/acme/app/pull/8'
const heads = (closed: boolean, next = true): PullRequestHeads => ({ [PR]: { head: A, patch: 'p1', at: 1, ...closed ? { closed: true as const } : {} },
...next ? { [NEXT]: { head: B, patch: 'p2', at: 2 } } : {} })
it('stops counting while another linked pull request is open, and the check is no longer passed', () => {
const c = check()
recordCheckVerdict(c, verdict('r', true, A, 'p1'), undefined, heads(false))
expect(c.passedBy).toBe('r')
deriveCheck(c, heads(true))
expect(c.passedBy).toBeNull()
expect(verdictCounts(verdict('r', true, A, 'p1'), heads(true))).toBe(false)
})
it('counts again once the pull request is open again', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), heads(false))).toBe(true)
})
it('keeps counting when no other pull request is open', () => {
expect(verdictCounts(verdict('r', true, A, 'p1'), heads(true, false))).toBe(true)
const bothClosed: PullRequestHeads = { [PR]: { head: A, patch: 'p1', at: 1, closed: true }, [NEXT]: { head: B, patch: 'p2', at: 2, closed: true } }
expect(verdictCounts(verdict('r', true, A, 'p1'), bothClosed)).toBe(true)
})
})