Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

15.8 KB
Raw Blame
/**
 * A verdict counts only at the pull request head it read (Dee, task #3394): a newer head stops it counting unless the pull
 * request's own patch is the same (a plain rebase, or a main merge that leaves the PR's diff unchanged).
 */
import { describe, expect, it } from 'vitest'
import { HEADS_KEPT, checkedPullRequest, deriveCheck, keepHead, recordCheckVerdict, refreshChecks, verdictCounts, watchedLinks, type PullRequestHeads } from './checkVerdicts.js'
import type { Check, CheckVerdict } from './model.js'

const PR = 'https://github.com/acme/app/pull/7'
const A = 'aaaaaaa1111111111111111111111111111111aa', B = 'bbbbbbb2222222222222222222222222222222bb'
const check = (): Check => ({ id: 'c', description: 'Signs in offline', independent: false, evidence: null, passedBy: null })
const verdict = (by: string, passed: boolean, head: string | undefined, patch?: string): CheckVerdict =>
  ({ by, passed, evidence: `${by} read it`, at: 1, ...head ? { pullRequest: PR, head } : {}, ...patch ? { patch } : {} })
const at = (head: string, patch: string | null): PullRequestHeads => ({ [PR]: { head, patch, at: 1 } })

describe('a verdict at a head', () => {
  it('counts at its head, in either case', () => {
    expect(verdictCounts(verdict('r', true, A), at(A, 'p1'))).toBe(true)
    expect(verdictCounts(verdict('r', true, A.toUpperCase()), at(A, 'p1'))).toBe(true)
  })

  // Security review of #3218 (Warden2), Important 2: a commit ground to share a short head's characters kept the verdict.
  it('never counts a shortened head at a head that merely starts with it', () => {
    const ground = `${A.slice(0, 7)}${'f'.repeat(33)}`
    expect(verdictCounts(verdict('r', true, A.slice(0, 7), 'p1'), at(ground, 'p2'))).toBe(false)
    expect(verdictCounts(verdict('r', true, A.slice(0, 7)), at(A, 'p1'))).toBe(false)
  })

  it('stops counting at a newer head with another patch, or a patch nobody could read', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, 'p2'))).toBe(false)
    expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, null))).toBe(false)
    expect(verdictCounts(verdict('r', true, A), at(B, 'p1'))).toBe(false)
  })

  it('keeps counting at a newer head with the same patch', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), at(B, 'p1'))).toBe(true)
  })

  it('counts while the task has no watched head', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), undefined)).toBe(true)
    expect(verdictCounts(verdict('r', true, undefined), undefined)).toBe(true)
  })

  // Review of #3218 (Ledger, round 3), Important 2: a task whose watched heads all left the watch (its links moved past them)
  // was watched: its verdicts stay unverified, never "as before".
  it('does not count once the task was watched, though it keeps no head now', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), {})).toBe(false)
    expect(verdictCounts(verdict('r', true, undefined), {})).toBe(false)
  })

  // Review of #3218 (Ledger), Important 2 and 6: once the task's pull requests are watched, a verdict whose own one has no head
  // (not read yet, or dropped from the kept heads) is unverified, never valid for every head.
  it('does not count on a pull request with no watched head while another of the task has one', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), { 'https://github.com/acme/app/pull/8': { head: A, patch: 'p1', at: 1 } })).toBe(false)
  })

  it('naming a head before the task linked a pull request, counts only at that head once one is watched', () => {
    const early: CheckVerdict = { by: 'r', passed: true, evidence: '', at: 1, head: A }
    expect(verdictCounts(early, undefined)).toBe(true)
    expect(verdictCounts(early, at(A, 'p1'))).toBe(true)
    expect(verdictCounts(early, at(B, 'p2'))).toBe(false)
  })

  // Security review of #3218 (Warden2), Important 3: a verdict naming no head was given the head watched last, which a push
  // during the review made code nobody read; and one from before heads counted at every head.
  it('naming no head, stops counting once a head is watched', () => {
    expect(verdictCounts(verdict('r', true, undefined), at(B, 'p2'))).toBe(false)
    expect(verdictCounts({ by: 'r', passed: true, evidence: '', at: 1, pullRequest: PR }, at(B, 'p2'))).toBe(false)
  })
})

describe('a check of verdicts at heads', () => {
  it('passes on a pass at the head, and reads not passed once that head is replaced', () => {
    const c = check()
    recordCheckVerdict(c, verdict('agent:r', true, A, 'p1'), undefined, at(A, 'p1'))
    expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
    deriveCheck(c, at(B, 'p2'))
    expect(c).toMatchObject({ passedBy: null, evidence: null })
    expect(c.verdicts![0]!.stale).toBe(B)
    // Back at the head it read (a force-push back), it counts again.
    deriveCheck(c, at(A, 'p1'))
    expect(c.passedBy).toBe('agent:r')
    expect(c.verdicts![0]!.stale).toBeUndefined()
  })

  it('a failure at an old head no longer fails the check; a pass at the new head passes it', () => {
    const c = check()
    recordCheckVerdict(c, verdict('agent:t', false, A, 'p1'), undefined, at(A, 'p1'))
    expect(c).toMatchObject({ passedBy: null, evidence: 'agent:t read it' })
    recordCheckVerdict(c, verdict('agent:r', true, B, 'p2'), undefined, at(B, 'p2'))
    expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
    expect(c.verdicts!.map(v => v.stale)).toEqual([B, undefined])
  })

  it('a failure at the head still wins over a pass', () => {
    const c = check()
    recordCheckVerdict(c, verdict('agent:t', false, B, 'p2'), undefined, at(B, 'p2'))
    recordCheckVerdict(c, verdict('agent:r', true, B, 'p2'), undefined, at(B, 'p2'))
    expect(c).toMatchObject({ passedBy: null, evidence: 'agent:t read it' })
  })

  it('verdicts with no head behave as they did while no head is watched, and are stale at the first', () => {
    const c = check()
    recordCheckVerdict(c, verdict('agent:r', true, undefined))
    expect(c).toMatchObject({ passedBy: 'agent:r', evidence: 'agent:r read it' })
    expect(c.verdicts![0]).not.toHaveProperty('stale')
    expect(refreshChecks([c], PR, undefined, at(A, 'p1'))).toEqual([c])
    expect(c.verdicts![0]!.stale).toBe(A)
  })
})

describe('a new head', () => {
  it('learns the patch of verdicts given before it was read, then keeps those with the same patch', () => {
    const c = check()
    // Recorded before the watch had read A's patch.
    recordCheckVerdict(c, verdict('agent:r', true, A), undefined, at(A, null))
    const changed = refreshChecks([c], PR, { head: A, patch: 'p1', at: 1 }, at(B, 'p1'))
    expect(c.verdicts![0]!.patch).toBe('p1')
    expect(c.passedBy).toBe('agent:r')
    expect(changed).toEqual([])
  })

  it('names the checks that stopped passing', () => {
    const c = check(), other = { ...check(), id: 'd' }
    recordCheckVerdict(c, verdict('agent:r', true, A, 'p1'), undefined, at(A, 'p1'))
    const changed = refreshChecks([c, other], PR, { head: A, patch: 'p1', at: 1 }, at(B, 'p2'))
    expect(changed).toEqual([c])
    expect(c.passedBy).toBeNull()
  })
})

// Review of #3218 (Ledger), Important 6: a task keeps the heads of the pull requests it links, at most HEADS_KEPT of them.
describe('the heads a task keeps', () => {
  const head = (n: number) => ({ head: n.toString(16).padStart(40, '0'), patch: null, at: n })
  const url = (n: number) => `https://github.com/acme/app/pull/${n}`

  it('keeps only pull requests it still links', () => {
    expect(Object.keys(keepHead({ [url(1)]: head(1), [url(2)]: head(2) }, url(3), head(3), [url(2), url(3)])).sort()).toEqual([url(2), url(3)])
  })

  it('keeps at most HEADS_KEPT, the most recently read', () => {
    let heads: PullRequestHeads = {}
    const links = Array.from({ length: HEADS_KEPT + 5 }, (_, i) => url(i + 1))
    for (let n = 1; n <= HEADS_KEPT + 5; n++) heads = keepHead(heads, url(n), head(n), links)
    expect(Object.keys(heads)).toHaveLength(HEADS_KEPT)
    expect(heads[url(1)]).toBeUndefined()
    expect(heads[url(HEADS_KEPT + 5)]).toBeDefined()
  })
})

// Review of #3218 (Ledger, round 2), Important 2: the watch read every link and the task kept 50 heads, so past 50 open pull
// requests each pass dropped a head it read again next pass, with no patch: a cascade of unread patches.
describe('the links a task is watched for', () => {
  const url = (n: number) => `https://github.com/acme/app/pull/${n}`
  it('are its newest HEADS_KEPT links, and its kept heads are theirs', () => {
    const links = Array.from({ length: HEADS_KEPT + 3 }, (_, i) => url(i + 1))
    expect(watchedLinks(links)).toEqual(links.slice(3))
    let heads: PullRequestHeads = {}
    for (const link of watchedLinks(links)) heads = keepHead(heads, link, { head: 'a'.repeat(40), patch: 'p', at: 1 }, links)
    expect(Object.keys(heads).sort()).toEqual(watchedLinks(links).sort())
    // A pass reading the same links again keeps every one.
    for (const link of watchedLinks(links)) heads = keepHead(heads, link, heads[link]!, links)
    expect(Object.keys(heads)).toHaveLength(HEADS_KEPT)
  })
})

// Security reviews of #3218 (Warden2, rounds 4 and 5) and review round 7 (Ledger): a reviewer reads the pull request's diff
// against a base branch, and no base the server infers (the one watched when the verdict came, or the one its head was first
// seen at) is known to be the one this reviewer read: a round trip of retargets defeats each. The checker names it, as it names
// the head; once a head with its base is watched, a verdict naming none counts at none.
describe('a verdict and the base branch it read against', () => {
  const on = (head: string, patch: string | null, base: string): PullRequestHeads => ({ [PR]: { head, patch, at: 1, base } })

  it('counts at its head only on the base it names', () => {
    const c = check()
    recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'evil' }, undefined, on(A, 'p-evil', 'evil'))
    expect(c.verdicts![0]).toMatchObject({ head: A, patch: 'p-evil', base: 'evil' })
    expect(c.passedBy).toBe('agent:r')
    refreshChecks([c], PR, { head: A, patch: 'p-evil', at: 1, base: 'evil' }, on(A, 'p-main', 'main'))
    expect(c.passedBy).toBeNull()
    expect(c.verdicts![0]!.stale).toBe(A)
    refreshChecks([c], PR, { head: A, patch: 'p-main', at: 1, base: 'main' }, on(A, 'p-evil', 'evil'))
    expect(c.passedBy).toBe('agent:r')
  })

  it('naming no base, counts at no watched base, and learns none', () => {
    const c = check()
    recordCheckVerdict(c, verdict('agent:r', true, A), undefined, on(A, 'p-main', 'main'))
    expect(c.verdicts![0]).not.toHaveProperty('base')
    expect(c.verdicts![0]).not.toHaveProperty('patch')
    expect(c.passedBy).toBeNull()
  })

  it('learns no patch read against another base', () => {
    const c = check()
    recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'evil' }, undefined, on(A, 'p-main', 'main'))
    expect(c.verdicts![0]).not.toHaveProperty('patch')
    expect(c.passedBy).toBeNull()
  })

  it('carries to a new head on its own base with the same patch', () => {
    const c = check()
    recordCheckVerdict(c, { ...verdict('agent:r', true, A), base: 'main' }, undefined, on(A, 'p1', 'main'))
    refreshChecks([c], PR, { head: A, patch: 'p1', at: 1, base: 'main' }, on(B, 'p1', 'main'))
    expect(c.passedBy).toBe('agent:r')
  })
})

// Task #3553: a head that is git's own merge of the reviewed head and main (mergeCarry.ts) carries its verdicts, on the same
// base only, and through a chain of such merges; any other head does not.
describe('a verdict at a head a clean main merge carries', () => {
  const C = 'ccccccc3333333333333333333333333333333cc', D = 'ddddddd4444444444444444444444444444444dd'
  const on = (head: string, carries: string[] | undefined, base = 'main'): PullRequestHeads => ({ [PR]: { head, patch: 'p9', at: 1, base, ...carries ? { carries } : {} } })
  const read = (head: string, base = 'main'): CheckVerdict => ({ ...verdict('r', true, head, 'p1'), base })

  it('counts at a head that carries its own', () => {
    expect(verdictCounts(read(A), on(B, [A]))).toBe(true)
    // Through a chain: D merged main into C, which merged main into A.
    expect(verdictCounts(read(A), on(D, [C, A]))).toBe(true)
  })

  it('counts only on its own base, and never for a head that carries another', () => {
    expect(verdictCounts(read(A, 'release'), on(B, [A]))).toBe(false)
    expect(verdictCounts(read(A), on(B, [C]))).toBe(false)
    expect(verdictCounts(read(A), on(B, undefined))).toBe(false)
    // Never a shortened head, as at its own head.
    expect(verdictCounts(read(A.slice(0, 7)), on(B, [A]))).toBe(false)
  })
})

// Task #3953: the pull request a task's checks count at is its own, not simply its newest link.
describe('checkedPullRequest', () => {
  const OWN = 'https://github.com/acme/app/pull/20', OTHER = 'https://github.com/acme/app/pull/19'
  const read = (extra: Partial<PullRequestHeads[string]> = {}) => ({ head: A, patch: 'p', at: 1, base: 'main', ...extra })
  const task = (over: Partial<Parameters<typeof checkedPullRequest>[0]> = {}) => ({ pullRequestUrls: [OWN, OTHER], pullRequestHeads: { [OWN]: read(), [OTHER]: read({ head: B }) }, ...over })
  const repository = 'https://github.com/acme/app'
  const said = (result: string) => ({ assignment: { result } as never })

  it('prefers the pull request its Engineer\'s latest status names over a newer link', () => {
    expect(checkedPullRequest(task(said(`Ready: ${OWN} at aaaaaaa`)), repository)).toBe(OWN)
  })
  it('then the code list, then the newest open link', () => {
    expect(checkedPullRequest(task({ codePullRequestUrls: [OWN] }), repository)).toBe(OWN)
    expect(checkedPullRequest(task(), repository)).toBe(OTHER)
  })
  it('never picks a closed pull request while another is open, even when its Engineer names it', () => {
    const heads = { [OWN]: read(), [OTHER]: read({ closed: true as const }) }
    expect(checkedPullRequest(task({ pullRequestHeads: heads, ...said(`Ready: ${OTHER}`) }), repository)).toBe(OWN)
    expect(checkedPullRequest(task({ pullRequestHeads: heads }), repository)).toBe(OWN)
  })
  it('keeps the newest link when every link is closed or none is read', () => {
    const closed = { [OWN]: read({ closed: true as const }), [OTHER]: read({ closed: true as const }) }
    expect(checkedPullRequest(task({ pullRequestHeads: closed }), repository)).toBe(OTHER)
    expect(checkedPullRequest(task({ pullRequestHeads: undefined }), repository)).toBe(OTHER)
    expect(checkedPullRequest({}, repository)).toBeUndefined()
  })
})

// Task #3953 (Hugo's review of #3435): a Yes recorded at a pull request later closed unmerged stops counting once another
// linked pull request is open; a merged one is never marked closed, and a task with nothing else open keeps its Yes.
describe('a verdict at a pull request closed unmerged', () => {
  const NEXT = 'https://github.com/acme/app/pull/8'
  const heads = (closed: boolean, next = true): PullRequestHeads => ({ [PR]: { head: A, patch: 'p1', at: 1, ...closed ? { closed: true as const } : {} },
    ...next ? { [NEXT]: { head: B, patch: 'p2', at: 2 } } : {} })

  it('stops counting while another linked pull request is open, and the check is no longer passed', () => {
    const c = check()
    recordCheckVerdict(c, verdict('r', true, A, 'p1'), undefined, heads(false))
    expect(c.passedBy).toBe('r')
    deriveCheck(c, heads(true))
    expect(c.passedBy).toBeNull()
    expect(verdictCounts(verdict('r', true, A, 'p1'), heads(true))).toBe(false)
  })
  it('counts again once the pull request is open again', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), heads(false))).toBe(true)
  })
  it('keeps counting when no other pull request is open', () => {
    expect(verdictCounts(verdict('r', true, A, 'p1'), heads(true, false))).toBe(true)
    const bothClosed: PullRequestHeads = { [PR]: { head: A, patch: 'p1', at: 1, closed: true }, [NEXT]: { head: B, patch: 'p2', at: 2, closed: true } }
    expect(verdictCounts(verdict('r', true, A, 'p1'), bothClosed)).toBe(true)
  })
})