Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

dsh/verdict: cover the watcher event by event

dee35c1f6275b2f
diff --git a/cli/src/dsh/verdict.spec.ts b/cli/src/dsh/verdict.spec.ts
index 707ea5f59..9dd9436d0 100644
--- a/cli/src/dsh/verdict.spec.ts
+++ b/cli/src/dsh/verdict.spec.ts
@@ -1,8 +1,9 @@
-import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import type { EventEmitter } from 'node:events'
import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
-import { DshVerdictWatcher, parseVerdict, type DshVerdict } from './verdict.js'
+import { DshVerdictWatcher, parseVerdict, readVerdictFile, type DshVerdict } from './verdict.js'
describe('parseVerdict', () => {
it('reduces a spec-1 verdict to the wire shape', () => {
@@ -111,3 +112,156 @@ describe('DshVerdictWatcher', () => {
watcher.unwatch('agent-2')
})
})
+
+describe('readVerdictFile and the edges of parseVerdict', () => {
+ it('reads a verdict off disk, or null when there is no file', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'dsh-verdict-file-'))
+ try {
+ writeFileSync(join(dir, 'verdict.json'), JSON.stringify({ spec: 1, ready: true, summary: ' ', findings: 'none', artifact: '' }))
+ expect(readVerdictFile(join(dir, 'verdict.json'))).toEqual({ ready: true, summary: null, errors: 0, warnings: 0, artifact: null, phases: [], updatedAt: null })
+ expect(readVerdictFile(join(dir, 'missing.json'))).toBeNull()
+ } finally {
+ rmSync(dir, { recursive: true, force: true })
+ }
+ })
+
+ it('caps the summary, refuses an overlong artifact, and skips a phase that is a list', () => {
+ const verdict = parseVerdict(JSON.stringify({
+ spec: 1, ready: false, summary: 'x'.repeat(300), artifact: `${'a/'.repeat(600)}b.step`,
+ phases: [['Build'], { id: ' ', name: 'Build Plate!', state: 'failed' }],
+ }))
+ expect(verdict?.summary).toHaveLength(200)
+ expect(verdict?.artifact).toBeNull()
+ expect(verdict?.phases).toEqual([{ id: 'build-plate-', name: 'Build Plate!', state: 'failed', artifact: null }])
+ })
+})
+
+describe('DshVerdictWatcher, event by event', () => {
+ let workspace: string
+ let watcher: DshVerdictWatcher | null = null
+ beforeEach(() => { workspace = mkdtempSync(join(tmpdir(), 'dsh-verdict-events-')) })
+ afterEach(async () => {
+ await watcher?.stop()
+ watcher = null
+ rmSync(workspace, { recursive: true, force: true })
+ })
+
+ const settle = (ms = 400): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms))
+ const verdict = (summary: string): string => JSON.stringify({ spec: 1, ready: false, summary })
+ /** The chokidar instance behind one agent's watch, to raise the error chokidar would. */
+ const innerWatcher = (w: DshVerdictWatcher, agentId: string): EventEmitter =>
+ (w as unknown as { watched: Map<string, { watcher: EventEmitter }> }).watched.get(agentId)!.watcher
+
+ it('publishes a verdict that appears, debounces a burst, ignores other files and unchanged rewrites, and says when it goes', async () => {
+ const file = join(workspace, '.harness', 'verdict.json')
+ const seen: Array<string | null> = []
+ watcher = new DshVerdictWatcher({ onChange: (_id, v) => seen.push(v ? v.summary : null) })
+ watcher.watch('a', file)
+ expect(seen).toEqual([])
+ expect(watcher.current('a')).toBeNull()
+ await settle()
+
+ writeFileSync(file, verdict('first'))
+ await vi.waitFor(() => expect(seen).toEqual(['first']), { timeout: 4_000 })
+ expect(watcher.current('a')?.summary).toBe('first')
+
+ // two changes inside the debounce are one publish, of the last write
+ const inner = innerWatcher(watcher, 'a')
+ /** Do something to the directory and wait until chokidar has told the watcher exactly that. */
+ const observed = async (event: 'add' | 'change' | 'unlink', path: string, act: () => void): Promise<void> => {
+ let listener: (name: string, at: string) => void = () => undefined
+ const landed = new Promise<boolean>((resolve) => {
+ listener = (name, at) => { if (name === event && at === path) resolve(true) }
+ inner.on('all', listener)
+ })
+ // a change within the same millisecond as the last one can read as no change at all
+ await settle(20)
+ act()
+ const seenIt = await Promise.race([landed, settle(3_000).then(() => false)])
+ inner.off('all', listener)
+ expect(seenIt, `${event} ${path}`).toBe(true)
+ }
+ await observed('change', file, () => writeFileSync(file, verdict('draft')))
+ await observed('change', file, () => writeFileSync(file, verdict('second')))
+ await vi.waitFor(() => expect(seen.at(-1)).toBe('second'), { timeout: 4_000 })
+ await settle()
+ expect(seen).toEqual(['first', 'second'])
+
+ // the same bytes again, and a neighbour in .harness/ added, changed and removed, publish nothing
+ await observed('change', file, () => writeFileSync(file, verdict('second')))
+ const neighbour = join(workspace, '.harness', 'other.json')
+ await observed('add', neighbour, () => writeFileSync(neighbour, '{}'))
+ await observed('change', neighbour, () => writeFileSync(neighbour, '{"a":1}'))
+ await observed('unlink', neighbour, () => rmSync(neighbour))
+ await settle(400)
+ expect(seen).toEqual(['first', 'second'])
+
+ rmSync(file)
+ await vi.waitFor(() => expect(seen.at(-1)).toBeNull(), { timeout: 4_000 })
+ expect(watcher.current('a')).toBeNull()
+ }, 15_000)
+
+ it('watching the same file again is a no-op; a different file replaces the watch', async () => {
+ const first = join(workspace, 'one', 'verdict.json')
+ const second = join(workspace, 'two', 'verdict.json')
+ mkdirSync(join(workspace, 'one'))
+ mkdirSync(join(workspace, 'two'))
+ writeFileSync(first, verdict('one'))
+ writeFileSync(second, verdict('two'))
+ const seen: Array<string | null> = []
+ watcher = new DshVerdictWatcher({ onChange: (_id, v) => seen.push(v ? v.summary : null) })
+ watcher.watch('a', first)
+ watcher.watch('a', first)
+ expect(seen).toEqual(['one'])
+ watcher.watch('a', second)
+ expect(seen).toEqual(['one', 'two'])
+ await settle()
+ writeFileSync(first, verdict('one again'))
+ await settle(600)
+ expect(seen).toEqual(['one', 'two'])
+ }, 10_000)
+
+ it('logs, and watches nothing, when the verdict directory cannot be made', () => {
+ writeFileSync(join(workspace, 'plain-file'), '')
+ const logs: string[] = []
+ watcher = new DshVerdictWatcher({ onChange: () => undefined, log: (line) => logs.push(line) })
+ watcher.watch('a', join(workspace, 'plain-file', '.harness', 'verdict.json'))
+ expect(logs).toEqual([expect.stringMatching(/^\[dsh\] verdict dir .*plain-file\/\.harness could not be created · ENOTDIR/)])
+ expect(watcher.current('a')).toBeNull()
+ // and with no log at all, still no throw
+ new DshVerdictWatcher({ onChange: () => undefined }).watch('b', join(workspace, 'plain-file', 'x', 'verdict.json'))
+ })
+
+ it('logs a watch error, whatever was thrown', () => {
+ const logs: string[] = []
+ watcher = new DshVerdictWatcher({ onChange: () => undefined, log: (line) => logs.push(line) })
+ watcher.watch('a', join(workspace, '.harness', 'verdict.json'))
+ innerWatcher(watcher, 'a').emit('error', new Error('EMFILE: too many open files'))
+ innerWatcher(watcher, 'a').emit('error', 'a string')
+ expect(logs).toEqual(['[dsh] verdict watch error · EMFILE: too many open files', '[dsh] verdict watch error · a string'])
+ // with nowhere to log, an error is still not a throw
+ const quiet = new DshVerdictWatcher({ onChange: () => undefined })
+ quiet.watch('q', join(workspace, 'q', 'verdict.json'))
+ expect(() => innerWatcher(quiet, 'q').emit('error', new Error('x'))).not.toThrow()
+ void quiet.stop()
+ })
+
+ it('unwatch and stop cancel a publish still in its debounce', async () => {
+ const file = join(workspace, '.harness', 'verdict.json')
+ const seen: Array<string | null> = []
+ watcher = new DshVerdictWatcher({ onChange: (id, v) => seen.push(`${id}:${v?.summary ?? null}`) })
+ watcher.unwatch('never-watched')
+ watcher.watch('a', file)
+ watcher.watch('b', join(workspace, 'b', '.harness', 'verdict.json'))
+ await settle()
+ // the add events land, then the watches go before their 150ms debounce fires
+ const landed = ['a', 'b'].map((id) => new Promise<void>((resolve) => innerWatcher(watcher!, id).once('add', () => resolve())))
+ writeFileSync(file, verdict('late'))
+ writeFileSync(join(workspace, 'b', '.harness', 'verdict.json'), verdict('late'))
+ await Promise.all(landed)
+ watcher.unwatch('a')
+ await watcher.stop()
+ await settle(400)
+ expect(seen).toEqual([])
+ }, 10_000)
+})
diff --git a/cli/src/dsh/verdict.ts b/cli/src/dsh/verdict.ts
index 370eb5d33..354d51c94 100644
--- a/cli/src/dsh/verdict.ts
+++ b/cli/src/dsh/verdict.ts
@@ -134,7 +134,8 @@ export class DshVerdictWatcher {
try {
mkdirSync(dir, { recursive: true })
} catch (error) {
- this.deps.log?.(`[dsh] verdict dir ${dir} could not be created · ${error instanceof Error ? error.message : error}`)
+ // mkdirSync throws only system errors (a file where a directory should be, no permission).
+ this.deps.log?.(`[dsh] verdict dir ${dir} could not be created · ${(error as Error).message}`)
return
}
const name = basename(file)