Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

13.8 KB
Raw Blame
import { randomUUID } from 'node:crypto'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { Readable } from 'node:stream'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import { PrismaClient } from '@prisma/client'
import type { AuthUser } from '../src/lib/ssoAuth.js'
import { SuperTabStore, type VoiceTranscription } from '../src/superTabs/store.js'
import { MemoryAttachmentStore } from '../src/superTabs/attachments.js'
import { logger } from '../src/utils/logger.js'
import { FIELDS, preview, row } from '../src/superTabs/read/inbox.js'
import { readBy } from '../src/superTabs/parts/indexed.js'

// Chat voice messages (task #2584): the phone uploads a WAV, the server transcribes it once before it is sent, and the words
// become the message's text, so agents read them in the delivery written with the message.
let replica: MongoMemoryReplSet
let db: PrismaClient
const owner: AuthUser = { sub: '230000000000000000000001', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const peer: AuthUser = { sub: '230000000000000000000002', email: '[email protected]', autonomousEnv: 'prod', role: 'user' }
const config = { engine: 'codex', model: 'fixture-model', effort: 'high', contextWindow: 200000, engineVersion: 'fixture' }

beforeAll(async () => {
  replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
    replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
  const uri = replica.getUri(`swarm_voice_messages_test_${randomUUID().replaceAll('-', '_')}`)
  await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
    env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
  })
  db = new PrismaClient({ datasources: { db: { url: uri } } })
  for (const user of [owner, peer]) {
    await db.user.create({ data: { id: user.sub, externalId: `fixture-${user.sub}`, email: user.email, autonomousEnv: user.autonomousEnv } })
  }
  await db.machine.create({ data: { machineId: 'mac', userId: owner.sub, apiKey: 'fixture-mac', managerId: 'fixture', autonomousEnv: 'prod', authMode: 'remote' } })
}, 120_000)
afterAll(async () => {
  await db?.$disconnect()
  await replica?.stop()
})

/** 16 kHz mono 16-bit PCM WAV, [seconds] long, loud. */
function wav(seconds: number): Buffer {
  const samples = Math.round(seconds * 16_000)
  const out = Buffer.alloc(44 + samples * 2)
  out.write('RIFF', 0, 'ascii')
  out.writeUInt32LE(36 + samples * 2, 4)
  out.write('WAVE', 8, 'ascii')
  out.write('fmt ', 12, 'ascii')
  out.writeUInt32LE(16, 16)
  out.writeUInt16LE(1, 20)
  out.writeUInt16LE(1, 22)
  out.writeUInt32LE(16_000, 24)
  out.writeUInt32LE(32_000, 28)
  out.writeUInt16LE(2, 32)
  out.writeUInt16LE(16, 34)
  out.write('data', 36, 'ascii')
  out.writeUInt32LE(samples * 2, 40)
  for (let i = 0; i < samples; i++) out.writeInt16LE(i % 2 ? 9_000 : -9_000, 44 + i * 2)
  return out
}

/** The transcription service and the daily voice quota, faked: what each was asked, and what each answers. */
function fakeVoice(words = 'Ship the parser today.') {
  const calls = { transcribe: 0, reserved: [] as string[], committed: [] as string[], released: [] as string[] }
  const voice: VoiceTranscription & { calls: typeof calls; answer: () => Promise<string>; quota: 'ok' | 'limit' } = {
    calls, quota: 'ok', answer: async () => words,
    async reserve(_user, id) { calls.reserved.push(id); return voice.quota },
    async transcribe() { calls.transcribe++; return voice.answer() },
    async commit(id) { calls.committed.push(id) },
    async release(id) { calls.released.push(id) },
  }
  return voice
}

async function swarm(voice = fakeVoice(), now: () => number = Date.now) {
  const files = new MemoryAttachmentStore()
  const store = new SuperTabStore(db, now, { store: files, prefix: 'super-tabs', publicBase: 'https://cdn.example.test' }, { voice })
  const tab = (await store.create(owner, randomUUID(), 'Voice', 'public')).id
  const execute = (command: Parameters<SuperTabStore['execute']>[2]['command'], user = owner, asAgentId?: string) =>
    store.execute(user, tab, { operationId: randomUUID(), command }, asAgentId)
  await execute({ type: 'machine.contribute', machineId: 'mac', name: 'Mac', cwd: '/repo', engines: ['codex'], maxAgents: 8 })
  const lead = String((await execute({ type: 'agent.add', conversationId: 'lead', machineId: 'mac', harnessId: randomUUID(), name: 'Lead', configuration: config, lead: true })).result.agentId)
  const upload = async (bytes: Buffer, name = 'Voice message.wav', mime = 'audio/wav', user = owner) =>
    (await store.upload(user, tab, 'lead', { name, mime, body: Readable.from([bytes]) })).attachment
  return { store, tab, execute, lead, upload, voice, files }
}

describe('reading a voice file', () => {
  it('answers its length, waveform and that it was transcribed, and transcribes it once', async () => {
    const f = await swarm()
    const file = await f.upload(wav(81))
    const first = await f.store.readVoice(owner, f.tab, file.id, 'en')
    expect(first.voice).toMatchObject({ durationMs: 81_000, transcribed: true })
    expect(first.voice.waveform).toHaveLength(48)
    expect(first.voice).not.toHaveProperty('transcript')
    expect(await f.store.readVoice(owner, f.tab, file.id, 'en')).toEqual(first)
    expect(f.voice.calls).toMatchObject({ transcribe: 1, reserved: [`voice:${file.id}`], committed: [`voice:${file.id}`] })
  })

  it('lets one of two calls at once transcribe, and the other wait', async () => {
    const f = await swarm()
    let finish!: (words: string) => void
    f.voice.answer = () => new Promise(resolve => { finish = resolve })
    const file = await f.upload(wav(2))
    const first = f.store.readVoice(owner, f.tab, file.id, 'en')
    await vi.waitFor(() => expect(f.voice.calls.transcribe).toBe(1))
    await expect(f.store.readVoice(owner, f.tab, file.id, 'en')).rejects.toMatchObject({ code: 'VOICE_TRANSCRIBING', status: 409 })
    finish('Hello.')
    expect((await first).voice.transcribed).toBe(true)
    expect(f.voice.calls.transcribe).toBe(1)
  })

  it('takes over a claim left for over two minutes', async () => {
    let at = Date.now()
    const f = await swarm(fakeVoice(), () => at)
    const file = await f.upload(wav(2))
    await db.superTabAttachment.update({ where: { id: file.id }, data: { voiceSince: new Date(at - 60_000) } })
    await expect(f.store.readVoice(owner, f.tab, file.id, 'en')).rejects.toMatchObject({ code: 'VOICE_TRANSCRIBING' })
    at += 61_000 + 60_000
    expect((await f.store.readVoice(owner, f.tab, file.id, 'en')).voice.transcribed).toBe(true)
  })

  it('refuses another actor\'s file, a sent one, an agent caller, a file that is not WAV and a clip over 10:05', async () => {
    const f = await swarm()
    const file = await f.upload(wav(2))
    await expect(f.store.readVoice(peer, f.tab, file.id, 'en')).rejects.toMatchObject({ status: 403 })
    const agents = (await f.store.upload(owner, f.tab, 'lead', { name: 'Voice message.wav', mime: 'audio/wav', body: Readable.from([wav(2)]) }, f.lead)).attachment
    await expect(f.store.readVoice(owner, f.tab, agents.id, 'en')).rejects.toMatchObject({ code: 'ATTACHMENT_UNAVAILABLE' })
    await expect(f.store.readVoice(owner, f.tab, file.id, 'en', f.lead)).rejects.toMatchObject({ code: 'VOICE_PEOPLE_ONLY' })
    const text = await f.upload(Buffer.from('hello, this is not audio at all, only some text that is long enough'), 'notes.wav')
    await expect(f.store.readVoice(owner, f.tab, text.id, 'en')).rejects.toMatchObject({ code: 'UNSUPPORTED_AUDIO', status: 415 })
    const png = await f.upload(wav(2), 'shot.png', 'image/png')
    await expect(f.store.readVoice(owner, f.tab, png.id, 'en')).rejects.toMatchObject({ code: 'UNSUPPORTED_AUDIO' })
    await f.execute({ type: 'message.send', conversationId: 'lead', text: '', replyTo: null, recipientIds: [], attachmentIds: [file.id] })
    await expect(f.store.readVoice(owner, f.tab, file.id, 'en')).rejects.toMatchObject({ code: 'ATTACHMENT_UNAVAILABLE' })
    // Over the form's 10 MB: a long clip is stored straight in the bucket in production; here, as if it were.
    const long = await f.upload(wav(1))
    const row = await db.superTabAttachment.findUniqueOrThrow({ where: { id: long.id } })
    const big = wav(606)
    f.files.objects.set(row.objectKey, { ...f.files.objects.get(row.objectKey)!, bytes: big })
    await db.superTabAttachment.update({ where: { id: long.id }, data: { size: big.length } })
    await expect(f.store.readVoice(owner, f.tab, long.id, 'en')).rejects.toMatchObject({ code: 'VOICE_TOO_LONG', status: 413 })
    expect(f.voice.calls.transcribe).toBe(0)
  })

  it('sends without words past the daily voice limit or when transcription fails, once, and logs no words', async () => {
    const voice = fakeVoice('my secret plan')
    const f = await swarm(voice)
    const logged: string[] = []
    const spies = (['info', 'warn', 'error'] as const).map(level => vi.spyOn(logger, level).mockImplementation((...args: unknown[]) => { logged.push(JSON.stringify(args)); return logger }))
    voice.quota = 'limit'
    const limited = await f.upload(wav(3))
    expect((await f.store.readVoice(owner, f.tab, limited.id, 'en')).voice).toMatchObject({ durationMs: 3000, transcribed: false, failure: 'limit' })
    expect(voice.calls.transcribe).toBe(0)
    voice.quota = 'ok'
    voice.answer = async () => { throw new Error('Deepgram HTTP 503') }
    const failed = await f.upload(wav(3))
    expect((await f.store.readVoice(owner, f.tab, failed.id, 'en')).voice).toMatchObject({ transcribed: false, failure: 'failed' })
    expect(voice.calls.released).toEqual([`voice:${failed.id}`])
    voice.answer = async () => 'my secret plan'
    expect((await f.store.readVoice(owner, f.tab, failed.id, 'en')).voice).toMatchObject({ failure: 'failed' })
    const spoken = await f.upload(wav(3))
    await f.store.readVoice(owner, f.tab, spoken.id, 'en')
    spies.forEach(spy => spy.mockRestore())
    expect(logged.some(line => line.includes('super-tab voice'))).toBe(true)
    expect(logged.join('\n')).not.toContain('my secret plan')
  })
})

describe('sending a voice message', () => {
  const send = (f: Awaited<ReturnType<typeof swarm>>, id: string) =>
    f.execute({ type: 'message.send', conversationId: 'lead', text: '', replyTo: null, recipientIds: [], attachmentIds: [id] })

  it('stores its words as the message\'s text and its facts on the file, never its words twice', async () => {
    const f = await swarm()
    const file = await f.upload(wav(81))
    await f.store.readVoice(owner, f.tab, file.id, 'en')
    await send(f, file.id)
    const message = (await db.superTabMessage.findMany({ where: { tabId: f.tab, conversationId: 'lead' } }))
      .find(m => JSON.stringify(m.attachments ?? null).includes(file.id))
    expect(message!.text).toBe('Ship the parser today.')
    expect(message!.attachments).toEqual([expect.objectContaining({ id: file.id, mime: 'audio/wav', voice: expect.objectContaining({ durationMs: 81_000, transcribed: true }) })])
    expect(JSON.stringify(message!.attachments)).not.toContain('Ship the parser')
    const [delivery] = await db.superTabDelivery.findMany({ where: { tabId: f.tab, messageId: message!.id } })
    expect(delivery!.agentId).toBe(f.lead)
  })

  it('waits for a file still being transcribed', async () => {
    const f = await swarm()
    const file = await f.upload(wav(2))
    await db.superTabAttachment.update({ where: { id: file.id }, data: { voiceSince: new Date() } })
    await expect(send(f, file.id)).rejects.toMatchObject({ code: 'VOICE_TRANSCRIBING', status: 409 })
  })
})

describe('what agents and previews read of a voice message', () => {
  async function sent(words: string, quota: 'ok' | 'limit' = 'ok') {
    const voice = fakeVoice(words)
    voice.quota = quota
    const f = await swarm(voice)
    const file = await f.upload(wav(81))
    await f.store.readVoice(owner, f.tab, file.id, 'en')
    const messageId = String((await f.execute({ type: 'message.send', conversationId: 'lead', text: '', replyTo: null, recipientIds: [], attachmentIds: [file.id] })).result.messageId)
    return { f, messageId }
  }
  const claim = async (f: Awaited<ReturnType<typeof swarm>>, messageId: string) => {
    const delivery = await db.superTabDelivery.findFirstOrThrow({ where: { tabId: f.tab, messageId } })
    return (await f.store.delivery(owner, f.tab, f.lead, delivery.id, { type: 'claim', claimId: randomUUID() })).message
  }

  it('hands an agent the words, marked as a voice message with its length', async () => {
    const { f, messageId } = await sent('hello team')
    expect((await claim(f, messageId))!.text).toBe('Voice message (1:21): hello team')
    // What is stored stays the words alone: people read them under the bubble.
    expect((await db.superTabMessage.findUniqueOrThrow({ where: { id: messageId } })).text).toBe('hello team')
  })

  it('tells an agent when a voice message has no words', async () => {
    const { f, messageId } = await sent('unheard', 'limit')
    expect((await claim(f, messageId))!.text).toBe('Voice message (1:21), not transcribed.')
  })

  it('previews it as "Voice message (1:21)", and a plain message as its text', async () => {
    const { f, messageId } = await sent('hello team')
    const plain = String((await f.execute({ type: 'message.send', conversationId: 'lead', text: 'Typed words', replyTo: null, recipientIds: [] })).result.messageId)
    const rows = (await readBy(db, 'SuperTabMessage', 'id', [{ $match: { _id: { $in: [messageId, plain] }, tabId: f.tab } },
      { $project: { ...FIELDS, conversationId: 1 } }])).map(row)
    expect(Object.fromEntries(rows.map(r => [r.id, preview(r).text]))).toEqual({ [messageId]: 'Voice message (1:21)', [plain]: 'Typed words' })
  })
})