Skip to content

The agent that grows with you

11.9 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'

import type * as data from './data'
import type { GroupActivityEntry } from './group-activity'
import type * as groupActivity from './group-activity'
import type * as groupChat from './group-chat'
import type * as groupRounds from './group-rounds'
import { createGroupGateway, drain, runTimersInline, scriptedStorage } from './group-test-utils'
import type { GatewayOptions, ScriptedGateway } from './group-test-utils'
import type { GroupMember } from './types'

// Collapsible group Activity view: a runtime-only, bounded feed of truthful
// turn events (queued / working / replied / passed / timed-out / failed /
// cancelled / settled / delivered) that the room shows in a quiet disclosure.
// The transcript stays the only durable record — activity is never persisted.

const { host } = vi.hoisted(() => ({ host: {} as Record<string, unknown> }))

vi.mock('@hermes/plugin-sdk', async () => {
  const { pluginSdkMock } = await import('./group-test-utils')

  return pluginSdkMock(host)
})

const MEMBERS: GroupMember[] = [
  { name: 'research', title: '' },
  { name: 'builder', title: '' },
  { name: 'ops', title: 'The Ops' }
]

/** A failed turn's activity row carries the typed reason the gateway sent. */
type ActivityRow = GroupActivityEntry

interface Room {
  activity: typeof groupActivity
  chat: typeof groupChat
  data: typeof data
  gateway: ScriptedGateway
  rounds: typeof groupRounds
}

async function loadRoom(options: GatewayOptions = {}): Promise<Room> {
  vi.resetModules()
  const gateway = createGroupGateway(options)

  for (const key of Object.keys(host)) {
    delete host[key]
  }

  Object.assign(host, gateway.host)

  const [activity, chat, data, rounds, shared] = await Promise.all([
    import('./group-activity'),
    import('./group-chat'),
    import('./data'),
    import('./group-rounds'),
    import('./shared')
  ])

  shared.setPluginCtx(scriptedStorage(gateway.storage))

  return { activity, chat, data, gateway, rounds }
}

function feed(room: Room, group: string): ActivityRow[] {
  return (room.activity.$groupActivity.get()[group]?.events || []) as ActivityRow[]
}

beforeEach(() => {
  runTimersInline()
})

afterEach(() => {
  vi.restoreAllMocks()
})

describe('turn arc', () => {
  it('a settled turn records the full truthful arc: queued, working, replies and passes, settled', async () => {
    const room = await loadRoom({
      turn: ({ profile, prompt }) =>
        profile === 'research' && !prompt.includes('(you)') ? 'I looked into it — ship it.' : '(pass)'
    })

    room.rounds.sendToGroupChat('Room', MEMBERS, 'please review')
    await drain(() => Boolean(room.chat.$groupChats.get().Room?.running))

    const events = feed(room, 'Room')
    const kinds = events.map(event => event.kind)

    expect(kinds[0]).toBe('queued')
    expect(kinds[kinds.length - 1]).toBe('settled')
    expect(kinds.filter(kind => kind === 'working').length).toBeGreaterThanOrEqual(3)
    expect(kinds).toContain('replied')
    expect(kinds.filter(kind => kind === 'passed').length).toBeGreaterThanOrEqual(2)
    expect(events.find(event => event.kind === 'replied')?.member).toBe('research')
  })

  it('records source-scoped members by member key and labels them through their owner meta', async () => {
    // #94869 / #102294: bot meta lives under `connectionId::profile`, so an
    // activity row recorded by bare name could neither find the title nor
    // tell two same-named connections apart.
    const room = await loadRoom({ turn: () => 'on it' })

    const scoped: GroupMember = {
      connectionId: 'local',
      connectionKind: 'local',
      connectionLabel: 'This device',
      name: 'research',
      route: { connectionId: 'local', mode: 'local', profile: 'research', targetProfile: 'research' },
      sourceScoped: true,
      title: ''
    }

    room.data.$botMeta.set({ 'local::research': { title: 'Radar' } })
    room.data.$lastRoster.set([scoped])

    room.rounds.sendToGroupChat('Scoped', [scoped], 'status?')
    await drain(() => Boolean(room.chat.$groupChats.get().Scoped?.running))

    const events = feed(room, 'Scoped').filter(event => event.member && event.member !== 'You')

    expect(events.length).toBeGreaterThan(0)
    expect(events.every(event => event.member === 'local::research')).toBe(true)
    expect(events.map(event => room.activity.groupActivityLabel(event))).toContain('Radar replied')
  })

  it('a failed member turn records failed instead of a phantom reply', async () => {
    const room = await loadRoom({
      turn: ({ profile }) => {
        if (profile === 'builder') {
          throw new Error('gateway hiccup')
        }

        return '(pass)'
      }
    })

    room.rounds.sendToGroupChat('Flaky', MEMBERS, 'anyone around?')
    await drain(() => Boolean(room.chat.$groupChats.get().Flaky?.running))

    expect(feed(room, 'Flaky').some(event => event.kind === 'failed' && event.member === 'builder')).toBe(true)
  })

  it('a failed member turn preserves and prefers its typed reason', async () => {
    const room = await loadRoom({
      turn: ({ profile }) => {
        if (profile === 'builder') {
          throw Object.assign(new Error('generic gateway failure'), {
            data: { reason: 'provider_auth_or_access' }
          })
        }

        return '(pass)'
      }
    })

    room.rounds.sendToGroupChat('Typed failure', MEMBERS, 'anyone around?')
    await drain(() => Boolean(room.chat.$groupChats.get()['Typed failure']?.running))

    const failed = feed(room, 'Typed failure').find(event => event.kind === 'failed' && event.member === 'builder')

    expect(failed?.reason).toBe('provider_auth_or_access')
    expect(Object.values(room.data.$botAttention.get())[0]?.reason).toBe('provider_auth_or_access')
  })

  // #117366: an untyped failure used to collapse to a bare "builder hit an
  // error" — no cause, nothing to act on. The row now keeps the error's first
  // line (secret spans redacted) and the badge still classifies from it.
  it('an untyped failed member turn surfaces the error first line, redacted, and keeps the badge fallback', async () => {
    const room = await loadRoom({
      turn: ({ profile }) => {
        if (profile === 'builder') {
          throw new Error(
            'No LLM provider configured for https://api.example.test/v1?api_key=sk-live-0123456789abcdef\n' +
              '    at runMemberTurn (group-turns.ts:1)'
          )
        }

        return '(pass)'
      }
    })

    room.rounds.sendToGroupChat('Untyped failure', MEMBERS, 'anyone around?')
    await drain(() => Boolean(room.chat.$groupChats.get()['Untyped failure']?.running))

    const failed = feed(room, 'Untyped failure').find(event => event.kind === 'failed' && event.member === 'builder')
    const label = room.activity.groupActivityLabel(failed!, 'Untyped failure')

    expect(label.startsWith('builder hit an error — No LLM provider configured for ')).toBe(true)
    expect(label).not.toContain('sk-live-0123456789abcdef')
    expect(label).not.toContain('runMemberTurn')
    expect(Object.values(room.data.$botAttention.get())[0]?.reason).toBe('missing_config')
  })

  // #116458: a member whose backend never got a pool slot is healthy; the
  // room must not describe pool starvation as a bot crash (and never badge it).
  it('a slot-wait timeout reads as could-not-start, not as a bot error', async () => {
    const room = await loadRoom({
      turn: ({ profile }) => {
        if (profile === 'builder') {
          throw new Error(
            'Error invoking remote method \'hermes:api\': Local backend start for "builder" timed out while waiting for a free slot.'
          )
        }

        return '(pass)'
      }
    })

    room.rounds.sendToGroupChat('Slot wait', MEMBERS, 'anyone around?')
    await drain(() => Boolean(room.chat.$groupChats.get()['Slot wait']?.running))

    const failed = feed(room, 'Slot wait').find(event => event.kind === 'failed' && event.member === 'builder')

    expect(failed?.reason).toBe(room.activity.GROUP_SLOT_WAIT_REASON)
    expect(room.activity.groupActivityLabel(failed!, 'Slot wait')).toBe(
      "builder couldn't start — too many bots running"
    )
    expect(room.data.$botAttention.get()).toEqual({})
  })
})

describe('epoch scoping', () => {
  it('queues follow-ups without cancelling the active turn or losing its reply delta', async () => {
    // A frozen clock with ever-SMALLER entry ids: every row lands in the same
    // millisecond and the gateway mirror merge (sort by `at`, then by id)
    // would order each new row before the last. A busy CI runner reached
    // this by chance (same-ms appends, random UUIDs) and the reordered log
    // fed the member its follow-up twice.
    let uuid = 0xffff

    vi.spyOn(Date, 'now').mockReturnValue(1_700_000_000_000)
    vi.spyOn(globalThis.crypto, 'randomUUID').mockImplementation(
      () =>
        `${(uuid--).toString(16).padStart(8, '0')}-0000-4000-8000-000000000000` as `${string}-${string}-${string}-${string}-${string}`
    )

    let release!: (reply: string) => void

    const first = new Promise<string>(resolve => {
      release = resolve
    })

    const room = await loadRoom({ turn: ({ n }) => (n === 1 ? first : '(pass)') })
    const member: GroupMember[] = [{ name: 'research', title: '' }]
    const thread = room.rounds.sendToGroupChat('Busy', member, 'first ask')!
    await drain(() => room.gateway.calls.length < 1, 50)
    const epoch = room.chat.$groupChats.get().Busy.epoch
    room.rounds.sendToGroupChat('Busy', member, 'follow-up', thread)
    await drain(() => false)
    expect(room.gateway.calls).toHaveLength(1)
    expect(room.chat.$groupChats.get().Busy.epoch).toBe(epoch)
    release('first reply')
    await drain(() => room.gateway.calls.length < 2)
    await drain(() => Boolean(room.chat.$groupChats.get().Busy?.running))
    expect(room.gateway.calls).toHaveLength(2)
    expect(room.gateway.calls[1].prompt).toMatch(/follow-up[\s\S]*first reply/)
    expect(room.gateway.calls[1].prompt).not.toContain('first ask')
    expect(feed(room, 'Busy').some(event => event.kind === 'cancelled')).toBe(false)
  })

  it('epoch filtering drops events from a superseded run', async () => {
    const room = await loadRoom()
    room.chat.updateGroupChat('Epoch', current => {
      current.log = []
      current.epoch = 5

      return current
    })
    room.activity.recordGroupActivity('Epoch', { kind: 'queued', member: 'You' })
    room.activity.recordGroupActivity('Epoch', { kind: 'working', member: 'research' })

    room.chat.updateGroupChat('Epoch', current => {
      current.epoch = 6

      return current
    })

    expect(room.activity.currentGroupActivity('Epoch')).toHaveLength(0)

    room.activity.recordGroupActivity('Epoch', { kind: 'working', member: 'research' })
    const current = room.activity.currentGroupActivity('Epoch')

    expect(current).toHaveLength(1)
    expect(current[0].kind).toBe('working')
  })
})

describe('feed shape', () => {
  it('the feed is bounded: it stops growing and keeps the newest events', async () => {
    const room = await loadRoom()
    room.chat.updateGroupChat('Cap', current => {
      current.log = []

      return current
    })

    for (let i = 0; i < 500; i++) {
      room.activity.recordGroupActivity('Cap', { kind: 'working', member: `member-${i}` })
    }

    const bounded = feed(room, 'Cap')

    expect(bounded.length).toBeLessThan(500)
    expect(bounded[bounded.length - 1].member).toBe('member-499')
    expect(bounded.some(event => event.member === 'member-0')).toBe(false)

    room.activity.recordGroupActivity('Cap', { kind: 'working', member: 'one more' })

    expect(feed(room, 'Cap')).toHaveLength(bounded.length)
  })

  it('activity is runtime-only: never persisted, never hydrated', async () => {
    const room = await loadRoom()

    room.rounds.sendToGroupChat('Volatile', MEMBERS, 'hello')
    await drain(() => Boolean(room.chat.$groupChats.get().Volatile?.running))

    expect(feed(room, 'Volatile').length).toBeGreaterThan(0)
    expect([...room.gateway.storage.keys()]).not.toContain('group-activity')
  })
})