Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

14.7 KB
Raw Blame
import { createServer, type Server } from 'node:net'
import { mkdtempSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY } from '../../shared/protocol-version'
import { ORCHESTRATION_WORKER_START_CLIENT_GRACE_MS } from '../../shared/orchestration-timing-budgets'
import { MAX_TIMER_DELAY_MS } from '../../shared/timer-delay'
import { orchestrationMutationRecoveryError } from '../orchestration-mutation-recovery'
import { reportCliError } from '../format'
import { RuntimeClient, RuntimeClientError, RuntimeRpcFailureError } from '../runtime-client'

const servers = new Set<Server>()

afterEach(async () => {
  vi.restoreAllMocks()
  await Promise.all(
    [...servers].map(
      (server) =>
        new Promise<void>((resolve) => {
          server.close(() => resolve())
        })
    )
  )
  servers.clear()
})

function writeRuntimeConnection(userDataPath: string, endpoint: string, runtimeId: string): void {
  writeFileSync(
    join(userDataPath, 'orca-runtime.json'),
    JSON.stringify({
      runtimeId,
      pid: 1,
      transports: [{ kind: 'unix', endpoint }],
      authToken: 'token',
      startedAt: 1
    })
  )
}

function expectPromptRetryBlockedJson(error: unknown, requestId: string): void {
  const log = vi.spyOn(console, 'log').mockImplementation(() => {})
  reportCliError(error, true)
  const output = JSON.parse(String(log.mock.calls[0]?.[0])) as {
    error: { data?: Record<string, unknown> }
  }
  expect(output.error.data).toMatchObject({
    deliveryOutcome: 'unknown',
    retrySafe: false,
    nextSteps: expect.arrayContaining([
      'Inspect the terminal output and agent state without sending input.'
    ])
  })
  expect(JSON.stringify(output)).not.toContain('--retry-request')
  expect(JSON.stringify(output)).not.toContain(requestId)
  expect(output.error.data).not.toHaveProperty('orchestrationRequestId')
}

describe('RuntimeClient orchestration recovery identity', () => {
  it('rejects a worker-start timeout whose client grace would overflow timers', () => {
    const client = new RuntimeClient(undefined, 60_000, null, null, 'orca')
    const resolve = (
      client as unknown as {
        resolveMethodTimeoutMs: (method: string, params?: unknown) => number
      }
    ).resolveMethodTimeoutMs.bind(client)
    const maxValid = MAX_TIMER_DELAY_MS - ORCHESTRATION_WORKER_START_CLIENT_GRACE_MS
    expect(resolve('orchestration.workerStart', { timeoutMs: maxValid })).toBe(MAX_TIMER_DELAY_MS)
    expect(() => resolve('orchestration.workerStart', { timeoutMs: maxValid + 1 })).toThrow(
      'derived timeout must be'
    )
  })

  it('attaches the request and exact retry identity to a real RPC failure response', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-recovery-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    let rpcRequestId: string | undefined
    let orchestrationRequestId: string | undefined
    const server = createServer((socket) => {
      let buffer = ''
      socket.setEncoding('utf8')
      socket.on('data', (chunk: string) => {
        buffer += chunk
        const newline = buffer.indexOf('\n')
        if (newline === -1) {
          return
        }
        const request = JSON.parse(buffer.slice(0, newline)) as { id: string; method: string }
        if (request.method === 'orchestration.workerStart') {
          rpcRequestId = request.id
          if (
            'orchestrationRequestId' in request &&
            typeof request.orchestrationRequestId === 'string'
          ) {
            orchestrationRequestId = request.orchestrationRequestId
          }
        }
        const response =
          request.method === 'status.get'
            ? {
                id: request.id,
                ok: true,
                result: { capabilities: [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY] },
                _meta: { runtimeId: 'runtime-1' }
              }
            : {
                id: request.id,
                ok: false,
                error: {
                  code: 'runtime_timeout',
                  message: 'request timed out',
                  data: { requestId: 'request_1', dispatchId: 'dispatch_1' }
                },
                _meta: { runtimeId: 'runtime-1' }
              }
        socket.end(`${JSON.stringify(response)}\n`)
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-1')

    const client = new RuntimeClient(userDataPath, 500, null, null, 'orca')
    try {
      await client.call('orchestration.workerStart', { task: 'task_1' })
      throw new Error('expected worker-start failure')
    } catch (error) {
      expect(error).toBeInstanceOf(RuntimeRpcFailureError)
      const recovered = orchestrationMutationRecoveryError(error) as {
        data?: Record<string, unknown>
        response?: { id?: string; _meta?: { runtimeId?: string } }
      }
      expect(recovered).toBeInstanceOf(RuntimeRpcFailureError)
      expect(rpcRequestId).toEqual(expect.any(String))
      expect(orchestrationRequestId).toEqual(expect.any(String))
      expect(recovered.response).toMatchObject({
        id: rpcRequestId,
        _meta: { runtimeId: 'runtime-1' }
      })
      expect(recovered.data).toMatchObject({
        orchestrationRequestId,
        dispatchId: 'dispatch_1',
        originalCommand: ['orca', 'orchestration', 'worker-start', '--task', 'task_1'],
        recovery: {
          queryCommand: [
            'orca',
            'orchestration',
            'worker-show',
            '--dispatch',
            'dispatch_1',
            '--json'
          ],
          retryCommand: [
            'orca',
            'orchestration',
            'worker-start',
            '--task',
            'task_1',
            '--retry-request',
            orchestrationRequestId
          ],
          recoveryBlocked: false
        }
      })
    }
  })

  it('keeps durable prompt retry when failure metadata proves the preflight runtime', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-current-prompt-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    const server = createServer((socket) => {
      socket.once('data', (data) => {
        const request = JSON.parse(String(data).trim()) as { id: string }
        socket.end(
          `${JSON.stringify({
            id: request.id,
            ok: false,
            error: { code: 'runtime_timeout', message: 'request timed out' },
            _meta: { runtimeId: 'runtime-current' }
          })}\n`
        )
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-current')

    const client = new RuntimeClient(userDataPath, 500, null, null, 'orca')
    const error = await client
      .call(
        'terminal.send',
        {
          terminal: 'term-current',
          text: 'review',
          enter: true,
          interrupt: false,
          agentPrompt: true,
          client: { id: 'orca-cli', type: 'desktop' }
        },
        {
          terminalPromptPreflight: { runtimeId: 'runtime-current' },
          orchestrationRequestId: 'prompt-current'
        }
      )
      .then(() => undefined)
      .catch((caught: unknown) => caught)

    expect(error).toBeInstanceOf(RuntimeRpcFailureError)
    expect(error).toMatchObject({
      data: { orchestrationRequestId: 'prompt-current' }
    })
    expect((error as Error).message).toContain('--retry-request prompt-current')
  })

  it('keeps the prompt retry ID when the attested runtime times out in transport', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-rt-timeout-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    let receivedRequest: Record<string, unknown> | undefined
    const server = createServer((socket) => {
      socket.once('data', (data) => {
        receivedRequest = JSON.parse(String(data).trim()) as Record<string, unknown>
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-current')

    const client = new RuntimeClient(userDataPath, 200, null, null, 'orca')
    const error = await client
      .call(
        'terminal.send',
        {
          terminal: 'term-current',
          text: 'review',
          enter: true,
          interrupt: false,
          agentPrompt: true,
          client: { id: 'orca-cli', type: 'desktop' }
        },
        {
          terminalPromptPreflight: { runtimeId: 'runtime-current' },
          orchestrationRequestId: 'prompt-transport-timeout'
        }
      )
      .then(() => undefined)
      .catch((caught: unknown) => caught)

    expect(receivedRequest?.orchestrationRequestId).toBe('prompt-transport-timeout')
    expect(error).toBeInstanceOf(RuntimeClientError)
    expect(error).not.toBeInstanceOf(RuntimeRpcFailureError)
    expect((error as RuntimeClientError).code).toBe('runtime_timeout')
    expect(error).toMatchObject({ data: { orchestrationRequestId: 'prompt-transport-timeout' } })
    expect((error as Error).message).toContain(
      '--retry-request prompt-transport-timeout --wait-submit <seconds>'
    )
    expect((error as RuntimeClientError).data).not.toHaveProperty('retrySafe')
  })

  it('blocks retry when a downgraded runtime rejects after capability preflight', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-downgraded-prompt-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    let receivedRequest: Record<string, unknown> | undefined
    const server = createServer((socket) => {
      socket.once('data', (data) => {
        const request = JSON.parse(String(data).trim()) as Record<string, unknown>
        receivedRequest = request
        socket.end(
          `${JSON.stringify({
            id: request.id,
            ok: false,
            error: { code: 'runtime_timeout', message: 'request timed out' },
            _meta: { runtimeId: 'runtime-after-downgrade' }
          })}\n`
        )
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-after-downgrade')

    const client = new RuntimeClient(userDataPath, 500, null, null, 'orca')
    const error = await client
      .call(
        'terminal.send',
        {
          terminal: 'term-downgraded',
          text: 'review',
          enter: true,
          interrupt: false,
          agentPrompt: true,
          client: { id: 'orca-cli', type: 'desktop' }
        },
        {
          terminalPromptPreflight: { runtimeId: 'runtime-before-downgrade' },
          orchestrationRequestId: 'prompt-downgraded-rejection'
        }
      )
      .then(() => undefined)
      .catch((caught: unknown) => caught)

    expect(receivedRequest?.orchestrationRequestId).toBe('prompt-downgraded-rejection')
    expect(error).toBeInstanceOf(RuntimeRpcFailureError)
    expectPromptRetryBlockedJson(error, 'prompt-downgraded-rejection')
  })

  it('blocks retry when a downgraded runtime loses the prompt reply', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-lost-prompt-reply-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    let receivedRequest: Record<string, unknown> | undefined
    const server = createServer((socket) => {
      socket.once('data', (data) => {
        const request = JSON.parse(String(data).trim()) as Record<string, unknown>
        receivedRequest = request
        socket.destroy()
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-after-downgrade')

    const client = new RuntimeClient(userDataPath, 500, null, null, 'orca')
    const error = await client
      .call(
        'terminal.send',
        {
          terminal: 'term-downgraded',
          text: 'review',
          enter: true,
          interrupt: false,
          agentPrompt: true,
          client: { id: 'orca-cli', type: 'desktop' }
        },
        {
          terminalPromptPreflight: { runtimeId: 'runtime-before-downgrade' },
          orchestrationRequestId: 'prompt-downgraded-lost-reply'
        }
      )
      .then(() => undefined)
      .catch((caught: unknown) => caught)

    expect(receivedRequest?.orchestrationRequestId).toBe('prompt-downgraded-lost-reply')
    expect(error).toBeInstanceOf(RuntimeClientError)
    expect(error).not.toBeInstanceOf(RuntimeRpcFailureError)
    expectPromptRetryBlockedJson(error, 'prompt-downgraded-lost-reply')
    expect(JSON.stringify((error as RuntimeClientError).data)).not.toContain('Update Orca')
  })

  it('reports an unknown legacy prompt outcome without advertising an unsafe retry', async () => {
    const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-legacy-prompt-'))
    const endpoint = join(userDataPath, 'runtime.sock')
    let receivedRequest: Record<string, unknown> | undefined
    const server = createServer((socket) => {
      socket.once('data', (data) => {
        receivedRequest = JSON.parse(String(data).trim()) as Record<string, unknown>
        socket.destroy()
      })
    })
    servers.add(server)
    await new Promise<void>((resolve) => server.listen(endpoint, resolve))
    writeRuntimeConnection(userDataPath, endpoint, 'runtime-legacy')

    const client = new RuntimeClient(userDataPath, 500, null, null, 'orca')
    const error = await client
      .call(
        'terminal.send',
        {
          terminal: 'term-legacy',
          text: 'review',
          enter: true,
          interrupt: false,
          agentPrompt: true,
          client: { id: 'orca-cli', type: 'desktop' }
        },
        { legacyTerminalPrompt: true }
      )
      .then(() => undefined)
      .catch((caught: unknown) => caught)

    expect(error).toBeInstanceOf(RuntimeClientError)
    expect(error).not.toBeInstanceOf(RuntimeRpcFailureError)
    expect(error).toMatchObject({
      data: {
        deliveryOutcome: 'unknown',
        retrySafe: false,
        nextSteps: expect.arrayContaining([
          'Inspect the terminal output and agent state without sending input.',
          'Update Orca on the execution host before future prompt sends that need durable retry.'
        ])
      }
    })
    expect((error as RuntimeClientError).data).not.toHaveProperty('orchestrationRequestId')
    expect(receivedRequest).not.toHaveProperty('orchestrationRequestId')
    expect((error as Error).message).not.toContain('--retry-request')
    expect((error as Error).message).toContain('do not resend automatically')
  })
})