Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

11.4 KB
Raw Blame
// Compare identical benchmark APIs on baseline/candidate; separate instrumented allocation counts from clean timing to avoid patch-induced deoptimization.
// Run: ORCA_BACKGROUND_LAUNCH=1 pnpm exec vitest run --config config/vitest.agent-status-benchmark.config.ts
// Reported-user scale: 423 worktrees/634 tabs, rather than the 100-worktree fixture.
import { writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import type { AppState } from '@/store/types'
import type { AgentStatusBatchUpdate } from '@/store/slices/agent-status/agent-status-contract'
import '@/store/slices/agent-status/agent-status'
import { MAX_LIVE_AGENT_STATUSES } from '@/store/slices/agent-status/agent-status-capacity-eviction'
import {
  createTestStore,
  makeTab,
  makeUnifiedTab,
  makeWorktree,
  TEST_REPO
} from '@/store/slices/store-test-helpers'
import { makePaneKey } from '../../src/shared/stable-pane-id'
import { getDefaultSettings } from '../../src/shared/constants'
import {
  createAgentStatusPaneRoutingIndex,
  resolvePaneKeyFromRoutingIndex
} from '@/hooks/ipc-events/agent-status-pane-routing-index'
import { resolvePaneKey } from '@/hooks/ipc-events/agent-status-routing'

const WORKTREES = 423
const EVENTS = 1_000
const BATCH_SIZE = 8
const LEAF_ID = '11111111-1111-4111-8111-111111111111'
const BASE_TIME = 2_000_000_000

const counters = { maps: 0, sets: 0, tabComparisons: 0 }

const NativeMap = globalThis.Map
const NativeSet = globalThis.Set

function withAllocationCounting<T>(run: () => T): T {
  class CountingMap<K, V> extends NativeMap<K, V> {
    constructor(entries?: Iterable<readonly [K, V]> | null) {
      super(entries)
      counters.maps += 1
    }
  }
  class CountingSet<V> extends NativeSet<V> {
    constructor(values?: Iterable<V> | null) {
      super(values)
      counters.sets += 1
    }
  }
  counters.maps = 0
  counters.sets = 0
  globalThis.Map = CountingMap
  globalThis.Set = CountingSet
  try {
    return run()
  } finally {
    globalThis.Map = NativeMap
    globalThis.Set = NativeSet
  }
}

/** Tab list whose iteration is observable, so the nested-loop resolver's comparisons are countable. */
class CountingTabList<T> extends Array<T> {
  [Symbol.iterator](): ArrayIterator<T> {
    const iterator = super[Symbol.iterator]()
    const next = iterator.next.bind(iterator)
    iterator.next = (...args) => {
      const result = next(...args)
      if (!result.done) {
        counters.tabComparisons += 1
      }
      return result
    }
    return iterator
  }
}

function buildFixture(countTabIteration: boolean) {
  const store = createTestStore()
  const settings = store.getState().settings ?? getDefaultSettings(tmpdir())
  const tabsByWorktree: AppState['tabsByWorktree'] = {}
  const unifiedTabsByWorktree: AppState['unifiedTabsByWorktree'] = {}
  const worktrees: ReturnType<typeof makeWorktree>[] = []
  const paneKeys: string[] = []
  const owners: { tabId: string; worktreeId: string }[] = []
  for (let index = 0; index < WORKTREES; index += 1) {
    const worktreeId = `wt-${index}`
    worktrees.push(makeWorktree({ id: worktreeId, repoId: TEST_REPO.id }))
    const tabs: ReturnType<typeof makeTab>[] = []
    const unified: ReturnType<typeof makeUnifiedTab>[] = []
    for (let tab = 0; tab < (index % 2 === 0 ? 1 : 2); tab += 1) {
      const tabId = `tab-${index}-${tab}`
      tabs.push(makeTab({ id: tabId, worktreeId, title: `Terminal ${index}-${tab}` }))
      unified.push(
        makeUnifiedTab({
          id: tabId,
          worktreeId,
          groupId: `group-${index}`,
          label: `Label ${index}`
        })
      )
      paneKeys.push(makePaneKey(tabId, LEAF_ID))
      owners.push({ tabId, worktreeId })
    }
    tabsByWorktree[worktreeId] = countTabIteration ? CountingTabList.from(tabs) : tabs
    unifiedTabsByWorktree[worktreeId] = unified
  }
  store.setState({
    repos: [TEST_REPO],
    worktreesByRepo: { [TEST_REPO.id]: worktrees },
    tabsByWorktree,
    unifiedTabsByWorktree,
    terminalLayoutsByTabId: {},
    setGeneratedTabTitlesFromAgentPrompts: () => {},
    settings: { ...settings, tabAutoGenerateTitle: false }
  })
  return { store, paneKeys, owners }
}

function measure(run: () => void): number {
  const start = performance.now()
  run()
  return performance.now() - start
}

function per1k(value: number): number {
  return Math.round((value / EVENTS) * 1000)
}

/** One burst-shaped pass: a fresh index per batch, one pane resolution per event. */
function runIndexedRouting(store: ReturnType<typeof createTestStore>, paneKeys: string[]): void {
  for (let event = 0; event < EVENTS; event += 1) {
    if (event % BATCH_SIZE === 0) {
      store.setState({ agentStatusEpoch: event })
    }
    const index = createAgentStatusPaneRoutingIndex(store.getState())
    resolvePaneKeyFromRoutingIndex(index, paneKeys[event % paneKeys.length])
  }
}

function runStandaloneRouting(state: AppState, paneKeys: string[]): void {
  for (let event = 0; event < EVENTS; event += 1) {
    resolvePaneKey(state, paneKeys[event % paneKeys.length])
  }
}

function buildBatches(
  owners: { tabId: string; worktreeId: string }[],
  paneKeys: string[]
): AgentStatusBatchUpdate[][] {
  const batches: AgentStatusBatchUpdate[][] = []
  for (let event = 0; event < EVENTS; event += 1) {
    const batchIndex = Math.floor(event / BATCH_SIZE)
    const owner = owners[event % owners.length]
    batches[batchIndex] ??= []
    batches[batchIndex].push({
      paneKey: paneKeys[event % paneKeys.length],
      payload: { state: 'working', prompt: `turn ${event}`, agentType: 'claude' },
      timing: { updatedAt: BASE_TIME + event, stateStartedAt: BASE_TIME + event },
      routing: { tabId: owner.tabId, worktreeId: owner.worktreeId }
    })
  }
  return batches
}

const nextTick = (): Promise<void> =>
  new Promise((resolve) => {
    queueMicrotask(resolve)
  })

async function runCommitPass(
  store: ReturnType<typeof createTestStore>,
  batches: AgentStatusBatchUpdate[][],
  onBatch?: (elapsed: number) => void
): Promise<void> {
  for (const batch of batches) {
    const elapsed = measure(() => {
      store.getState().setAgentStatuses(batch)
    })
    onBatch?.(elapsed)
    // Each 33 ms burst is its own tick in production; let the deferred freshness scan run.
    await nextTick()
  }
}

describe('agent-status hot path benchmark', () => {
  it('reports routing, resolution and commit cost at 423 worktrees', async () => {
    const report: Record<string, number> = {}
    const fixture = buildFixture(false)
    const index = createAgentStatusPaneRoutingIndex(fixture.store.getState())
    for (const [position, paneKey] of fixture.paneKeys.entries()) {
      const resolved = resolvePaneKeyFromRoutingIndex(index, paneKey)
      expect(resolved).toMatchObject({
        exists: true,
        owningWorktreeId: fixture.owners[position].worktreeId
      })
      expect(resolved).toEqual(resolvePaneKey(fixture.store.getState(), paneKey))
    }

    {
      const alloc = buildFixture(false)
      withAllocationCounting(() => runIndexedRouting(alloc.store, alloc.paneKeys))
      report['routing.indexed.mapAllocationsPer1kEvents'] = per1k(counters.maps)
      report['routing.indexed.setAllocationsPer1kEvents'] = per1k(counters.sets)

      const warm = buildFixture(false)
      runIndexedRouting(warm.store, warm.paneKeys)
      const clean = buildFixture(false)
      report['routing.indexed.ms'] = Number(
        measure(() => runIndexedRouting(clean.store, clean.paneKeys)).toFixed(2)
      )
    }

    {
      const alloc = buildFixture(true)
      counters.tabComparisons = 0
      withAllocationCounting(() => runStandaloneRouting(alloc.store.getState(), alloc.paneKeys))
      report['routing.standalone.tabComparisonsPer1kEvents'] = per1k(counters.tabComparisons)
      report['routing.standalone.mapAllocationsPer1kEvents'] = per1k(counters.maps)

      const warm = buildFixture(false)
      runStandaloneRouting(warm.store.getState(), warm.paneKeys)
      const clean = buildFixture(false)
      report['routing.standalone.ms'] = Number(
        measure(() => runStandaloneRouting(clean.store.getState(), clean.paneKeys)).toFixed(2)
      )
    }

    {
      const alloc = buildFixture(true)
      counters.tabComparisons = 0
      runIndexedRouting(alloc.store, alloc.paneKeys)
      report['routing.indexed.tabComparisonsPer1kEvents'] = per1k(counters.tabComparisons)
    }

    // Store commits: 1,000 updates folded in 125 transactions, one microtask tick per transaction.
    {
      const counted = buildFixture(false)
      const nativeObjectAssign = Object.assign
      const nativeObjectValues = Object.values
      let objectAssignCalls = 0
      let objectAssignPropertyCopies = 0
      let freshnessEntryVisits = 0
      // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: `Object.assign` is an overload set no single arrow can satisfy; this wrapper only counts calls and forwards every argument to the captured native implementation.
      Object.assign = ((
        target: Record<string, unknown>,
        ...sources: readonly Record<string, unknown>[]
      ) => {
        objectAssignCalls += 1
        for (const source of sources) {
          if (source && typeof source === 'object') {
            objectAssignPropertyCopies += Object.keys(source).length
          }
        }
        return nativeObjectAssign(target, ...sources)
      }) as typeof Object.assign
      // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: same overload-set limit as the `Object.assign` wrapper above; this one counts visited entries and returns the native result unchanged.
      Object.values = ((value: Record<string, unknown>) => {
        const result = nativeObjectValues(value)
        freshnessEntryVisits += result.length
        return result
      }) as typeof Object.values
      const countedBatches = buildBatches(counted.owners, counted.paneKeys)
      try {
        await runCommitPass(counted.store, countedBatches)
      } finally {
        Object.assign = nativeObjectAssign
        Object.values = nativeObjectValues
      }
      report['commit.objectAssignCallsPer1kUpdates'] = per1k(objectAssignCalls)
      report['commit.stagedPropertyCopiesPer1kUpdates'] = per1k(objectAssignPropertyCopies)
      report['commit.freshnessEntryVisitsPer1kUpdates'] = per1k(freshnessEntryVisits)
      report['commit.transactions'] = countedBatches.length
      const committed = counted.store.getState().agentStatusByPaneKey
      const expected = new Map(
        countedBatches
          .flat()
          .slice(-MAX_LIVE_AGENT_STATUSES)
          .map((update) => [update.paneKey, update])
      )
      expect(Object.keys(committed).sort()).toEqual([...expected.keys()].sort())
      for (const [paneKey, update] of expected) {
        expect(committed[paneKey]).toMatchObject({
          ...update.payload,
          ...update.timing,
          ...update.routing
        })
      }

      const warm = buildFixture(false)
      await runCommitPass(warm.store, buildBatches(warm.owners, warm.paneKeys))
      const clean = buildFixture(false)
      let elapsed = 0
      await runCommitPass(clean.store, buildBatches(clean.owners, clean.paneKeys), (batchMs) => {
        elapsed += batchMs
      })
      report['commit.ms'] = Number(elapsed.toFixed(2))
    }

    const outputPath =
      process.env.ORCA_AGENT_STATUS_BENCH_OUTPUT ??
      join(tmpdir(), 'agent-status-hot-path-benchmark.json')
    writeFileSync(
      outputPath,
      `${JSON.stringify({ worktrees: WORKTREES, events: EVENTS, report }, null, 2)}\n`
    )
  })
})