Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

11.8 KB
Raw Blame
import assert from 'node:assert/strict'
import { createHash } from 'node:crypto'
import { readFile, writeFile } from 'node:fs/promises'
import { dirname, extname, join, relative, sep } from 'node:path'
import { performance } from 'node:perf_hooks'
import { fileURLToPath } from 'node:url'
import { build, version } from 'esbuild'

const MIB = 1024 * 1024
const ROOT = fileURLToPath(new URL('../..', import.meta.url))
const BASELINE_MODULES = [
  'src/main/claude-usage/scanner.ts',
  'src/main/claude-usage/transcript-record-parser.ts',
  'src/main/claude-usage/usage-aggregation.ts',
  'src/main/claude-usage/worktree-attribution.ts',
  'src/main/claude-usage/transcript-file-discovery.ts'
]

export async function describeClaudeUsageBenchmarkTooling() {
  const paths = [
    'config/scripts/claude-usage-append-benchmark.mjs',
    'config/scripts/claude-usage-benchmark-scanner.mjs',
    'config/scripts/counterbalanced-benchmark-schedule.mjs',
    'src/packages/process-host/src/run-process.ts',
    'src/packages/process-host/dist/run-process.js'
  ]
  return {
    esbuildVersion: version,
    baselineModules: BASELINE_MODULES,
    sha256: Object.fromEntries(
      await Promise.all(
        paths.map(async (path) => [
          path,
          createHash('sha256')
            .update(await readFile(join(ROOT, ...path.split('/'))))
            .digest('hex')
        ])
      )
    )
  }
}

export async function readClaudeUsageBenchmarkBaselineSources() {
  const { runProcessSync } = await import('@orca/process-host')
  const readGit = (args) => {
    const result = runProcessSync({
      program: 'git',
      args,
      cwd: ROOT,
      timeoutMs: 10_000,
      maxOutputBytes: MIB,
      env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
    })
    assert.equal(result.code, 0, `Git baseline read failed: ${result.stderr}`)
    return result.stdout
  }
  const baselineRef = process.env.ORCA_CLAUDE_USAGE_APPEND_BENCH_BASELINE ?? 'HEAD'
  const baselineCommit = readGit(['rev-parse', '--verify', `${baselineRef}^{commit}`]).trim()
  const sources = new Map(
    BASELINE_MODULES.map((modulePath) => [
      join(ROOT, ...modulePath.split('/')),
      readGit(['show', `${baselineCommit}:${modulePath}`])
    ])
  )
  return { baselineRef, baselineCommit, sources }
}

export async function loadClaudeUsageBenchmarkScanner(
  home,
  baselineSources = new Map(),
  worktreeSources = new Map(),
  baselineCommit = null
) {
  const sourceFingerprints = {}
  const { runProcessSync } = await import('@orca/process-host')
  const result = await build({
    stdin: {
      contents: `export { scanClaudeUsageFiles } from './src/main/claude-usage/scanner';
        export { resetReadMetrics, readMetrics } from 'orca-benchmark-fs';
        export { open as openForBenchmark } from 'node:fs/promises';
        export { createReadStream as createReadStreamForBenchmark } from 'node:fs';`,
      resolveDir: ROOT,
      loader: 'ts'
    },
    platform: 'node',
    format: 'esm',
    bundle: true,
    write: false,
    plugins: [
      {
        name: 'isolated-transcript-filesystem',
        setup(bundler) {
          bundler.onLoad({ filter: /[/\\]src[/\\].*\.[cm]?[jt]sx?$/ }, async (args) => {
            const modulePath = relative(ROOT, args.path).split(sep).join('/')
            if (!modulePath.startsWith('src/')) {
              return undefined
            }
            if (typeof baselineCommit === 'string' && !baselineSources.has(args.path)) {
              const pinned = runProcessSync({
                program: 'git',
                args: ['show', `${String(baselineCommit)}:${modulePath}`],
                cwd: ROOT,
                timeoutMs: 10_000,
                maxOutputBytes: MIB,
                env: { ...process.env, ORCA_BACKGROUND_LAUNCH: '1' }
              })
              assert.equal(pinned.code, 0, `Pinned source read failed: ${pinned.stderr}`)
              assert.equal(pinned.outputTruncated, false, `Pinned source truncated: ${modulePath}`)
              baselineSources.set(args.path, pinned.stdout)
            }
            if (!baselineSources.has(args.path) && !worktreeSources.has(args.path)) {
              worktreeSources.set(args.path, readFile(args.path, 'utf8'))
            }
            const contents =
              baselineSources.get(args.path) ?? (await worktreeSources.get(args.path))
            sourceFingerprints[modulePath] = {
              source: baselineSources.has(args.path)
                ? (baselineCommit ?? 'baseline-map')
                : 'worktree',
              sha256: createHash('sha256').update(contents).digest('hex')
            }
            const extension = extname(args.path).slice(1)
            const loader =
              extension === 'tsx' || extension === 'jsx'
                ? extension
                : extension.endsWith('ts')
                  ? 'ts'
                  : 'js'
            return { contents, loader, resolveDir: dirname(args.path) }
          })
          bundler.onResolve(
            { filter: /^node:(fs(?:\/promises)?|os)$|^orca-benchmark-fs$/ },
            (args) => {
              if (args.path === 'orca-benchmark-fs') {
                return { path: 'node:fs', namespace: 'benchmark' }
              }
              if (args.namespace === 'benchmark') {
                return { path: args.path, external: true }
              }
              return { path: args.path, namespace: 'benchmark' }
            }
          )
          bundler.onResolve({ filter: /claude-profile-installed-router$/ }, () => ({
            path: 'profiles',
            namespace: 'benchmark'
          }))
          bundler.onLoad({ filter: /.*/, namespace: 'benchmark' }, (args) => {
            if (args.path === 'profiles') {
              return { contents: 'export const claudeProfileHistoryDirs = () => [];' }
            }
            if (args.path === 'node:os') {
              return {
                contents: `export * from 'node:os';
                  export const homedir = () => ${JSON.stringify(home)};`
              }
            }
            if (args.path === 'node:fs/promises') {
              return {
                contents: `import * as fs from 'node:fs/promises';
                  import { recordStream, recordRead } from 'orca-benchmark-fs';
                  export * from 'node:fs/promises';
                  export function readFile() {
                    throw new Error('Extend benchmark accounting before using fs.promises.readFile');
                  }
                  export async function open(path, ...args) {
                    const handle = await fs.open(path, ...args);
                    return new Proxy(handle, {
                      get(target, key) {
                        if (key === 'readFile' || key === 'readv') {
                          throw new Error('Extend benchmark accounting before using FileHandle.' + key);
                        }
                        const value = Reflect.get(target, key, target);
                        if (key === 'createReadStream') {
                          return (options) => recordStream(path, options, value.call(target, options));
                        }
                        if (key === 'read') {
                          return async (...readArgs) => {
                            const result = await value.apply(target, readArgs);
                            const options = Buffer.isBuffer(readArgs[0]) ? null : readArgs[0];
                            recordRead(path, options?.position ?? readArgs[3] ?? null, result.bytesRead);
                            return result;
                          };
                        }
                        return typeof value === 'function' ? value.bind(target) : value;
                      }
                    });
                  }`
              }
            }
            return {
              contents: `import * as fs from 'node:fs';
                export * from 'node:fs';
                let reads = [];
                export function readFile() {
                  throw new Error('Extend benchmark accounting before using fs.readFile');
                }
                export function readFileSync() {
                  throw new Error('Extend benchmark accounting before using fs.readFileSync');
                }
                export function recordStream(path, options, stream) {
                  reads.push({ stream, kind: 'stream', path: String(path), start: options?.start ?? 0,
                    end: options?.end ?? null });
                  return stream;
                }
                export function recordRead(path, start, bytes) {
                  reads.push({ kind: 'read', path: String(path), start, bytes });
                }
                export function createReadStream(path, options) {
                  return recordStream(path, options, fs.createReadStream(path, options));
                }
                export function resetReadMetrics() { reads = []; }
                export function readMetrics() {
                  return reads.map(({ stream, ...range }) => ({ ...range, bytes: stream?.bytesRead ?? range.bytes }));
                }`
            }
          })
        }
      }
    ]
  })
  return {
    bundleSha256: createHash('sha256').update(result.outputFiles[0].text).digest('hex'),
    scanner: await import(
      `data:text/javascript;base64,${Buffer.from(result.outputFiles[0].text).toString('base64')}`
    ),
    sourceFingerprints: Object.fromEntries(
      Object.entries(sourceFingerprints).sort(([left], [right]) => left.localeCompare(right))
    )
  }
}

export async function verifyClaudeUsageBenchmarkReadAccounting(scanner, transcriptPath) {
  const payload = Buffer.from('工具結果 🌊'.repeat(5_000))
  await writeFile(transcriptPath, payload)
  scanner.resetReadMetrics()
  const handle = await scanner.openForBenchmark(transcriptPath, 'r')
  try {
    await handle.read(Buffer.alloc(11), 0, 11, 1)
    const stream = handle.createReadStream({ start: 0, end: payload.length - 1, autoClose: false })
    for await (const chunk of stream) {
      void chunk
    }
    const reads = scanner.readMetrics()
    assert.equal(
      reads.reduce((sum, read) => sum + read.bytes, 0),
      payload.length + 11
    )
    assert.equal(reads.filter((read) => read.kind === 'read').length, 1)
    assert.equal(reads.filter((read) => read.kind === 'stream').length, 1)
  } finally {
    await handle.close()
  }
  for await (const chunk of scanner.createReadStreamForBenchmark(transcriptPath)) {
    void chunk
  }
  assert.equal(
    scanner.readMetrics().reduce((sum, read) => sum + read.bytes, 0),
    payload.length * 2 + 11
  )
}

export async function measureClaudeUsageBenchmarkScan(
  scanner,
  transcriptPaths,
  previousProcessedFiles = []
) {
  scanner.resetReadMetrics()
  const start = performance.now()
  const result = await scanner.scanClaudeUsageFiles([], previousProcessedFiles, undefined, [])
  const durationMs = performance.now() - start
  const reads = scanner.readMetrics()
  const allowedPaths = new Set(
    typeof transcriptPaths === 'string' ? [transcriptPaths] : transcriptPaths
  )
  assert(
    reads.every((read) => allowedPaths.has(read.path)),
    'scanner read an unexpected file'
  )
  const serializationStart = performance.now()
  const serializedCache = JSON.stringify(result.processedFiles)
  const cacheSerializationMs = performance.now() - serializationStart
  return {
    result,
    cache: JSON.parse(serializedCache),
    metrics: {
      durationMs,
      cacheSerializationMs,
      totalDurationMs: durationMs + cacheSerializationMs,
      bytesRead: reads.reduce((sum, read) => sum + read.bytes, 0),
      directReadBytes: reads
        .filter((read) => read.kind === 'read')
        .reduce((sum, read) => sum + read.bytes, 0),
      readCount: reads.length,
      streamCount: reads.filter((read) => read.kind === 'stream').length,
      cachedStateBytes: Buffer.byteLength(serializedCache)
    }
  }
}