Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

4.9 KB
Raw Blame
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { join } from 'node:path'

export const SERVE_SUPERVISOR_FAILURE_WINDOW_MS = 15 * 60_000
/** This many failures inside the window and the supervisor gives up until a new build or --force. */
export const SERVE_SUPERVISOR_FAILURE_LIMIT = 6
export const SERVE_SUPERVISOR_BACKOFF_BASE_MS = 1_000
export const SERVE_SUPERVISOR_BACKOFF_MAX_MS = 30_000
export const SERVE_SUPERVISOR_STANDBY_BASE_MS = 5_000
export const SERVE_SUPERVISOR_STANDBY_MAX_MS = 60_000
export const SERVE_GAVE_UP_ADVICE =
  'Fix the cause, then run `orca serve service install` again (or, outside a service, `orca serve --supervise --force` once).'

/** Persist across service-manager restarts; launchd has no start limit. */
export type ServeSupervisorBrake = {
  schemaVersion: 1
  build: string
  failures: number[]
  gaveUp: { at: number; reason: string } | null
}

export type SupervisedExitAction =
  | { kind: 'restart'; delayMs: number }
  | { kind: 'give-up'; reason: string }

export function serveSupervisorBrakePath(userDataPath: string): string {
  return join(userDataPath, 'serve-supervisor-brake.json')
}

export function emptyServeSupervisorBrake(build: string): ServeSupervisorBrake {
  return { schemaVersion: 1, build, failures: [], gaveUp: null }
}

export function brakeForBuild(
  record: ServeSupervisorBrake | null,
  build: string
): ServeSupervisorBrake {
  return record && record.build === build ? record : emptyServeSupervisorBrake(build)
}

export function recordSupervisedFailure(
  record: ServeSupervisorBrake,
  input: { now: number; reason: string; random: () => number }
): { record: ServeSupervisorBrake; action: SupervisedExitAction } {
  const failures = [
    ...record.failures.filter(
      (at) => at <= input.now && input.now - at < SERVE_SUPERVISOR_FAILURE_WINDOW_MS
    ),
    input.now
  ]
  if (failures.length >= SERVE_SUPERVISOR_FAILURE_LIMIT) {
    const reason = `Orca serve failed ${failures.length} times in ${SERVE_SUPERVISOR_FAILURE_WINDOW_MS / 60_000} minutes; the supervisor stopped restarting it. Last failure: ${input.reason}`
    return {
      record: { ...record, failures, gaveUp: { at: input.now, reason } },
      action: { kind: 'give-up', reason }
    }
  }
  return {
    record: { ...record, failures },
    action: {
      kind: 'restart',
      delayMs: jitter(
        Math.min(
          SERVE_SUPERVISOR_BACKOFF_MAX_MS,
          SERVE_SUPERVISOR_BACKOFF_BASE_MS * 2 ** (failures.length - 1)
        ),
        input.random
      )
    }
  }
}

/** While another process serves the profile: wait longer each time, never count it as a failure. */
export function standbyDelayMs(attempt: number, random: () => number): number {
  return jitter(
    Math.min(SERVE_SUPERVISOR_STANDBY_MAX_MS, SERVE_SUPERVISOR_STANDBY_BASE_MS * 2 ** attempt),
    random
  )
}

function jitter(ms: number, random: () => number): number {
  // Jitter prevents services that failed together from retrying in lockstep.
  return Math.round(ms * (0.8 + 0.4 * random()))
}

export function parseServeSupervisorBrake(value: unknown): ServeSupervisorBrake | null {
  if (typeof value !== 'object' || value === null) {
    return null
  }
  if (
    !('schemaVersion' in value) ||
    value.schemaVersion !== 1 ||
    !('build' in value) ||
    typeof value.build !== 'string' ||
    !('failures' in value) ||
    !Array.isArray(value.failures) ||
    !('gaveUp' in value)
  ) {
    return null
  }
  const failures = value.failures.filter((at): at is number => typeof at === 'number')
  const gaveUp = value.gaveUp
  const parsedGaveUp =
    typeof gaveUp === 'object' &&
    gaveUp !== null &&
    'at' in gaveUp &&
    typeof gaveUp.at === 'number' &&
    'reason' in gaveUp &&
    typeof gaveUp.reason === 'string'
      ? { at: gaveUp.at, reason: gaveUp.reason }
      : null
  return { schemaVersion: 1, build: value.build, failures, gaveUp: parsedGaveUp }
}

export function readServeSupervisorBrake(userDataPath: string): ServeSupervisorBrake | null {
  try {
    return parseServeSupervisorBrake(
      JSON.parse(readFileSync(serveSupervisorBrakePath(userDataPath), 'utf8'))
    )
  } catch {
    return null
  }
}

export function writeServeSupervisorBrake(
  userDataPath: string,
  record: ServeSupervisorBrake
): void {
  const target = serveSupervisorBrakePath(userDataPath)
  const temporary = `${target}.${process.pid}.tmp`
  try {
    // The server may fail before creating its profile.
    mkdirSync(userDataPath, { recursive: true, mode: 0o700 })
    writeFileSync(temporary, JSON.stringify(record), { mode: 0o600 })
    renameSync(temporary, target)
  } catch (error) {
    // Fail open: an unwritable profile only loses the count across a supervisor restart.
    process.stderr.write(`[serve] could not record supervisor restarts: ${String(error)}\n`)
  }
}

export function clearServeSupervisorBrake(userDataPath: string): void {
  try {
    rmSync(serveSupervisorBrakePath(userDataPath), { force: true })
  } catch {}
}