Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
40.6 KB
Raw Blame
import { randomUUID } from "node:crypto"
import { Daytona, DaytonaConflictError, DaytonaNotFoundError, type CreateSandboxFromImageParams, type CreateSandboxFromSnapshotParams, type Sandbox } from "@daytonaio/sdk"
import {
  renderCheckpointExistsCommand,
  renderCheckpointFlushCommand,
  renderOmniRushBootstrapCommand,
  renderRestoreMarkerExistsCommand,
  shellQuote,
  type OmniRushBootstrapConfig,
  type OmniRushCheckpointConfig,
} from "@omnirush-ee/cloud-runtime/bootstrap"
import { eq } from "@omnirush-ee/den-db/drizzle"
import { DaytonaSandboxTable, WorkerTable } from "@omnirush-ee/den-db/schema"
import { createDenTypeId } from "@omnirush-ee/utils/typeid"
import { db } from "../db.js"
import { env } from "../env.js"
import { appLogger } from "../observability/logger.js"

type WorkerId = typeof DaytonaSandboxTable.$inferSelect.worker_id

type ProvisionInput = {
  workerId: WorkerId
  name: string
  hostToken: string
  clientToken: string
  activityToken: string
}
type SandboxNameInput = Pick<ProvisionInput, "workerId" | "name">

type ProvisionedInstance = {
  provider: string
  url: string
  status: "provisioning" | "healthy"
  region?: string
  imageVersion?: string | null
}

export class DaytonaSandboxMissingError extends Error {
  constructor(message: string) {
    super(message)
    this.name = "DaytonaSandboxMissingError"
  }
}

export type StopWorkerOnDaytonaResult =
  | { status: "no_sandbox" }
  | { status: "stopped" }

type DaytonaCreateParams = CreateSandboxFromImageParams | CreateSandboxFromSnapshotParams
type DaytonaVolumeRuntime = {
  id: string
  state?: string | null
}
type DaytonaSessionCommand = {
  exitCode?: number | null
}
type DaytonaSessionCommandLogs = {
  stdout?: string | null
  stderr?: string | null
}
type DaytonaSessionCommandResult = {
  cmdId: string
}
export type DaytonaSandboxRuntime = {
  id: string
  state: string | null
  target: string | null
  refreshData: () => Promise<unknown>
  start: (timeout?: number) => Promise<unknown>
  stop: (timeout?: number) => Promise<unknown>
  delete: (timeout?: number) => Promise<unknown>
  getSignedPreviewUrl: (port: number, expiresInSeconds?: number) => Promise<{ url: string }>
  process: {
    createSession: (sessionId: string) => Promise<unknown>
    executeSessionCommand: (sessionId: string, request: { command: string; runAsync: boolean }, timeout?: number) => Promise<DaytonaSessionCommandResult>
    getSessionCommand: (sessionId: string, commandId: string) => Promise<DaytonaSessionCommand>
    getSessionCommandLogs: (sessionId: string, commandId: string) => Promise<DaytonaSessionCommandLogs>
  }
}
type UpsertDaytonaSandboxInput = {
  workerId: WorkerId
  sandboxId: string
  workspaceVolumeId: string
  dataVolumeId: string
  signedPreviewUrl: string
  signedPreviewUrlExpiresAt: Date
  region: string | null
}
export type DaytonaProvisioningRuntime = {
  getVolume: (name: string, create?: boolean) => Promise<DaytonaVolumeRuntime>
  getSandbox: (sandboxIdOrName: string) => Promise<DaytonaSandboxRuntime>
  createSandbox: (params: DaytonaCreateParams) => Promise<DaytonaSandboxRuntime>
  upsertSandbox: (input: UpsertDaytonaSandboxInput) => Promise<void>
  checkpointExists: (input: { workerId: WorkerId; sharedVolume: DaytonaVolumeRuntime }) => Promise<boolean>
  verifyRestoreMarker: (sandbox: DaytonaSandboxRuntime) => Promise<boolean>
  waitForHealth: typeof waitForHealth
  now?: () => number
}

const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
const maxSignedPreviewExpirySeconds = 60 * 60 * 24
const signedPreviewRefreshLeadMs = 5 * 60 * 1000
const wakeStartMaxAttempts = 3
const wakeStartRetryBackoffMs = 250
const wakeStartStateChangeTimeoutMs = 60_000
const healthRequestTimeoutMs = 5_000
const createConflictLookupMaxAttempts = 6
const createConflictLookupBackoffMs = 2_000
const logger = appLogger.child({ component: "daytona_provisioner" })

const slug = (value: string) =>
  value
    .toLowerCase()
    .replace(/[^a-z0-9-]+/g, "-")
    .replace(/-+/g, "-")
    .replace(/^-|-$/g, "")

function createDaytonaClient() {
  return new Daytona({
    apiKey: env.daytona.apiKey,
    apiUrl: env.daytona.apiUrl,
    ...(env.daytona.target ? { target: env.daytona.target } : {}),
  })
}

async function listDaytonaSandboxIdsByLabels(labels: Record<string, string>) {
  const daytona = createDaytonaClient()
  const ids: string[] = []

  for await (const sandbox of daytona.list({ labels, limit: 100 })) {
    ids.push(sandbox.id)
  }

  return ids
}

function normalizedSignedPreviewExpirySeconds() {
  return Math.max(
    1,
    Math.min(env.daytona.signedPreviewExpiresSeconds, maxSignedPreviewExpirySeconds),
  )
}

function signedPreviewRefreshAt(expiresInSeconds: number, issuedAtMs = Date.now()) {
  return new Date(
    issuedAtMs + Math.max(0, expiresInSeconds * 1000 - signedPreviewRefreshLeadMs),
  )
}

function workerActivityHeartbeatUrl(workerId: WorkerId) {
  const base = env.workerActivityBaseUrl.replace(/\/+$/, "")
  return `${base}/v1/workers/${encodeURIComponent(workerId)}/activity-heartbeat`
}

function assertDaytonaConfig() {
  if (!env.daytona.apiKey) {
    throw new Error("DAYTONA_API_KEY is required for daytona provisioner")
  }
}

function isDaytonaNotFoundError(error: unknown) {
  if (!(error instanceof Error)) {
    return false
  }

  if (error instanceof DaytonaNotFoundError || error.name === "DaytonaNotFoundError") {
    return true
  }

  const message = error.message.toLowerCase()
  return message.includes("not found") || message.includes("404")
}

export function isDaytonaSandboxMissingError(error: unknown) {
  return error instanceof DaytonaSandboxMissingError
    || (error instanceof Error && error.name === "DaytonaSandboxMissingError")
}

export function isDaytonaConflictError(error: unknown) {
  return error instanceof DaytonaConflictError
    || (error instanceof Error && error.name === "DaytonaConflictError")
}

function workerHint(workerId: WorkerId) {
  return workerId.replace(/-/g, "").slice(0, 12)
}

function sandboxLabels(workerId: WorkerId) {
  return {
    "omnirush.den.provider": "daytona",
    "omnirush.den.worker-id": workerId,
  }
}

export function daytonaSandboxName(input: SandboxNameInput) {
  return slug(
    `${env.daytona.sandboxNamePrefix}-${input.name}-${workerHint(input.workerId)}`,
  ).slice(0, 63)
}

function currentDaytonaImageVersion() {
  return env.daytona.snapshot ?? null
}

function snapshotShortVersion(snapshot: string) {
  return slug(snapshot).slice(0, 24) || "snapshot"
}

export function daytonaSandboxNameForSnapshot(input: SandboxNameInput, snapshot: string) {
  const shortVersion = snapshotShortVersion(snapshot)
  const base = daytonaSandboxName(input)
  const baseLength = Math.max(1, 63 - shortVersion.length - 1)
  return slug(`${base.slice(0, baseLength)}-${shortVersion}`).slice(0, 63)
}

export function currentDaytonaSandboxName(input: SandboxNameInput) {
  const snapshot = currentDaytonaImageVersion()
  return snapshot ? daytonaSandboxNameForSnapshot(input, snapshot) : daytonaSandboxName(input)
}

function daytonaSandboxLookupNames(input: ProvisionInput) {
  const names: string[] = []
  for (const name of [currentDaytonaSandboxName(input), daytonaSandboxName(input)]) {
    if (!names.includes(name)) {
      names.push(name)
    }
  }
  return names
}

function recoveryDaytonaSandboxName(input: SandboxNameInput) {
  const suffix = randomUUID().replace(/-/g, "").slice(0, 8)
  const base = currentDaytonaSandboxName(input)
  const baseLength = Math.max(1, 63 - suffix.length - 10)
  return slug(`${base.slice(0, baseLength)}-recovery-${suffix}`).slice(0, 63)
}

function isStoppedSandboxState(state: string | null) {
  return state?.toLowerCase() === "stopped"
}

function isStartedSandboxState(state: string | null) {
  return state?.toLowerCase() === "started"
}

function daytonaErrorMessage(error: unknown) {
  return error instanceof Error ? error.message.toLowerCase() : ""
}

function isDaytonaStateChangeInProgressConflict(error: unknown) {
  const message = daytonaErrorMessage(error)
  return isDaytonaConflictError(error) && message.includes("state change") && message.includes("progress")
}

function isTransientDaytonaStartError(error: unknown) {
  const message = daytonaErrorMessage(error)
  if (!message) {
    return false
  }

  return /\b5\d\d\b/.test(message)
    || [
      "econnreset",
      "econnrefused",
      "etimedout",
      "enotfound",
      "fetch failed",
      "network",
      "socket hang up",
      "temporarily unavailable",
      "timeout",
    ].some((marker) => message.includes(marker))
}

async function waitForDaytonaStartConflictToSettle(sandbox: DaytonaSandboxRuntime) {
  const startedAt = Date.now()
  let sawTransitionState = false

  while (Date.now() - startedAt < wakeStartStateChangeTimeoutMs) {
    await sandbox.refreshData()

    if (isStartedSandboxState(sandbox.state)) {
      return "started"
    }

    if (isStoppedSandboxState(sandbox.state) && (sawTransitionState || Date.now() - startedAt >= env.daytona.pollIntervalMs)) {
      return "stopped"
    }

    if (!isStoppedSandboxState(sandbox.state)) {
      sawTransitionState = true
    }

    await sleep(env.daytona.pollIntervalMs)
  }

  throw new Error(`Timed out waiting for Daytona sandbox ${sandbox.id} state change to settle`)
}

function sharedVolumeName() {
  return slug(env.daytona.sharedVolumeName).slice(0, 63)
}

function workerVolumeRootSubpath(workerId: WorkerId) {
  return `workers/${workerId}`
}

function workspaceVolumeSubpath(workerId: WorkerId) {
  return `${workerVolumeRootSubpath(workerId)}/workspace`
}

function dataVolumeSubpath(workerId: WorkerId) {
  return `${workerVolumeRootSubpath(workerId)}/data`
}

function sharedVolumeMounts(workerId: WorkerId, volumeId: string) {
  return [
    {
      volumeId,
      mountPath: env.daytona.workspaceMountPath,
      subpath: workspaceVolumeSubpath(workerId),
    },
    {
      volumeId,
      mountPath: env.daytona.dataMountPath,
      subpath: dataVolumeSubpath(workerId),
    },
  ]
}

function checkpointConfig(): OmniRushCheckpointConfig {
  return {
    dataMountPath: env.daytona.dataMountPath,
    runtimeDataPath: env.daytona.runtimeDataPath,
    runtimeWorkspacePath: env.daytona.runtimeWorkspacePath,
    sidecarDir: env.daytona.sidecarDir,
    intervalSeconds: env.daytona.checkpointIntervalSeconds,
    keep: env.daytona.checkpointKeep,
  }
}

function bootstrapConfig(input: ProvisionInput): OmniRushBootstrapConfig {
  return {
    ...checkpointConfig(),
    workspaceMountPath: env.daytona.workspaceMountPath,
    port: env.daytona.omnirushPort,
    workerId: input.workerId,
    clientToken: input.clientToken,
    hostToken: input.hostToken,
    activityHeartbeat: {
      url: workerActivityHeartbeatUrl(input.workerId),
      token: input.activityToken,
    },
    runtimeProvider: "daytona",
    imageDescription: "Daytona runtime image",
    rebuildHint: "rebuild and republish the Daytona snapshot",
  }
}

export function checkpointFlushCommand() {
  return renderCheckpointFlushCommand(checkpointConfig())
}

export function buildOmniRushStartCommand(input: ProvisionInput) {
  return renderOmniRushBootstrapCommand(bootstrapConfig(input))
}

async function waitForVolumeReady(getVolume: DaytonaProvisioningRuntime["getVolume"], name: string, timeoutMs: number) {
  const startedAt = Date.now()

  while (Date.now() - startedAt < timeoutMs) {
    const volume = await getVolume(name)
    if (volume.state === "ready") {
      return volume
    }
    await sleep(env.daytona.pollIntervalMs)
  }

  throw new Error(`Timed out waiting for Daytona volume ${name} to become ready`)
}

function buildVolumeCleanupCommand(workerId: WorkerId) {
  return [
    "node -e",
    shellQuote(
      [
        'const fs = require("node:fs")',
        'const path = require("node:path")',
        'for (const dir of process.argv.slice(1)) {',
        '  fs.mkdirSync(dir, { recursive: true })',
        '  for (const entry of fs.readdirSync(dir)) {',
        '    fs.rmSync(path.join(dir, entry), { recursive: true, force: true })',
        '  }',
        '}',
      ].join("; "),
    ),
    shellQuote(env.daytona.workspaceMountPath),
    shellQuote(env.daytona.dataMountPath),
  ].join(" ")
}

async function cleanupWorkerDataOnDaytona(daytona: Daytona, workerId: WorkerId) {
  let sharedVolume

  try {
    sharedVolume = await waitForVolumeReady(
      (name, create) => daytona.volume.get(name, create),
      sharedVolumeName(),
      env.daytona.createTimeoutSeconds * 1000,
    )
  } catch (error) {
    logger.warn("failed to resolve shared Daytona volume", { worker_id: workerId, error })
    return
  }

  let cleanupSandbox: Awaited<ReturnType<typeof daytona.create>> | null = null

  try {
    cleanupSandbox = await daytona.create(
      {
        name: slug(`den-daytona-cleanup-${workerHint(workerId)}`).slice(0, 63),
        image: env.daytona.image,
        public: false,
        autoStopInterval: 0,
        autoArchiveInterval: 0,
        autoDeleteInterval: 0,
        ephemeral: true,
        envVars: {
          DEN_RUNTIME_PROVIDER: "daytona-cleanup",
          DEN_WORKER_ID: workerId,
        },
        resources: {
          cpu: 1,
          memory: 1,
          disk: 4,
        },
        volumes: sharedVolumeMounts(workerId, sharedVolume.id),
      },
      { timeout: env.daytona.createTimeoutSeconds },
    )

    const result = await cleanupSandbox.process.executeCommand(
      buildVolumeCleanupCommand(workerId),
      undefined,
      undefined,
      env.daytona.deleteTimeoutSeconds,
    )

    if (result.exitCode !== 0) {
      throw new Error(result.result?.trim() || `cleanup command exited with ${result.exitCode}`)
    }
  } catch (error) {
    logger.warn("failed to cleanup Daytona worker data", { worker_id: workerId, error })
  } finally {
    if (cleanupSandbox) {
      await cleanupSandbox.delete(env.daytona.deleteTimeoutSeconds).catch((error) => {
        logger.warn("failed to delete Daytona cleanup sandbox", { worker_id: workerId, error })
      })
    }
  }
}

export async function waitForHealth(url: string, timeoutMs: number, sandbox: DaytonaSandboxRuntime, sessionId: string, commandId: string) {
  const startedAt = Date.now()

  while (Date.now() - startedAt < timeoutMs) {
    try {
      const remainingMs = timeoutMs - (Date.now() - startedAt)
      const response = await fetch(`${url.replace(/\/$/, "")}/health`, {
        method: "GET",
        signal: AbortSignal.timeout(Math.max(1, Math.min(remainingMs, healthRequestTimeoutMs))),
      })
      if (response.ok) {
        return
      }
    } catch {
      // ignore transient startup failures
    }

    try {
      const command = await sandbox.process.getSessionCommand(sessionId, commandId)
      if (typeof command.exitCode === "number" && command.exitCode !== 0) {
        const logs = await sandbox.process.getSessionCommandLogs(sessionId, commandId)
        throw new Error(
          [
            `omnirush session exited with ${command.exitCode}`,
            logs.stdout?.trim() ? `stdout:\n${logs.stdout.trim().slice(-4000)}` : "",
            logs.stderr?.trim() ? `stderr:\n${logs.stderr.trim().slice(-4000)}` : "",
          ]
            .filter(Boolean)
            .join("\n\n"),
        )
      }
    } catch (error) {
      if (error instanceof Error && error.message.startsWith("omnirush session exited")) {
        throw error
      }
    }

    const remainingAfterProbeMs = timeoutMs - (Date.now() - startedAt)
    if (remainingAfterProbeMs > 0) {
      await sleep(Math.min(env.daytona.pollIntervalMs, remainingAfterProbeMs))
    }
  }

  const logs = await sandbox.process.getSessionCommandLogs(sessionId, commandId).catch(
    () => null,
  )
  throw new Error(
    [
      `Timed out waiting for Daytona worker health at ${url.replace(/\/$/, "")}/health`,
      logs?.stdout?.trim() ? `stdout:\n${logs.stdout.trim().slice(-4000)}` : "",
      logs?.stderr?.trim() ? `stderr:\n${logs.stderr.trim().slice(-4000)}` : "",
    ]
      .filter(Boolean)
      .join("\n\n"),
  )
}

async function waitForSessionCommandExit(sandbox: DaytonaSandboxRuntime, sessionId: string, commandId: string, timeoutMs: number) {
  const startedAt = Date.now()

  while (Date.now() - startedAt < timeoutMs) {
    const command = await sandbox.process.getSessionCommand(sessionId, commandId)
    if (typeof command.exitCode === "number") {
      return command.exitCode
    }

    await sleep(env.daytona.pollIntervalMs)
  }

  return null
}

async function runSandboxShellCommand(sandbox: DaytonaSandboxRuntime, sessionId: string, command: string, timeoutSeconds: number) {
  await sandbox.process.createSession(sessionId)
  const result = await sandbox.process.executeSessionCommand(
    sessionId,
    { command: `sh -lc ${shellQuote(command)}`, runAsync: false },
    timeoutSeconds,
  )
  return waitForSessionCommandExit(sandbox, sessionId, result.cmdId, timeoutSeconds * 1000)
}

function checkpointExistsCommand() {
  return renderCheckpointExistsCommand(checkpointConfig())
}

function restoreMarkerExistsCommand() {
  return renderRestoreMarkerExistsCommand(checkpointConfig())
}

async function verifyRestoreMarker(sandbox: DaytonaSandboxRuntime) {
  const exitCode = await runSandboxShellCommand(
    sandbox,
    `omnirush-restore-verify-${Date.now()}`,
    restoreMarkerExistsCommand(),
    env.daytona.createTimeoutSeconds,
  )
  return exitCode === 0
}

async function upsertDaytonaSandbox(input: UpsertDaytonaSandboxInput) {
  const existing = await db
    .select({ id: DaytonaSandboxTable.id })
    .from(DaytonaSandboxTable)
    .where(eq(DaytonaSandboxTable.worker_id, input.workerId))
    .limit(1)

  if (existing.length > 0) {
    await db
      .update(DaytonaSandboxTable)
      .set({
        sandbox_id: input.sandboxId,
        workspace_volume_id: input.workspaceVolumeId,
        data_volume_id: input.dataVolumeId,
        signed_preview_url: input.signedPreviewUrl,
        signed_preview_url_expires_at: input.signedPreviewUrlExpiresAt,
        region: input.region,
      })
      .where(eq(DaytonaSandboxTable.worker_id, input.workerId))
    return
  }

  await db.insert(DaytonaSandboxTable).values({
    id: createDenTypeId("daytonaSandbox"),
    worker_id: input.workerId,
    sandbox_id: input.sandboxId,
    workspace_volume_id: input.workspaceVolumeId,
    data_volume_id: input.dataVolumeId,
    signed_preview_url: input.signedPreviewUrl,
    signed_preview_url_expires_at: input.signedPreviewUrlExpiresAt,
    region: input.region,
  })
}

export async function getDaytonaSandboxRecord(workerId: WorkerId) {
  const rows = await db
    .select()
    .from(DaytonaSandboxTable)
    .where(eq(DaytonaSandboxTable.worker_id, workerId))
    .limit(1)

  return rows[0] ?? null
}

export async function inspectDaytonaSandbox(workerId: WorkerId) {
  assertDaytonaConfig()

  const record = await getDaytonaSandboxRecord(workerId)
  if (!record) {
    return null
  }

  const daytona = createDaytonaClient()
  try {
    const sandbox = await daytona.get(record.sandbox_id)
    await sandbox.refreshData()
    return { state: sandbox.state ?? null }
  } catch (error) {
    if (isDaytonaNotFoundError(error)) {
      return null
    }

    throw error
  }
}

export async function refreshDaytonaSignedPreview(workerId: WorkerId) {
  assertDaytonaConfig()

  const record = await getDaytonaSandboxRecord(workerId)
  if (!record) {
    return null
  }

  const daytona = createDaytonaClient()
  const sandbox = await daytona.get(record.sandbox_id)
  await sandbox.refreshData()

  const expiresInSeconds = normalizedSignedPreviewExpirySeconds()
  const issuedAtMs = Date.now()
  const preview = await sandbox.getSignedPreviewUrl(env.daytona.omnirushPort, expiresInSeconds)
  const expiresAt = signedPreviewRefreshAt(expiresInSeconds, issuedAtMs)

  await db
    .update(DaytonaSandboxTable)
    .set({
      signed_preview_url: preview.url,
      signed_preview_url_expires_at: expiresAt,
      region: sandbox.target,
    })
    .where(eq(DaytonaSandboxTable.worker_id, workerId))

  return {
    ...record,
    signed_preview_url: preview.url,
    signed_preview_url_expires_at: expiresAt,
    region: sandbox.target,
  }
}

function toDaytonaSandboxRuntime(sandbox: Sandbox): DaytonaSandboxRuntime {
  return {
    get id() {
      return sandbox.id
    },
    get state() {
      return sandbox.state ?? null
    },
    get target() {
      return sandbox.target ?? null
    },
    refreshData: () => sandbox.refreshData(),
    start: (timeout) => sandbox.start(timeout),
    stop: (timeout) => sandbox.stop(timeout),
    delete: (timeout) => sandbox.delete(timeout),
    getSignedPreviewUrl: (port, expiresInSeconds) => sandbox.getSignedPreviewUrl(port, expiresInSeconds),
    process: {
      createSession: (sessionId) => sandbox.process.createSession(sessionId),
      executeSessionCommand: (sessionId, request, timeout) => sandbox.process.executeSessionCommand(sessionId, request, timeout),
      getSessionCommand: (sessionId, commandId) => sandbox.process.getSessionCommand(sessionId, commandId),
      getSessionCommandLogs: (sessionId, commandId) => sandbox.process.getSessionCommandLogs(sessionId, commandId),
    },
  }
}

async function checkpointExistsOnDaytonaVolume(daytona: Daytona, workerId: WorkerId, sharedVolume: DaytonaVolumeRuntime) {
  let probeSandbox: DaytonaSandboxRuntime | null = null

  try {
    const suffix = randomUUID().replace(/-/g, "").slice(0, 8)
    probeSandbox = toDaytonaSandboxRuntime(await daytona.create(
      buildCheckpointProbeCreateParams({
        workerId,
        name: slug(`den-daytona-ckpt-${workerHint(workerId)}-${suffix}`).slice(0, 63),
        sharedVolume,
      }),
      { timeout: env.daytona.createTimeoutSeconds },
    ))

    const exitCode = await runSandboxShellCommand(
      probeSandbox,
      `omnirush-ckpt-probe-${workerHint(workerId)}-${Date.now()}`,
      checkpointExistsCommand(),
      env.daytona.createTimeoutSeconds,
    )
    return exitCode === 0
  } catch (error) {
    logger.warn("failed to inspect Daytona checkpoint volume", { worker_id: workerId, error })
    return false
  } finally {
    if (probeSandbox) {
      await probeSandbox.delete(env.daytona.deleteTimeoutSeconds).catch((error) => {
        logger.warn("failed to delete Daytona checkpoint probe sandbox", { worker_id: workerId, error })
      })
    }
  }
}

function buildCheckpointProbeCreateParams(input: { workerId: WorkerId; name: string; sharedVolume: DaytonaVolumeRuntime }): DaytonaCreateParams {
  const base = {
    name: input.name,
    public: false,
    autoStopInterval: 0,
    autoArchiveInterval: 0,
    autoDeleteInterval: 0,
    ephemeral: true,
    envVars: {
      DEN_RUNTIME_PROVIDER: "daytona-checkpoint-probe",
      DEN_WORKER_ID: input.workerId,
    },
    volumes: sharedVolumeMounts(input.workerId, input.sharedVolume.id),
  }

  if (env.daytona.snapshot) {
    return {
      ...base,
      snapshot: env.daytona.snapshot,
    }
  }

  return {
    ...base,
    image: env.daytona.image,
    resources: {
      cpu: 1,
      memory: 1,
      disk: 4,
    },
  }
}

function createDaytonaProvisioningRuntime(daytona: Daytona): DaytonaProvisioningRuntime {
  return {
    getVolume: (name, create) => daytona.volume.get(name, create),
    getSandbox: async (sandboxIdOrName) => toDaytonaSandboxRuntime(await daytona.get(sandboxIdOrName)),
    createSandbox: async (params) => {
      if ("image" in params) {
        return toDaytonaSandboxRuntime(await daytona.create(params, { timeout: env.daytona.createTimeoutSeconds }))
      }

      return toDaytonaSandboxRuntime(await daytona.create(params, { timeout: env.daytona.createTimeoutSeconds }))
    },
    upsertSandbox: upsertDaytonaSandbox,
    checkpointExists: (input) => checkpointExistsOnDaytonaVolume(daytona, input.workerId, input.sharedVolume),
    verifyRestoreMarker,
    waitForHealth,
  }
}

async function getSharedDaytonaVolume(runtime: DaytonaProvisioningRuntime) {
  const sharedVolumeNameValue = sharedVolumeName()
  await runtime.getVolume(sharedVolumeNameValue, true)
  return waitForVolumeReady(
    runtime.getVolume,
    sharedVolumeNameValue,
    env.daytona.createTimeoutSeconds * 1000,
  )
}

function buildDaytonaCreateParams(input: ProvisionInput, name: string, sharedVolume: DaytonaVolumeRuntime): DaytonaCreateParams {
  const labels = sandboxLabels(input.workerId)
  const base = {
    name,
    autoStopInterval: env.daytona.autoStopInterval,
    autoArchiveInterval: env.daytona.autoArchiveInterval,
    autoDeleteInterval: env.daytona.autoDeleteInterval,
    public: env.daytona.public,
    labels,
    envVars: {
      DEN_WORKER_ID: input.workerId,
      DEN_RUNTIME_PROVIDER: "daytona",
    },
    volumes: sharedVolumeMounts(input.workerId, sharedVolume.id),
  }

  if (env.daytona.snapshot) {
    return {
      ...base,
      snapshot: env.daytona.snapshot,
    }
  }

  return {
    ...base,
    image: env.daytona.image,
    resources: {
      cpu: env.daytona.resources.cpu,
      memory: env.daytona.resources.memory,
      disk: env.daytona.resources.disk,
    },
  }
}

async function getSandboxByName(runtime: DaytonaProvisioningRuntime, name: string) {
  try {
    const sandbox = await runtime.getSandbox(name)
    await sandbox.refreshData()
    return sandbox
  } catch (error) {
    if (isDaytonaNotFoundError(error)) {
      return null
    }

    throw error
  }
}

async function getSandboxAfterCreateConflict(
  runtime: DaytonaProvisioningRuntime,
  lookupNames: string[],
  wait: (ms: number) => Promise<unknown>,
) {
  for (let attempt = 1; attempt <= createConflictLookupMaxAttempts; attempt += 1) {
    for (const lookupName of lookupNames) {
      const sandbox = await getSandboxByName(runtime, lookupName)
      if (sandbox) return sandbox
    }

    if (attempt < createConflictLookupMaxAttempts) {
      await wait(createConflictLookupBackoffMs)
    }
  }

  return null
}

type StartedOmniRushProcess = {
  signedPreviewUrl: string
  signedPreviewUrlExpiresAt: Date
}

function provisionedInstance(url: string, region: string | null, imageVersion: string | null | undefined = currentDaytonaImageVersion()): ProvisionedInstance {
  return {
    provider: "daytona",
    url,
    status: "healthy",
    region: region ?? undefined,
    imageVersion,
  }
}

async function startOmniRushProcessOnDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  sandbox: DaytonaSandboxRuntime
  sessionId: string
}): Promise<StartedOmniRushProcess> {
  await input.sandbox.process.createSession(input.sessionId)
  const command = await input.sandbox.process.executeSessionCommand(
    input.sessionId,
    {
      command: buildOmniRushStartCommand(input.provisionInput),
      runAsync: true,
    },
    0,
  )

  const expiresInSeconds = normalizedSignedPreviewExpirySeconds()
  const issuedAtMs = input.runtime.now ? input.runtime.now() : Date.now()
  const preview = await input.sandbox.getSignedPreviewUrl(env.daytona.omnirushPort, expiresInSeconds)
  await input.runtime.waitForHealth(preview.url, env.daytona.healthcheckTimeoutMs, input.sandbox, input.sessionId, command.cmdId)
  return {
    signedPreviewUrl: preview.url,
    signedPreviewUrlExpiresAt: signedPreviewRefreshAt(expiresInSeconds, issuedAtMs),
  }
}

async function persistDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  sandbox: DaytonaSandboxRuntime
  started: StartedOmniRushProcess
  workspaceVolumeId: string
  dataVolumeId: string
}) {
  await input.runtime.upsertSandbox({
    workerId: input.provisionInput.workerId,
    sandboxId: input.sandbox.id,
    workspaceVolumeId: input.workspaceVolumeId,
    dataVolumeId: input.dataVolumeId,
    signedPreviewUrl: input.started.signedPreviewUrl,
    signedPreviewUrlExpiresAt: input.started.signedPreviewUrlExpiresAt,
    region: input.sandbox.target,
  })
}

async function startOmniRushOnDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  sandbox: DaytonaSandboxRuntime
  sessionId: string
  workspaceVolumeId: string
  dataVolumeId: string
  imageVersion?: string | null
}): Promise<ProvisionedInstance> {
  const started = await startOmniRushProcessOnDaytonaSandbox(input)
  await persistDaytonaSandbox({
    provisionInput: input.provisionInput,
    runtime: input.runtime,
    sandbox: input.sandbox,
    started,
    workspaceVolumeId: input.workspaceVolumeId,
    dataVolumeId: input.dataVolumeId,
  })

  return provisionedInstance(started.signedPreviewUrl, input.sandbox.target, input.imageVersion)
}

async function startDaytonaSandboxForWake(input: { workerId: WorkerId; sandbox: DaytonaSandboxRuntime }) {
  let lastError: unknown

  for (let attempt = 1; attempt <= wakeStartMaxAttempts; attempt += 1) {
    try {
      await input.sandbox.start(env.daytona.createTimeoutSeconds)
      return
    } catch (error) {
      lastError = error

      if (isDaytonaStateChangeInProgressConflict(error)) {
        logger.warn("Daytona sandbox start already in progress; waiting for convergence", { worker_id: input.workerId, sandbox_id: input.sandbox.id, error })
        const settledState = await waitForDaytonaStartConflictToSettle(input.sandbox)
        if (settledState === "started") {
          return
        }
        continue
      }

      if (!isTransientDaytonaStartError(error) || attempt === wakeStartMaxAttempts) {
        throw error
      }

      logger.warn("transient Daytona sandbox start failure; retrying", { worker_id: input.workerId, sandbox_id: input.sandbox.id, attempt, error })
      await sleep(wakeStartRetryBackoffMs)
      await input.sandbox.refreshData().catch(() => undefined)
      if (isStartedSandboxState(input.sandbox.state)) {
        return
      }
    }
  }

  throw lastError ?? new Error(`Daytona sandbox ${input.sandbox.id} start failed`)
}

function shouldRecycleDaytonaSandbox(input: { workerImageVersion: string | null; sandboxState: string | null }) {
  const imageVersion = currentDaytonaImageVersion()
  return Boolean(imageVersion && input.workerImageVersion !== imageVersion && isStoppedSandboxState(input.sandboxState))
}

async function wakeExistingDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  sandbox: DaytonaSandboxRuntime
  workspaceVolumeId: string
  dataVolumeId: string
  imageVersion?: string | null
}) {
  if (isStartedSandboxState(input.sandbox.state)) {
    // A failed health probe means an already-running OmniRush.ai process cannot be
    // trusted. Restart the sandbox before launching a new process so recovery
    // cannot leave two servers competing for the same port and state files.
    await input.sandbox.stop(env.daytona.stopTimeoutSeconds ?? env.daytona.deleteTimeoutSeconds)
    await input.sandbox.refreshData()
  }
  await startDaytonaSandboxForWake({ workerId: input.provisionInput.workerId, sandbox: input.sandbox })

  return startOmniRushOnDaytonaSandbox({
    provisionInput: input.provisionInput,
    runtime: input.runtime,
    sandbox: input.sandbox,
    sessionId: `omnirush-wake-${workerHint(input.provisionInput.workerId)}-${Date.now()}`,
    workspaceVolumeId: input.workspaceVolumeId,
    dataVolumeId: input.dataVolumeId,
    imageVersion: input.imageVersion,
  })
}

async function recycleDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  oldSandbox: DaytonaSandboxRuntime
  sharedVolume: DaytonaVolumeRuntime
  oldWorkspaceVolumeId: string
  oldDataVolumeId: string
  oldImageVersion: string | null
  replacementName?: string
  requireRestoreMarker?: boolean
}): Promise<ProvisionedInstance> {
  let replacementSandbox: DaytonaSandboxRuntime | null = null

  try {
    replacementSandbox = await input.runtime.createSandbox(
      buildDaytonaCreateParams(
        input.provisionInput,
        input.replacementName ?? currentDaytonaSandboxName(input.provisionInput),
        input.sharedVolume,
      ),
    )
    const started = await startOmniRushProcessOnDaytonaSandbox({
      provisionInput: input.provisionInput,
      runtime: input.runtime,
      sandbox: replacementSandbox,
      sessionId: `omnirush-recycle-${workerHint(input.provisionInput.workerId)}-${Date.now()}`,
    })
    const restored = input.requireRestoreMarker === false
      ? true
      : await input.runtime.verifyRestoreMarker(replacementSandbox)
    if (!restored) {
      throw new Error("Daytona replacement did not restore an OmniRush.ai checkpoint")
    }

    await persistDaytonaSandbox({
      provisionInput: input.provisionInput,
      runtime: input.runtime,
      sandbox: replacementSandbox,
      started,
      workspaceVolumeId: input.sharedVolume.id,
      dataVolumeId: input.sharedVolume.id,
    })
    await input.oldSandbox.delete(env.daytona.deleteTimeoutSeconds)
    return provisionedInstance(started.signedPreviewUrl, replacementSandbox.target)
  } catch (error) {
    if (replacementSandbox) {
      await replacementSandbox.delete(env.daytona.deleteTimeoutSeconds).catch((deleteError) => {
        logger.warn("failed to delete failed Daytona replacement sandbox", { worker_id: input.provisionInput.workerId, error: deleteError })
      })
    }

    logger.warn("Daytona sandbox recycle failed; waking existing sandbox", { worker_id: input.provisionInput.workerId, error })
    return wakeExistingDaytonaSandbox({
      provisionInput: input.provisionInput,
      runtime: input.runtime,
      sandbox: input.oldSandbox,
      workspaceVolumeId: input.oldWorkspaceVolumeId,
      dataVolumeId: input.oldDataVolumeId,
      imageVersion: input.oldImageVersion,
    })
  }
}

async function adoptDaytonaSandbox(input: {
  provisionInput: ProvisionInput
  runtime: DaytonaProvisioningRuntime
  sandbox: DaytonaSandboxRuntime
  sharedVolume: DaytonaVolumeRuntime
}): Promise<ProvisionedInstance> {
  return wakeExistingDaytonaSandbox({
    provisionInput: input.provisionInput,
    runtime: input.runtime,
    sandbox: input.sandbox,
    workspaceVolumeId: input.sharedVolume.id,
    dataVolumeId: input.sharedVolume.id,
  })
}

export async function provisionWorkerOnDaytonaWithRuntime(
  input: ProvisionInput,
  runtime: DaytonaProvisioningRuntime,
  options: { sleep?: (ms: number) => Promise<unknown> } = {},
): Promise<ProvisionedInstance> {
  const name = currentDaytonaSandboxName(input)
  const lookupNames = daytonaSandboxLookupNames(input)
  const sharedVolume = await getSharedDaytonaVolume(runtime)
  for (const lookupName of lookupNames) {
    const existingSandbox = await getSandboxByName(runtime, lookupName)
    if (existingSandbox) {
      return adoptDaytonaSandbox({ provisionInput: input, runtime, sandbox: existingSandbox, sharedVolume })
    }
  }

  let createdSandbox: DaytonaSandboxRuntime | null = null
  try {
    createdSandbox = await runtime.createSandbox(buildDaytonaCreateParams(input, name, sharedVolume))
    return startOmniRushOnDaytonaSandbox({
      provisionInput: input,
      runtime,
      sandbox: createdSandbox,
      sessionId: `omnirush-${workerHint(input.workerId)}`,
      workspaceVolumeId: sharedVolume.id,
      dataVolumeId: sharedVolume.id,
    })
  } catch (error) {
    if (createdSandbox) {
      await createdSandbox.delete(env.daytona.deleteTimeoutSeconds).catch(() => undefined)
    }

    if (isDaytonaConflictError(error)) {
      const conflictSandbox = await getSandboxAfterCreateConflict(runtime, lookupNames, options.sleep ?? sleep)
      if (conflictSandbox) {
        return adoptDaytonaSandbox({ provisionInput: input, runtime, sandbox: conflictSandbox, sharedVolume })
      }
    }

    throw error
  }
}

export async function provisionWorkerOnDaytona(
  input: ProvisionInput,
): Promise<ProvisionedInstance> {
  assertDaytonaConfig()

  const daytona = createDaytonaClient()
  return provisionWorkerOnDaytonaWithRuntime(input, createDaytonaProvisioningRuntime(daytona))
}

// This preserves customer data: deprovisionWorkerOnDaytona erases workers/<id>/,
// while stopWorkerOnDaytona must not delete the sandbox, cleanup data, or touch volumes.
export async function stopWorkerOnDaytona(workerId: WorkerId): Promise<StopWorkerOnDaytonaResult> {
  assertDaytonaConfig()

  const record = await getDaytonaSandboxRecord(workerId)
  if (!record) {
    return { status: "no_sandbox" }
  }

  const daytona = createDaytonaClient()
  const sandbox = await daytona.get(record.sandbox_id)
  if (sandbox.state === "stopped") {
    return { status: "stopped" }
  }

  await sandbox.stop(env.daytona.stopTimeoutSeconds ?? env.daytona.deleteTimeoutSeconds)

  return { status: "stopped" }
}

export async function flushWorkerCheckpointOnDaytona(workerId: WorkerId) {
  assertDaytonaConfig()

  const record = await getDaytonaSandboxRecord(workerId)
  if (!record) {
    return false
  }

  const daytona = createDaytonaClient()
  const sandbox = await daytona.get(record.sandbox_id)
  await sandbox.refreshData()
  const exitCode = await runSandboxShellCommand(
    toDaytonaSandboxRuntime(sandbox),
    `omnirush-update-flush-${workerHint(workerId)}-${Date.now()}`,
    checkpointFlushCommand(),
    env.daytona.createTimeoutSeconds,
  )

  return exitCode === 0
}

export type DaytonaSandboxWakeRecord = {
  sandbox_id: string
  workspace_volume_id: string
  data_volume_id: string
}

async function getWorkerImageVersion(workerId: WorkerId) {
  const rows = await db
    .select({ image_version: WorkerTable.image_version })
    .from(WorkerTable)
    .where(eq(WorkerTable.id, workerId))
    .limit(1)

  return rows[0]?.image_version ?? null
}

export async function wakeWorkerOnDaytonaWithRuntime(
  input: ProvisionInput,
  runtime: DaytonaProvisioningRuntime,
  record: DaytonaSandboxWakeRecord,
  workerImageVersion: string | null,
): Promise<ProvisionedInstance> {
  const sandbox = await runtime.getSandbox(record.sandbox_id)
  await sandbox.refreshData()

  if (shouldRecycleDaytonaSandbox({ workerImageVersion, sandboxState: sandbox.state })) {
    const sharedVolume = await getSharedDaytonaVolume(runtime)
    const hasCheckpoint = await runtime.checkpointExists({ workerId: input.workerId, sharedVolume })
    if (hasCheckpoint) {
      return recycleDaytonaSandbox({
        provisionInput: input,
        runtime,
        oldSandbox: sandbox,
        sharedVolume,
        oldWorkspaceVolumeId: record.workspace_volume_id,
        oldDataVolumeId: record.data_volume_id,
        oldImageVersion: workerImageVersion,
      })
    }
  }

  try {
    return await wakeExistingDaytonaSandbox({
      provisionInput: input,
      runtime,
      sandbox,
      workspaceVolumeId: record.workspace_volume_id,
      dataVolumeId: record.data_volume_id,
      imageVersion: workerImageVersion,
    })
  } catch (wakeError) {
    const sharedVolume = await getSharedDaytonaVolume(runtime)
    const hasCheckpoint = await runtime.checkpointExists({ workerId: input.workerId, sharedVolume })
    if (!hasCheckpoint && workerImageVersion !== null) {
      throw wakeError
    }

    logger.warn("Daytona sandbox wake failed; replacing the unhealthy sandbox", {
      worker_id: input.workerId,
      sandbox_id: sandbox.id,
      checkpoint_available: hasCheckpoint,
      error: wakeError,
    })
    return recycleDaytonaSandbox({
      provisionInput: input,
      runtime,
      oldSandbox: sandbox,
      sharedVolume,
      oldWorkspaceVolumeId: record.workspace_volume_id,
      oldDataVolumeId: record.data_volume_id,
      oldImageVersion: workerImageVersion,
      replacementName: recoveryDaytonaSandboxName(input),
      requireRestoreMarker: hasCheckpoint,
    })
  }
}

export async function wakeWorkerOnDaytona(
  input: ProvisionInput,
): Promise<ProvisionedInstance> {
  assertDaytonaConfig()

  const record = await getDaytonaSandboxRecord(input.workerId)
  if (!record) {
    throw new DaytonaSandboxMissingError(`Daytona sandbox record missing for worker ${input.workerId}`)
  }

  const daytona = createDaytonaClient()
  const runtime = createDaytonaProvisioningRuntime(daytona)
  const workerImageVersion = await getWorkerImageVersion(input.workerId)
  try {
    return await wakeWorkerOnDaytonaWithRuntime(input, runtime, record, workerImageVersion)
  } catch (error) {
    if (isDaytonaNotFoundError(error)) {
      throw new DaytonaSandboxMissingError(`Daytona sandbox ${record.sandbox_id} missing for worker ${input.workerId}`)
    }

    throw error
  }
}

export async function deprovisionWorkerOnDaytona(workerId: WorkerId) {
  assertDaytonaConfig()

  const daytona = createDaytonaClient()
  const record = await getDaytonaSandboxRecord(workerId)

  if (record) {
    try {
      const sandbox = await daytona.get(record.sandbox_id)
      await sandbox.delete(env.daytona.deleteTimeoutSeconds)
    } catch (error) {
      logger.warn("failed to delete Daytona sandbox", { worker_id: workerId, sandbox_id: record.sandbox_id, error })
    }

    await cleanupWorkerDataOnDaytona(daytona, workerId)

    return
  }

  const sandboxIds = await listDaytonaSandboxIdsByLabels(sandboxLabels(workerId))

  for (const sandboxId of sandboxIds) {
    await daytona
      .get(sandboxId)
      .then((sandbox) => sandbox.delete(env.daytona.deleteTimeoutSeconds))
      .catch((error) => {
        logger.warn("failed to delete Daytona sandbox", { worker_id: workerId, sandbox_id: sandboxId, error })
      })
  }

  await cleanupWorkerDataOnDaytona(daytona, workerId)
}