Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
35.6 KB
Raw Blame
import { createHash } from "node:crypto"
import { and, asc, eq, inArray, isNull, sql } from "@omnirush-ee/den-db/drizzle"
import {
  LlmProviderModelTable,
  LlmProviderTable,
  WorkerTable,
  WorkerTokenTable,
} from "@omnirush-ee/den-db/schema"
import { db } from "../db.js"
import { env } from "../env.js"
import { organizationAllowsManagedModels } from "../inference.js"
import { appLogger } from "../observability/logger.js"
import { fetchPreviewNoRedirect, fetchWithConnectRetry, previewFetch } from "../workers/preview-fetch.js"
import {
  decodeProviderCredential,
  readProviderEnvNames,
  runtimeProviderEnvNames,
  selectLegacyScalarCredentialEnvName,
  selectPrimaryCredentialEnvName,
  toRuntimeProviderEnv,
} from "./provider-credentials.js"

type JsonRecord = Record<string, unknown>
type OrganizationId = typeof LlmProviderTable.$inferSelect.organizationId
type WorkerId = typeof WorkerTable.$inferSelect.id
type WorkerTokenScope = typeof WorkerTokenTable.$inferSelect.scope
type LlmProviderId = typeof LlmProviderTable.$inferSelect.id
type LlmProviderSource = typeof LlmProviderTable.$inferSelect.source

type EnvEntry = {
  key: string
  value: string
}

type EnvSnapshot = Map<string, string | null>

type WorkerToken = {
  scope: WorkerTokenScope
  token: string
}

export type CloudProviderMaterializationProvider = {
  id: LlmProviderId
  source: LlmProviderSource
  providerId: string
  name: string
  providerConfig: JsonRecord
  apiKey: string | null
  models: Array<{
    modelId: string
    name: string
    modelConfig: JsonRecord
  }>
}

export type CloudProviderMaterializationStore = {
  listProviders: (organizationId: OrganizationId) => Promise<CloudProviderMaterializationProvider[]>
  getActiveTokens: (workerId: WorkerId) => Promise<WorkerToken[]>
}

type MaterializedProvider = {
  provider: CloudProviderMaterializationProvider
  runtimeProviderId: string
  config: JsonRecord
  envEntries: EnvEntry[]
}

type PreparedMaterialization = {
  fingerprint: string
  providers: MaterializedProvider[]
  envEntries: EnvEntry[]
  /**
   * Entries an earlier release wrote under a catalog provider's declared name
   * before runtime names were provider-scoped. Deleted only while the worker
   * still holds exactly the value now written under the scoped name.
   */
  supersededEntries: EnvEntry[]
}

type FetchImpl = (input: string, init?: RequestInit) => Promise<Response>

type MaterializationLogger = {
  warn: (message: string, metadata?: JsonRecord) => void
  error: (message: string, metadata?: JsonRecord) => void
}

export type CloudProviderMaterializationResult =
  | {
      ok: true
      status: "applied" | "noop" | "cached"
      fingerprint: string
      providers: number
    }
  | {
      ok: false
      status: "failed"
      error: "provider_materialization_failed"
      reason: string
      message: string
      fingerprint: string | null
      providers: number
    }
  | {
      ok: false
      status: "unsupported"
      error: "provider_materialization_unsupported"
      reason: string
      message: string
      fingerprint: string | null
      providers: number
    }

export type MaterializeCloudWorkerProviders = typeof materializeCloudWorkerProviders

const logger = appLogger.child({ component: "cloud_provider_materialization" })
const requestTimeoutMs = 8_000
/**
 * Cache of what we have already materialized, keyed by worker AND instance.
 *
 * Keying by worker alone was a bug: a recycle onto a new snapshot replaces the
 * sandbox, and the new instance starts with an empty env store (only the runtime
 * config survives, on the shared volume). With a worker-only key den-api kept
 * answering "cached" and never wrote the credential into the new instance, so
 * every provider failed with "API key is missing" while still showing up in the
 * picker.
 */
const materializedFingerprintByWorkerInstance = new Map<string, string>()
// Only write-phase failures are cached. Read failures can occur while the
// sandbox engine is booting and are safe to retry immediately.
const materializationFailureByWorkerInstance = new Map<string, {
  failedAt: number
  result: Extract<CloudProviderMaterializationResult, { ok: false }>
}>()

function materializationCacheKey(workerId: WorkerId, instanceUrl: string): string {
  return `${workerId}\u0000${instanceUrl}`
}
const unsupportedLogFingerprintByWorker = new Map<WorkerId, string>()
const modelConfigPassthroughKeys = [
  "family",
  "release_date",
  "attachment",
  "reasoning",
  "temperature",
  "tool_call",
  "interleaved",
  "cost",
  "limit",
  "modalities",
  "status",
  "options",
  "headers",
  "provider",
  "variants",
]

const databaseMaterializationStore: CloudProviderMaterializationStore = {
  async listProviders(organizationId) {
    const managedModelsAllowed = await organizationAllowsManagedModels(organizationId)
    const providers = await db
      .select()
      .from(LlmProviderTable)
      .where(and(
        eq(LlmProviderTable.organizationId, organizationId),
        managedModelsAllowed ? undefined : sql`${LlmProviderTable.source} <> 'omnirush'`,
      ))
      .orderBy(asc(LlmProviderTable.id))

    if (providers.length === 0) {
      return []
    }

    const providerIds = providers.map((provider) => provider.id)
    const models = await db
      .select()
      .from(LlmProviderModelTable)
      .where(inArray(LlmProviderModelTable.llmProviderId, providerIds))
      .orderBy(asc(LlmProviderModelTable.llmProviderId), asc(LlmProviderModelTable.modelId))

    const modelsByProvider = new Map<LlmProviderId, CloudProviderMaterializationProvider["models"]>()
    for (const model of models) {
      const existing = modelsByProvider.get(model.llmProviderId) ?? []
      existing.push({
        modelId: model.modelId,
        name: model.name,
        modelConfig: model.modelConfig,
      })
      modelsByProvider.set(model.llmProviderId, existing)
    }

    return providers.map((provider) => ({
      id: provider.id,
      source: provider.source,
      providerId: provider.providerId,
      name: provider.name,
      providerConfig: provider.providerConfig,
      apiKey: provider.apiKey ?? null,
      models: modelsByProvider.get(provider.id) ?? [],
    }))
  },
  async getActiveTokens(workerId) {
    return db
      .select({ scope: WorkerTokenTable.scope, token: WorkerTokenTable.token })
      .from(WorkerTokenTable)
      .where(and(eq(WorkerTokenTable.worker_id, workerId), isNull(WorkerTokenTable.revoked_at)))
  },
}

function isRecord(value: unknown): value is JsonRecord {
  return typeof value === "object" && value !== null && !Array.isArray(value)
}

function stableValue(value: unknown): unknown {
  if (Array.isArray(value)) {
    return value.map(stableValue)
  }

  if (!isRecord(value)) {
    return value
  }

  return Object.fromEntries(
    Object.keys(value)
      .sort()
      .map((key) => [key, stableValue(value[key])]),
  )
}

function stableJson(value: unknown) {
  return JSON.stringify(stableValue(value))
}

function hashString(value: string) {
  return createHash("sha256").update(value).digest("hex")
}

function normalizeBaseUrl(value: string) {
  return value.trim().replace(/\/+$/, "")
}

function readString(value: unknown) {
  return typeof value === "string" && value.trim() ? value.trim() : null
}

function readStringList(value: unknown): string[] {
  return Array.isArray(value)
    ? value.filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0)
    : []
}

function runtimeProviderId(provider: Pick<CloudProviderMaterializationProvider, "id" | "source">) {
  return provider.source === "omnirush" ? "omnirush" : provider.id.trim()
}

function isCloudManagedProviderKey(providerId: string) {
  return /^lpr_/i.test(providerId) || providerId.trim() === "omnirush"
}

function upsertEnvEntry(entries: EnvEntry[], key: string, value: string) {
  const trimmedKey = key.trim()
  const trimmedValue = value.trim()
  if (!trimmedKey || !trimmedValue) {
    return
  }

  const existing = entries.find((entry) => entry.key === trimmedKey)
  if (existing) {
    existing.value = trimmedValue
    return
  }

  entries.push({ key: trimmedKey, value: trimmedValue })
}

function readOmniRushInferenceBaseUrl(providerConfig: JsonRecord) {
  const options = providerConfig.options
  if (isRecord(options)) {
    const baseUrl = readString(options.baseURL)
    if (baseUrl) {
      return baseUrl.replace(/\/api\/v1\/?$/, "")
    }
  }

  const api = readString(providerConfig.api)
  return api ? api.replace(/\/api\/v1\/?$/, "") : null
}

function providerEnvEntries(provider: CloudProviderMaterializationProvider): EnvEntry[] {
  const entries: EnvEntry[] = []
  const stored = decodeProviderCredential(provider.apiKey)
  // Stored rows keep the catalog's declared names; the worker sees the
  // provider-scoped runtime names for both the block and the multi-env map.
  const credential = toRuntimeProviderEnv({
    id: provider.id,
    source: provider.source,
    providerConfig: provider.providerConfig,
    apiKeys: stored.apiKeys,
  })
  const envNames = readProviderEnvNames(credential.providerConfig)

  if (credential.apiKeys) {
    const keys = Object.keys(credential.apiKeys)
    const orderedNames = [
      ...envNames.filter((name) => keys.includes(name)),
      ...keys.filter((name) => !envNames.includes(name)),
    ]
    for (const name of orderedNames) {
      upsertEnvEntry(entries, name, credential.apiKeys[name] ?? "")
    }
  }

  if (stored.apiKey && envNames[0]) {
    upsertEnvEntry(
      entries,
      selectLegacyScalarCredentialEnvName(envNames) ?? envNames[0],
      stored.apiKey,
    )
  }

  const primaryCredentialEnvName = selectPrimaryCredentialEnvName(envNames, entries.map((entry) => entry.key))
  const primaryCredential = stored.apiKey?.trim() || entries.find((entry) => entry.key === primaryCredentialEnvName)?.value || ""
  if (provider.source === "omnirush" && primaryCredential) {
    upsertEnvEntry(entries, "OMNIRUSH_API_KEY", primaryCredential)
    const baseUrl = readOmniRushInferenceBaseUrl(provider.providerConfig)
    if (baseUrl) {
      upsertEnvEntry(entries, "OMNIRUSH_INFERENCE_BASE_URL", baseUrl)
    }
  }

  return entries
}

function buildModelConfig(model: CloudProviderMaterializationProvider["models"][number]) {
  const next: JsonRecord = {
    id: model.modelId,
    name: model.name,
  }

  for (const key of modelConfigPassthroughKeys) {
    const value = model.modelConfig[key]
    if (value !== undefined) {
      next[key] = value
    }
  }

  return next
}

function buildProviderConfig(provider: CloudProviderMaterializationProvider) {
  const models: JsonRecord = {}
  const sortedModels = [...provider.models].sort((left, right) => left.modelId.localeCompare(right.modelId))
  for (const model of sortedModels) {
    models[model.modelId] = buildModelConfig(model)
  }

  const config: JsonRecord = {
    id: provider.providerId,
    name: provider.name,
    env: runtimeProviderEnvNames(provider),
  }

  if (Object.keys(models).length > 0 || provider.source !== "omnirush") {
    config.models = models
  }

  const npm = readString(provider.providerConfig.npm)
  if (npm) {
    config.npm = npm
  }

  const api = readString(provider.providerConfig.api)
  if (api) {
    config.api = api
  }

  const options = provider.providerConfig.options
  if (isRecord(options)) {
    config.options = options
  }

  const whitelist = readStringList(provider.providerConfig.whitelist)
  if (whitelist.length > 0) {
    config.whitelist = whitelist
  }

  const blacklist = readStringList(provider.providerConfig.blacklist)
  if (blacklist.length > 0) {
    config.blacklist = blacklist
  }

  return config
}

function providerHasRequiredCredential(provider: CloudProviderMaterializationProvider, envEntries: EnvEntry[]) {
  const envNames = runtimeProviderEnvNames(provider)
  if (envNames.length === 0) {
    return true
  }

  return selectPrimaryCredentialEnvName(envNames, envEntries.map((entry) => entry.key)) !== null
}

function prepareMaterialization(providers: CloudProviderMaterializationProvider[]): PreparedMaterialization {
  const materialized = providers
    .map((provider) => {
      const envEntries = providerEnvEntries(provider)
      if (!providerHasRequiredCredential(provider, envEntries)) {
        return null
      }

      return {
        provider,
        runtimeProviderId: runtimeProviderId(provider),
        config: buildProviderConfig(provider),
        envEntries,
      }
    })
    .filter((entry): entry is MaterializedProvider => entry !== null)
    .sort((left, right) => left.runtimeProviderId.localeCompare(right.runtimeProviderId))

  const envEntries: EnvEntry[] = []
  for (const provider of materialized) {
    for (const entry of provider.envEntries) {
      upsertEnvEntry(envEntries, entry.key, entry.value)
    }
  }
  const desiredKeys = new Set(envEntries.map((entry) => entry.key))
  const supersededEntries: EnvEntry[] = []
  for (const { provider, envEntries: written } of materialized) {
    const declared = readProviderEnvNames(provider.providerConfig)
    runtimeProviderEnvNames(provider).forEach((runtimeName, index) => {
      const declaredName = declared[index]
      const value = written.find((entry) => entry.key === runtimeName)?.value
      if (declaredName && declaredName !== runtimeName && value !== undefined && !desiredKeys.has(declaredName)) {
        upsertEnvEntry(supersededEntries, declaredName, value)
      }
    })
  }

  const fingerprintPayload = materialized.map((entry) => ({
    id: entry.provider.id,
    runtimeProviderId: entry.runtimeProviderId,
    source: entry.provider.source,
    providerId: entry.provider.providerId,
    config: entry.config,
    keyHashes: entry.envEntries
      .map((envEntry) => ({ key: envEntry.key, hash: hashString(envEntry.value) }))
      .sort((left, right) => left.key.localeCompare(right.key)),
  }))

  return {
    fingerprint: `owp:v1:${hashString(stableJson(fingerprintPayload))}`,
    providers: materialized,
    envEntries,
    supersededEntries,
  }
}

function staleSupersededKeys(supersededEntries: EnvEntry[], snapshot: EnvSnapshot) {
  return supersededEntries.filter((entry) => snapshot.get(entry.key) === entry.value).map((entry) => entry.key)
}

export function computeCloudProviderMaterializationFingerprint(providers: CloudProviderMaterializationProvider[]) {
  return prepareMaterialization(providers).fingerprint
}

async function fetchWithTimeout(fetchImpl: FetchImpl, url: string, init: RequestInit = {}) {
  const controller = new AbortController()
  const timeout = setTimeout(() => controller.abort(), requestTimeoutMs)
  try {
    return await fetchPreviewNoRedirect(fetchImpl, url, { ...init, signal: controller.signal })
  } finally {
    clearTimeout(timeout)
  }
}

async function readJsonObject(response: Response) {
  const text = await response.text()
  if (!text.trim()) {
    return {}
  }

  const parsed = JSON.parse(text)
  return isRecord(parsed) ? parsed : {}
}

function bearerHeaders(clientToken: string) {
  return {
    Accept: "application/json",
    "Content-Type": "application/json",
    Authorization: `Bearer ${clientToken}`,
  }
}

function hostTokenHeaders(hostToken: string) {
  return {
    Accept: "application/json",
    "Content-Type": "application/json",
    "X-OmniRush-Host-Token": hostToken,
  }
}

class MaterializationHttpError extends Error {
  readonly label: string
  readonly status: number
  readonly responseBody: string

  constructor(label: string, status: number, responseBody = "") {
    super(`${label}_failed_${status}`)
    this.name = "MaterializationHttpError"
    this.label = label
    this.status = status
    this.responseBody = responseBody
  }
}

async function requestJson(input: {
  fetchImpl: FetchImpl
  label: string
  url: string
  init: RequestInit
}) {
  const response = await fetchWithTimeout(input.fetchImpl, input.url, input.init)
  if (!response.ok) {
    throw new MaterializationHttpError(input.label, response.status)
  }

  return readJsonObject(response)
}

async function requestOk(input: {
  fetchImpl: FetchImpl
  label: string
  url: string
  init: RequestInit
}) {
  const response = await fetchWithTimeout(input.fetchImpl, input.url, input.init)
  if (!response.ok) {
    throw new MaterializationHttpError(input.label, response.status, await response.text())
  }
}

async function readRuntimeManagedProviders(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  clientToken: string
}) {
  const payload = await requestJson({
    fetchImpl: input.fetchImpl,
    label: "engine_config_read",
    url: `${input.instanceUrl}/opencode/config`,
    init: {
      method: "GET",
      headers: bearerHeaders(input.clientToken),
    },
  })
  const provider = isRecord(payload.provider) ? payload.provider : null
  const managed: JsonRecord = {}
  if (!provider) {
    return managed
  }

  for (const [providerId, config] of Object.entries(provider)) {
    if (isCloudManagedProviderKey(providerId) && isRecord(config)) {
      managed[providerId] = config
    }
  }

  return managed
}

function readRuntimeSnapshotVersion(payload: JsonRecord) {
  const services = Array.isArray(payload.services) ? payload.services : []
  let fallback: string | null = null
  for (const service of services) {
    if (!isRecord(service)) {
      continue
    }

    const version = readString(service.actualVersion) ?? readString(service.targetVersion)
    if (!version) {
      continue
    }

    fallback = fallback ?? version
    const serviceName = readString(service.name)?.toLowerCase()
    if (serviceName?.includes("omnirush") || serviceName?.includes("server")) {
      return version
    }
  }

  return fallback
}

async function readInstanceVersion(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  clientToken: string
}) {
  try {
    const payload = await requestJson({
      fetchImpl: input.fetchImpl,
      label: "runtime_versions_read",
      url: `${input.instanceUrl}/runtime/versions`,
      init: {
        method: "GET",
        headers: bearerHeaders(input.clientToken),
      },
    })

    return readRuntimeSnapshotVersion(payload)
  } catch {
    return null
  }
}

function buildRuntimeProviderPatch(prepared: PreparedMaterialization, currentManagedProviders: JsonRecord) {
  const desiredIds = new Set(prepared.providers.map((provider) => provider.runtimeProviderId))
  const patch: JsonRecord = {}
  for (const providerId of Object.keys(currentManagedProviders)) {
    if (!desiredIds.has(providerId)) {
      patch[providerId] = null
    }
  }

  for (const provider of prepared.providers) {
    patch[provider.runtimeProviderId] = provider.config
  }

  return patch
}

function materializedProviderStateMatches(prepared: PreparedMaterialization, currentManagedProviders: JsonRecord) {
  const desiredManagedProviders: JsonRecord = {}
  for (const provider of prepared.providers) {
    desiredManagedProviders[provider.runtimeProviderId] = provider.config
  }

  return stableJson(currentManagedProviders) === stableJson(desiredManagedProviders)
}

function materializedEnvStateMatches(entries: EnvEntry[], snapshot: EnvSnapshot) {
  return entries.every((entry) => snapshot.get(entry.key) === entry.value)
}

function buildRuntimeProviderRollbackPatch(prepared: PreparedMaterialization, currentManagedProviders: JsonRecord) {
  const affectedIds = new Set([
    ...Object.keys(currentManagedProviders),
    ...prepared.providers.map((provider) => provider.runtimeProviderId),
  ])
  const patch: JsonRecord = {}
  for (const providerId of affectedIds) {
    patch[providerId] = currentManagedProviders[providerId] ?? null
  }

  return patch
}

async function readEnvEntry(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  key: string
}) {
  const response = await fetchWithTimeout(input.fetchImpl, `${input.instanceUrl}/env/${encodeURIComponent(input.key)}`, {
    method: "GET",
    headers: hostTokenHeaders(input.hostToken),
  })
  if (response.status === 404) {
    return null
  }
  if (!response.ok) {
    throw new Error(`env_read_failed_${response.status}`)
  }

  const payload = await readJsonObject(response)
  const item = isRecord(payload.item) ? payload.item : null
  return item && typeof item.value === "string" ? { key: input.key, value: item.value } : null
}

async function readEnvSnapshot(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  keys: string[]
}): Promise<EnvSnapshot> {
  const snapshot: EnvSnapshot = new Map()
  for (const key of input.keys) {
    if (snapshot.has(key)) {
      continue
    }
    const existing = await readEnvEntry({
      fetchImpl: input.fetchImpl,
      instanceUrl: input.instanceUrl,
      hostToken: input.hostToken,
      key,
    })
    snapshot.set(key, existing?.value ?? null)
  }

  return snapshot
}

async function writeEnvEntries(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  entries: EnvEntry[]
}) {
  if (input.entries.length === 0) {
    return
  }

  await requestOk({
    fetchImpl: input.fetchImpl,
    label: "env_write",
    url: `${input.instanceUrl}/env`,
    init: {
      method: "PUT",
      headers: hostTokenHeaders(input.hostToken),
      body: JSON.stringify({ entries: input.entries }),
    },
  })
}

async function deleteEnvEntry(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  key: string
  ignoreNotFound?: boolean
}) {
  const response = await fetchWithTimeout(input.fetchImpl, `${input.instanceUrl}/env/${encodeURIComponent(input.key)}`, {
    method: "DELETE",
    headers: hostTokenHeaders(input.hostToken),
  })
  if (response.status === 404 && input.ignoreNotFound) {
    return
  }
  if (!response.ok) {
    throw new Error(`env_delete_failed_${response.status}`)
  }
}

async function restoreEnvSnapshot(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  snapshot: EnvSnapshot
}) {
  const entries: EnvEntry[] = []
  const deleteKeys: string[] = []
  for (const [key, value] of input.snapshot) {
    if (value === null) {
      deleteKeys.push(key)
    } else {
      entries.push({ key, value })
    }
  }

  await writeEnvEntries({
    fetchImpl: input.fetchImpl,
    instanceUrl: input.instanceUrl,
    hostToken: input.hostToken,
    entries,
  })
  for (const key of deleteKeys) {
    await deleteEnvEntry({
      fetchImpl: input.fetchImpl,
      instanceUrl: input.instanceUrl,
      hostToken: input.hostToken,
      key,
      ignoreNotFound: true,
    })
  }
}

async function patchRuntimeProviders(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  hostToken: string
  patch: JsonRecord
}) {
  const response = await fetchWithTimeout(input.fetchImpl, `${input.instanceUrl}/runtime-config/providers`, {
    method: "PATCH",
    headers: hostTokenHeaders(input.hostToken),
    body: JSON.stringify({ provider: input.patch }),
  })
  if (!response.ok) {
    throw new MaterializationHttpError("runtime_provider_patch", response.status, await response.text())
  }

  // An instance older than this route answers 200 with the SPA index.html
  // instead of 404, because the web root is the catch-all. Observed on a real
  // worker still running omnirush-server 0.18.3: the patch "succeeded", the
  // engine ended up with zero providers, and the org saw an opaque failure
  // instead of "this workspace needs an update". Treat a non-JSON body as an
  // unsupported route so the caller can degrade honestly.
  const body = await response.text()
  if (!looksLikeJsonObject(body)) {
    throw new MaterializationHttpError("runtime_provider_patch", UNSUPPORTED_ROUTE_STATUS, body.slice(0, 200))
  }
}

/**
 * Synthetic status for "this instance does not implement the route", used when
 * the instance answers 200 with something that is not the route's JSON.
 */
const UNSUPPORTED_ROUTE_STATUS = 501

function looksLikeJsonObject(body: string): boolean {
  const trimmed = body.trim()
  if (!trimmed.startsWith("{")) return false
  try {
    return typeof JSON.parse(trimmed) === "object"
  } catch {
    return false
  }
}

async function verifyRuntimeProviders(input: {
  fetchImpl: FetchImpl
  instanceUrl: string
  clientToken: string
  providerIds: string[]
}) {
  if (input.providerIds.length === 0) {
    return
  }

  const payload = await requestJson({
    fetchImpl: input.fetchImpl,
    label: "provider_readback",
    url: `${input.instanceUrl}/opencode/config`,
    init: {
      method: "GET",
      headers: bearerHeaders(input.clientToken),
    },
  })
  const provider = isRecord(payload.provider) ? payload.provider : null
  for (const providerId of input.providerIds) {
    if (!provider || !Object.prototype.hasOwnProperty.call(provider, providerId)) {
      throw new Error(`provider_readback_missing_${providerId}`)
    }
  }
}

async function resolveTokens(input: {
  workerId: WorkerId
  hostToken?: string
  clientToken?: string
  store: CloudProviderMaterializationStore
}) {
  let hostToken = input.hostToken?.trim() ?? ""
  let clientToken = input.clientToken?.trim() ?? ""
  if (hostToken && clientToken) {
    return { hostToken, clientToken }
  }

  const tokens = await input.store.getActiveTokens(input.workerId)
  hostToken = hostToken || tokens.find((token) => token.scope === "host")?.token.trim() || ""
  clientToken = clientToken || tokens.find((token) => token.scope === "client")?.token.trim() || ""
  return { hostToken, clientToken }
}

function failureResult(input: {
  reason: string
  error: unknown
  fingerprint: string | null
  providers: number
}) {
  const message = input.error instanceof MaterializationHttpError && input.error.responseBody
    ? `${input.error.message}: ${input.error.responseBody}`
    : input.error instanceof Error ? input.error.message : input.reason
  return {
    ok: false,
    status: "failed",
    error: "provider_materialization_failed",
    reason: input.reason,
    message,
    fingerprint: input.fingerprint,
    providers: input.providers,
  } satisfies CloudProviderMaterializationResult
}

function unsupportedResult(input: {
  reason: string
  fingerprint: string | null
  providers: number
}) {
  return {
    ok: false,
    status: "unsupported",
    error: "provider_materialization_unsupported",
    reason: input.reason,
    message: input.reason,
    fingerprint: input.fingerprint,
    providers: input.providers,
  } satisfies CloudProviderMaterializationResult
}

function unsupportedProviderRouteReason(error: unknown) {
  return error instanceof MaterializationHttpError
    && error.label === "runtime_provider_patch"
    && (error.status === 404 || error.status === 405 || error.status === UNSUPPORTED_ROUTE_STATUS)
    ? error.message
    : null
}

function logFailure(input: {
  logger: MaterializationLogger
  workerId: WorkerId
  organizationId: OrganizationId
  result: Extract<CloudProviderMaterializationResult, { ok: false }>
  cause: unknown
}) {
  const metadata = {
    worker_id: input.workerId,
    organization_id: input.organizationId,
    reason: input.result.reason,
    message: input.result.message,
    fingerprint: input.result.fingerprint,
    providers: input.result.providers,
  }
  if (
    input.cause instanceof MaterializationHttpError
    && input.cause.status >= 400
    && input.cause.status < 500
    && (input.cause.label === "env_write" || input.cause.label === "runtime_provider_patch")
  ) {
    input.logger.error("cloud provider materialization write rejected", {
      ...metadata,
      status: input.cause.status,
      response_body: input.cause.responseBody,
    })
    return
  }

  input.logger.warn("cloud provider materialization failed", metadata)
}

async function logUnsupportedOnce(input: {
  logger: MaterializationLogger
  workerId: WorkerId
  organizationId: OrganizationId
  fingerprint: string
  providers: number
  reason: string
  fetchImpl: FetchImpl
  instanceUrl: string
  clientToken: string
}) {
  if (unsupportedLogFingerprintByWorker.get(input.workerId) === input.fingerprint) {
    return
  }

  unsupportedLogFingerprintByWorker.set(input.workerId, input.fingerprint)
  const instanceVersion = await readInstanceVersion({
    fetchImpl: input.fetchImpl,
    instanceUrl: input.instanceUrl,
    clientToken: input.clientToken,
  })

  input.logger.warn("cloud provider materialization unsupported by worker version", {
    worker_id: input.workerId,
    organization_id: input.organizationId,
    reason: input.reason,
    fingerprint: input.fingerprint,
    providers: input.providers,
    instance_version: instanceVersion,
  })
}

// den-api must tolerate instances older than itself: sandbox images ship on the
// release/snapshot cadence while den-api deploys from dev.
export async function materializeCloudWorkerProviders(input: {
  organizationId: OrganizationId
  workerId: WorkerId
  instanceUrl: string
  hostToken?: string
  clientToken?: string
  force?: boolean
  store?: CloudProviderMaterializationStore
  fetchImpl?: FetchImpl
  logger?: MaterializationLogger
  now?: () => number
}): Promise<CloudProviderMaterializationResult> {
  const materializationLogger = input.logger ?? logger
  const store = input.store ?? databaseMaterializationStore
  const fetchImpl = input.fetchImpl ?? ((url, init = {}) => fetchWithConnectRetry({
    fetchImpl: previewFetch(),
    url,
    init,
  }))
  const now = input.now ?? Date.now
  const instanceUrl = normalizeBaseUrl(input.instanceUrl)
  const cacheKey = materializationCacheKey(input.workerId, instanceUrl)
  let fingerprint: string | null = null
  let providerCount = 0
  let writePhaseStarted = false

  try {
    if (!instanceUrl) {
      throw new Error("instance_url_missing")
    }

    const recentFailure = materializationFailureByWorkerInstance.get(cacheKey)
    if (!input.force && recentFailure && now() - recentFailure.failedAt < env.cloudMaterializationFailureCooldownMs) {
      return recentFailure.result
    }

    const providers = await store.listProviders(input.organizationId)
    const prepared = prepareMaterialization(providers)
    fingerprint = prepared.fingerprint
    providerCount = prepared.providers.length

    if (!input.force && materializedFingerprintByWorkerInstance.get(cacheKey) === fingerprint) {
      materializationFailureByWorkerInstance.delete(cacheKey)
      return { ok: true, status: "cached", fingerprint, providers: providerCount }
    }

    const tokens = await resolveTokens({
      workerId: input.workerId,
      hostToken: input.hostToken,
      clientToken: input.clientToken,
      store,
    })
    if (!tokens.hostToken || !tokens.clientToken) {
      throw new Error("worker_tokens_missing")
    }
    const desiredProviderIds = prepared.providers.map((provider) => provider.runtimeProviderId)
    const currentManagedProviders = await readRuntimeManagedProviders({
      fetchImpl,
      instanceUrl,
      clientToken: tokens.clientToken,
    })
    const providerPatch = buildRuntimeProviderPatch(prepared, currentManagedProviders)
    const providerRollbackPatch = buildRuntimeProviderRollbackPatch(prepared, currentManagedProviders)
    const envSnapshot = await readEnvSnapshot({
      fetchImpl,
      instanceUrl,
      hostToken: tokens.hostToken,
      keys: [...prepared.envEntries, ...prepared.supersededEntries].map((entry) => entry.key),
    })
    const staleKeys = staleSupersededKeys(prepared.supersededEntries, envSnapshot)
    // Roll back only what this pass writes or deletes; a bare-name entry that
    // is left alone is never touched on the way out either.
    const touchedKeys = new Set([...prepared.envEntries.map((entry) => entry.key), ...staleKeys])
    const envRollbackSnapshot: EnvSnapshot = new Map([...envSnapshot].filter(([key]) => touchedKeys.has(key)))
    if (
      materializedProviderStateMatches(prepared, currentManagedProviders)
      && materializedEnvStateMatches(prepared.envEntries, envSnapshot)
      && staleKeys.length === 0
    ) {
      materializedFingerprintByWorkerInstance.set(cacheKey, fingerprint)
      materializationFailureByWorkerInstance.delete(cacheKey)
      return { ok: true, status: "noop", fingerprint, providers: providerCount }
    }

    let envWritten = false
    let providerPatched = false

    try {
      writePhaseStarted = true
      await writeEnvEntries({
        fetchImpl,
        instanceUrl,
        hostToken: tokens.hostToken,
        entries: prepared.envEntries,
      })
      envWritten = prepared.envEntries.length > 0
      providerPatched = true
      await patchRuntimeProviders({
        fetchImpl,
        instanceUrl,
        hostToken: tokens.hostToken,
        patch: providerPatch,
      })
      await verifyRuntimeProviders({
        fetchImpl,
        instanceUrl,
        clientToken: tokens.clientToken,
        providerIds: desiredProviderIds,
      })
    } catch (error) {
      const unsupportedReason = unsupportedProviderRouteReason(error)
      if (unsupportedReason) {
        materializedFingerprintByWorkerInstance.delete(cacheKey)
        await logUnsupportedOnce({
          logger: materializationLogger,
          workerId: input.workerId,
          organizationId: input.organizationId,
          fingerprint: prepared.fingerprint,
          providers: providerCount,
          reason: unsupportedReason,
          fetchImpl,
          instanceUrl,
          clientToken: tokens.clientToken,
        })
        const result = unsupportedResult({
          reason: unsupportedReason,
          fingerprint: prepared.fingerprint,
          providers: providerCount,
        })
        materializationFailureByWorkerInstance.set(cacheKey, { failedAt: now(), result })
        return result
      }

      if (providerPatched) {
        await patchRuntimeProviders({
          fetchImpl,
          instanceUrl,
          hostToken: tokens.hostToken,
          patch: providerRollbackPatch,
        }).catch((rollbackError) => {
          materializationLogger.warn("cloud provider rollback failed", {
            worker_id: input.workerId,
            organization_id: input.organizationId,
            reason: rollbackError instanceof Error ? rollbackError.message : "runtime_config_rollback_failed",
          })
        })
      }
      if (envWritten) {
        await restoreEnvSnapshot({
          fetchImpl,
          instanceUrl,
          hostToken: tokens.hostToken,
          snapshot: envRollbackSnapshot,
        }).catch((rollbackError) => {
          materializationLogger.warn("cloud provider env rollback failed", {
            worker_id: input.workerId,
            organization_id: input.organizationId,
            reason: rollbackError instanceof Error ? rollbackError.message : "env_rollback_failed",
          })
        })
      }
      throw error
    }

    // The org credential an earlier release left under the bare catalog name
    // would keep enabling OpenCode's built-in vendor catalog. Remove it only on
    // an exact value match (a different value is the member's own), and only
    // after the scoped names are in place; a failed delete is retried by the
    // next pass because it keeps the state out of noop.
    for (const key of staleKeys) {
      await deleteEnvEntry({ fetchImpl, instanceUrl, hostToken: tokens.hostToken, key, ignoreNotFound: true }).catch(
        (deleteError) => {
          materializationLogger.warn("cloud provider stale env cleanup failed", {
            worker_id: input.workerId,
            organization_id: input.organizationId,
            env_key: key,
            reason: deleteError instanceof Error ? deleteError.message : "env_delete_failed",
          })
        },
      )
    }

    materializedFingerprintByWorkerInstance.set(cacheKey, fingerprint)
    materializationFailureByWorkerInstance.delete(cacheKey)
    return { ok: true, status: "applied", fingerprint, providers: providerCount }
  } catch (error) {
    const result = failureResult({
      reason: error instanceof Error ? error.message : "provider_materialization_failed",
      error,
      fingerprint,
      providers: providerCount,
    })
    logFailure({
      logger: materializationLogger,
      workerId: input.workerId,
      organizationId: input.organizationId,
      result,
      cause: error,
    })
    if (writePhaseStarted) {
      materializationFailureByWorkerInstance.set(cacheKey, { failedAt: now(), result })
    }
    return result
  }
}