Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
30.3 KB
Raw Blame
import { execFile, execFileSync, spawn } from 'node:child_process'
import { createHash, randomUUID } from 'node:crypto'
import { existsSync, readFileSync } from 'node:fs'
import { homedir } from 'node:os'
import { resolve } from 'node:path'
import { promisify } from 'node:util'
import { firstDataLine, kubectlContext, runControlPostgresSql, sqlLiteral } from './gfsFixtureCore'
import {
  type GfsUploadProductEnvPair,
  type GfsUploadProductEnvState,
  type GfsUploadProductMutationAdapter,
  type GfsUploadProductMutationLease,
  type GfsUploadProductMutationMarker,
  type GfsUploadProductMutationOwnership,
  applyGfsUploadProductMutation,
  beginGfsUploadProductMutation,
  parseGfsUploadProductMutationMarker,
  recoverGfsUploadProductMutation,
  restoreGfsUploadProductMutation,
  sameGfsUploadProductEnvPair,
  serializeGfsUploadProductMutationMarker,
} from './gfsUploadProductMutation'
import { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'

export { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'

const execFileAsync = promisify(execFile)
const HCC_NAMESPACE = 'control-plane'
const HCC_DEPLOYMENT = 'host-context-controller'
const GFS_NAMESPACE = 'gfs'
const GFS_WRITER_DEPLOYMENT = 'gfsc-writer'
const GFS_READER_DEPLOYMENT = 'gfsc-reader'
const GFS_V2_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_V2_ENABLED'
const GFS_V2_WRITER_ENV = 'GFS_UPLOAD_V2_ENABLED'
const GFS_PRODUCT_MAX_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_PRODUCT_MAX_FILE_BYTES'
const GFS_PRODUCT_MAX_HCC_ALIAS_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_MAX_FILE_BYTES'
const GFS_PRODUCT_MAX_WRITER_ENV = 'GFS_UPLOAD_PRODUCT_MAX_FILE_BYTES'
const GFS_PRODUCT_MAX_WRITER_ALIAS_ENV = 'GFS_UPLOAD_MAX_FILE_BYTES'
const GFS_PRODUCT_MUTATION_MARKER = 'gfs-upload-product-limit-recovery'
const GFS_PRODUCT_MUTATION_MARKER_KEY = 'transaction.json'
const BRANCH_CONTEXT_RE = /^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/

export interface GfsRuntimeImageMarker {
  clusterFingerprint?: string
  imageSource?: string
  imageTag?: string
  imagesGeneratedAt?: string
}

export interface ExpectedGfsRuntimeImageMarker {
  imageSource?: string
  imageTag?: string
  clusterFingerprint?: string
  imagesGeneratedAt?: string
}

export type GfsUploadProductMaxState = GfsUploadProductEnvState
export type GfsUploadProductMaxInputs = GfsUploadProductEnvPair

/**
 * Local Minikube image mode intentionally has no registry tag. GHCR mode must
 * carry one because it is the immutable image coordinate that proves which
 * artifact was deployed. Both modes still require the marker's source and
 * acquisition timestamp so an empty local tag cannot be confused with a
 * missing marker field.
 */
export function validateGfsRuntimeImageMarker(
  marker: GfsRuntimeImageMarker,
  expected: ExpectedGfsRuntimeImageMarker = {}
): void {
  if (
    !marker.clusterFingerprint ||
    !marker.imageSource ||
    typeof marker.imageTag !== 'string' ||
    !marker.imagesGeneratedAt
  ) {
    throw new Error(
      'refusing GFS E2E mutation: pre-gate marker lacks cluster fingerprint, image coordinates, or acquisition stamp'
    )
  }
  if (marker.imageSource !== 'local' && marker.imageSource !== 'ghcr') {
    throw new Error(`refusing GFS E2E mutation: unsupported imageSource ${marker.imageSource}`)
  }
  if (marker.imageSource === 'ghcr' && marker.imageTag.length === 0) {
    throw new Error(
      'refusing GFS E2E mutation: GHCR image marker is missing its immutable imageTag'
    )
  }
  if (expected.imageSource && marker.imageSource !== expected.imageSource) {
    throw new Error(
      'refusing GFS E2E mutation: imageSource does not match the caller-owned profile'
    )
  }
  if (expected.imageTag !== undefined && marker.imageTag !== expected.imageTag) {
    throw new Error('refusing GFS E2E mutation: imageTag does not match the caller-owned profile')
  }
  if (expected.clusterFingerprint && marker.clusterFingerprint !== expected.clusterFingerprint) {
    throw new Error(
      'refusing GFS E2E mutation: clusterFingerprint does not match the caller-owned profile'
    )
  }
  if (expected.imagesGeneratedAt && marker.imagesGeneratedAt !== expected.imagesGeneratedAt) {
    throw new Error(
      'refusing GFS E2E mutation: imagesGeneratedAt does not match the caller-owned profile'
    )
  }
}

function profilePorts(context: string): Map<string, string> {
  const path =
    process.env.CLERUM_PROFILE_PORTS_ENV ||
    process.env.E2E_PROFILE_PORTS_ENV ||
    resolve(homedir(), '.cache/clerum/minikube-profiles', context, 'ports.env')
  if (!existsSync(path)) {
    throw new Error(`refusing GFS E2E mutation: profile ports.env is missing at ${path}`)
  }
  const values = new Map<string, string>()
  for (const line of readFileSync(path, 'utf8').split(/\r?\n/)) {
    const match = /^(?:export\s+)?([A-Z0-9_]+)=(.*)$/.exec(line.trim())
    if (match) values.set(match[1]!, match[2]!.replace(/^['"]|['"]$/g, ''))
  }
  return values
}

function normalizeUrl(value: string): string {
  return value.replace(/\/$/, '')
}

/**
 * Mutating negative journeys are allowed only on the exact branch profile
 * whose generated port map is present in the caller environment. A context
 * name alone is not ownership proof: another healthy Minikube profile could
 * otherwise receive an HCC mutation or writer restart.
 */
interface PreGateMarkerData {
  worktreeId?: string
  gitHead?: string
  clusterFingerprint?: string
  imageSource?: string
  imageTag?: string
  imagesGeneratedAt?: string
}

interface PreGateMarkerResource {
  metadata?: { uid?: string }
  data?: PreGateMarkerData
}

async function assertOwnedRuntimeContext(): Promise<GfsUploadProductMutationOwnership> {
  const context = kubectlContext()
  if (/prod|production/i.test(context)) {
    throw new Error(`refusing GFS E2E mutation on production-like context ${context}`)
  }
  const explicitlyAllowed = new Set(
    (process.env.E2E_ALLOWED_CONTEXTS || '')
      .split(',')
      .map(value => value.trim())
      .filter(Boolean)
  )
  if (!BRANCH_CONTEXT_RE.test(context) && !explicitlyAllowed.has(context)) {
    throw new Error(
      `refusing GFS E2E mutation on unscoped context ${context}; use a generated branch profile or E2E_ALLOWED_CONTEXTS`
    )
  }
  const ports = profilePorts(context)
  const required = [
    ['CONTROL_UI_BASE_URL', 'CONTROL_UI_URL'],
    ['EXTERNAL_REST_API_BASE_URL', 'EXTERNAL_REST_API_URL'],
  ] as const
  for (const [actualKey, profileKey] of required) {
    const actual = process.env[actualKey] || process.env[actualKey.replace('_BASE_URL', '_URL')]
    const expected = ports.get(profileKey)
    if (!expected || !actual || normalizeUrl(actual) !== normalizeUrl(expected)) {
      throw new Error(
        `refusing GFS E2E mutation: ${actualKey} does not match ${profileKey} in profile ports.env`
      )
    }
  }
  const rpcActual = process.env.RPC_PROXY_BASE_URL || process.env.RPC_PROXY_URL
  if (rpcActual) {
    const rpcExpected = ports.get('RPC_PROXY_URL')
    if (!rpcExpected || normalizeUrl(rpcActual) !== normalizeUrl(rpcExpected)) {
      throw new Error(
        'refusing GFS E2E mutation: RPC_PROXY_BASE_URL does not match RPC_PROXY_URL in profile ports.env'
      )
    }
  }

  const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
    cwd: process.cwd(),
    encoding: 'utf8',
  }).trim()
  const dirty = execFileSync('git', ['status', '--porcelain', '--untracked-files=normal'], {
    cwd: repoRoot,
    encoding: 'utf8',
  }).trim()
  if (dirty) {
    throw new Error(
      'refusing GFS E2E mutation: the worktree is dirty; commit and run pre-gate-sync before fault injection'
    )
  }
  const expectedWorktreeId = createHash('sha1').update(repoRoot).digest('hex')
  const expectedHead = execFileSync('git', ['rev-parse', 'HEAD'], {
    cwd: repoRoot,
    encoding: 'utf8',
  }).trim()
  const branch = execFileSync('git', ['branch', '--show-current'], {
    cwd: repoRoot,
    encoding: 'utf8',
  }).trim()
  if (!branch) {
    throw new Error('refusing GFS E2E mutation from a detached HEAD')
  }
  const marker = JSON.parse(
    await kubectl([
      '-n',
      HCC_NAMESPACE,
      'get',
      'configmap/clerum-pre-gate-sync-state',
      '-o',
      'json',
    ])
  ) as PreGateMarkerResource
  if (marker.data?.worktreeId !== expectedWorktreeId) {
    throw new Error(
      'refusing GFS E2E mutation: cluster ownership marker does not match this worktree; run pre-gate-sync first'
    )
  }
  if (marker.data?.gitHead !== expectedHead) {
    throw new Error(
      `refusing GFS E2E mutation: cluster gitHead ${marker.data?.gitHead || '<missing>'} does not match ${expectedHead}`
    )
  }
  const markerData = marker.data
  const expectedImageSource = process.env.E2E_IMAGE_SOURCE || process.env.IMAGE_SOURCE || 'local'
  const expectedImageTag = process.env.E2E_IMAGE_TAG || process.env.IMAGE_TAG
  const expectedClusterFingerprint =
    process.env.E2E_EXPECTED_CLUSTER_FINGERPRINT || process.env.E2E_CLUSTER_FINGERPRINT
  const expectedImagesGeneratedAt = process.env.E2E_EXPECTED_IMAGES_GENERATED_AT
  validateGfsRuntimeImageMarker(markerData ?? {}, {
    imageSource: expectedImageSource,
    ...(expectedImageTag !== undefined ? { imageTag: expectedImageTag } : {}),
    ...(expectedClusterFingerprint ? { clusterFingerprint: expectedClusterFingerprint } : {}),
    ...(expectedImagesGeneratedAt ? { imagesGeneratedAt: expectedImagesGeneratedAt } : {}),
  })
  if (!marker.metadata?.uid) {
    throw new Error('refusing GFS E2E mutation: pre-gate marker has no immutable UID')
  }
  return {
    context,
    branch,
    worktreeId: expectedWorktreeId,
    gitHead: expectedHead,
    clusterFingerprint: markerData?.clusterFingerprint ?? '',
    preGateMarkerUid: marker.metadata.uid,
  }
}

async function assertOwnedRecoveryContext(marker: GfsUploadProductMutationMarker): Promise<void> {
  const context = kubectlContext()
  if (/prod|production/i.test(context) || !BRANCH_CONTEXT_RE.test(context)) {
    throw new Error(`refusing GFS E2E recovery on non-branch or production-like context ${context}`)
  }
  if (context !== marker.ownership.context) {
    throw new Error(
      `refusing GFS E2E recovery: marker context ${marker.ownership.context} does not match ${context}`
    )
  }
  const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
    cwd: process.cwd(),
    encoding: 'utf8',
  }).trim()
  const branch = execFileSync('git', ['branch', '--show-current'], {
    cwd: repoRoot,
    encoding: 'utf8',
  }).trim()
  if (!branch || branch !== marker.ownership.branch) {
    throw new Error(
      `refusing GFS E2E recovery: marker branch ${marker.ownership.branch} does not match ` +
        `${branch || '<detached>'}`
    )
  }
  const preGate = JSON.parse(
    await kubectl([
      '-n',
      HCC_NAMESPACE,
      'get',
      'configmap/clerum-pre-gate-sync-state',
      '-o',
      'json',
    ])
  ) as PreGateMarkerResource
  if (
    preGate.metadata?.uid !== marker.ownership.preGateMarkerUid ||
    preGate.data?.clusterFingerprint !== marker.ownership.clusterFingerprint
  ) {
    throw new Error(
      'refusing GFS E2E recovery: cluster ownership evidence no longer matches the recorded lease'
    )
  }
}

async function authorizeNewProductMutation(): Promise<GfsUploadProductMutationOwnership> {
  const ownership = await assertOwnedRuntimeContext()
  if (!BRANCH_CONTEXT_RE.test(ownership.context)) {
    throw new Error(
      `refusing GFS product-limit mutation on non-branch-owned context ${ownership.context}`
    )
  }
  return ownership
}

async function kubectl(args: string[], timeout = 240_000): Promise<string> {
  const result = await execFileAsync('kubectl', ['--context', kubectlContext(), ...args], {
    encoding: 'utf8',
    timeout,
    maxBuffer: 2 * 1024 * 1024,
  })
  return result.stdout
}

async function kubectlWithInput(args: string[], input: string, timeout = 240_000): Promise<string> {
  return new Promise((resolvePromise, reject) => {
    const child = spawn('kubectl', ['--context', kubectlContext(), ...args], {
      stdio: ['pipe', 'pipe', 'pipe'],
    })
    let stdout = ''
    let stderr = ''
    const timer = setTimeout(() => child.kill('SIGTERM'), timeout)
    child.stdout.setEncoding('utf8')
    child.stderr.setEncoding('utf8')
    child.stdout.on('data', chunk => {
      stdout += chunk
    })
    child.stderr.on('data', chunk => {
      stderr += chunk
    })
    child.on('error', error => {
      clearTimeout(timer)
      reject(error)
    })
    child.on('close', code => {
      clearTimeout(timer)
      if (code === 0) resolvePromise(stdout)
      else reject(new Error(`kubectl exited with ${code}: ${stderr.trim() || '<no stderr>'}`))
    })
    child.stdin.end(input)
  })
}

async function renderedWriterFlag(): Promise<string> {
  return (
    await kubectl([
      '-n',
      GFS_NAMESPACE,
      'get',
      `deployment/${GFS_WRITER_DEPLOYMENT}`,
      '-o',
      `jsonpath={range .spec.template.spec.containers[*]}{range .env[?(@.name=="${GFS_V2_WRITER_ENV}")]}{.value}{end}{end}`,
    ])
  ).trim()
}

interface RenderedWriterEnv {
  name?: unknown
  value?: unknown
}

interface RenderedWriterDeployment {
  spec?: {
    template?: {
      spec?: {
        containers?: Array<{ env?: RenderedWriterEnv[] }>
      }
    }
  }
}

function parseDeploymentEnvState(
  containers: Array<{ env?: RenderedWriterEnv[] }>,
  envName: string,
  owner: string
): GfsUploadProductMaxState {
  const matches = containers.flatMap(container => {
    if (container.env === undefined) return []
    if (!Array.isArray(container.env)) {
      throw new Error(`invalid ${envName} ${owner} deployment env shape`)
    }
    return container.env.filter(env => env?.name === envName)
  })
  if (matches.length === 0) return { kind: 'absent' }
  if (matches.length !== 1 || typeof matches[0]?.value !== 'string') {
    throw new Error(`invalid ${envName} ${owner} deployment value or duplicate authority`)
  }
  return {
    kind: 'explicit',
    value: validateGfsUploadProductMaxBytes(parseCanonicalProductMax(matches[0].value)),
  }
}

export function parseGfsUploadProductMaxInputs(raw: string): GfsUploadProductMaxInputs {
  let deployment: RenderedWriterDeployment
  try {
    deployment = JSON.parse(raw) as RenderedWriterDeployment
  } catch {
    throw new Error('invalid HCC product-limit deployment JSON')
  }
  const containers = deployment.spec?.template?.spec?.containers
  if (
    !Array.isArray(containers) ||
    containers.some(container => !container || typeof container !== 'object')
  ) {
    throw new Error('invalid HCC product-limit deployment shape')
  }
  const pair = {
    canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ENV, 'HCC'),
    alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ALIAS_ENV, 'HCC'),
  }
  if (
    pair.canonical.kind === 'explicit' &&
    pair.alias.kind === 'explicit' &&
    pair.canonical.value !== pair.alias.value
  ) {
    throw new Error('conflicting canonical and deprecated HCC GFS product-limit values')
  }
  return pair
}

function parseRenderedWriterProductMaxInputs(raw: string): GfsUploadProductMaxInputs {
  let deployment: RenderedWriterDeployment
  try {
    deployment = JSON.parse(raw) as RenderedWriterDeployment
  } catch {
    throw new Error('invalid GFSC writer product-limit deployment JSON')
  }
  const containers = deployment.spec?.template?.spec?.containers
  if (!Array.isArray(containers)) {
    throw new Error('invalid GFSC writer product-limit deployment shape')
  }
  return {
    canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ENV, 'GFSC writer'),
    alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ALIAS_ENV, 'GFSC writer'),
  }
}

export function buildGfsUploadProductMaxInputEnvArgs(pair: GfsUploadProductMaxInputs): string[] {
  const arg = (name: string, state: GfsUploadProductMaxState): string =>
    state.kind === 'absent'
      ? `${name}-`
      : `${name}=${validateGfsUploadProductMaxBytes(state.value)}`
  return [
    arg(GFS_PRODUCT_MAX_HCC_ENV, pair.canonical),
    arg(GFS_PRODUCT_MAX_HCC_ALIAS_ENV, pair.alias),
  ]
}

function productMaxPairForValue(value: number): GfsUploadProductMaxInputs {
  const validated = validateGfsUploadProductMaxBytes(value)
  return {
    canonical: { kind: 'explicit', value: validated },
    alias: { kind: 'explicit', value: validated },
  }
}

async function readHccProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {
  return parseGfsUploadProductMaxInputs(
    await kubectl(['-n', HCC_NAMESPACE, 'get', `deployment/${HCC_DEPLOYMENT}`, '-o', 'json'])
  )
}

async function readWriterProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {
  return parseRenderedWriterProductMaxInputs(
    await kubectl(['-n', GFS_NAMESPACE, 'get', `deployment/${GFS_WRITER_DEPLOYMENT}`, '-o', 'json'])
  )
}

/**
 * Reads the complete deployment object so an absent env can be distinguished
 * from a present-but-malformed value. A malformed deployment or duplicate
 * authoritative env fails closed instead of being treated as the default.
 */
export function parseRenderedGfsUploadProductMax(raw: string): string | undefined {
  let deployment: RenderedWriterDeployment
  try {
    deployment = JSON.parse(raw) as RenderedWriterDeployment
  } catch {
    throw new Error(`invalid rendered ${GFS_PRODUCT_MAX_WRITER_ENV} deployment JSON`)
  }

  const containers = deployment.spec?.template?.spec?.containers
  if (!Array.isArray(containers)) {
    throw new Error(`invalid rendered ${GFS_PRODUCT_MAX_WRITER_ENV} deployment shape`)
  }
  const matches = containers.flatMap(container => {
    if (container === null || typeof container !== 'object') {
      throw new Error(`invalid rendered ${GFS_PRODUCT_MAX_WRITER_ENV} container shape`)
    }
    if (container.env === undefined) return []
    if (!Array.isArray(container.env)) {
      throw new Error(`invalid rendered ${GFS_PRODUCT_MAX_WRITER_ENV} env shape`)
    }
    return container.env.filter(env => env?.name === GFS_PRODUCT_MAX_WRITER_ENV)
  })
  if (matches.length === 0) return undefined
  if (matches.length !== 1 || typeof matches[0]?.value !== 'string') {
    throw new Error(`invalid rendered ${GFS_PRODUCT_MAX_WRITER_ENV} value`)
  }
  return matches[0].value
}

export function parseGfsUploadProductMaxState(raw: string): GfsUploadProductMaxState {
  const value = parseRenderedGfsUploadProductMax(raw)
  if (value === undefined) return { kind: 'absent' }
  return {
    kind: 'explicit',
    value: validateGfsUploadProductMaxBytes(parseCanonicalProductMax(value)),
  }
}

function parseCanonicalProductMax(value: string): number {
  if (!/^[1-9]\d*$/.test(value)) {
    throw new Error(
      `GFS writer rendered an invalid ${GFS_PRODUCT_MAX_WRITER_ENV} value: ${value || '<empty>'}`
    )
  }
  return Number(value)
}

async function renderedWriterProductMax(): Promise<string | undefined> {
  return parseRenderedGfsUploadProductMax(
    await kubectl(['-n', GFS_NAMESPACE, 'get', `deployment/${GFS_WRITER_DEPLOYMENT}`, '-o', 'json'])
  )
}

async function waitForRenderedWriterFlag(expected: boolean): Promise<void> {
  const expectedValue = String(expected)
  const deadline = Date.now() + 240_000
  let lastValue = ''
  while (Date.now() < deadline) {
    lastValue = await renderedWriterFlag()
    if (lastValue === expectedValue) return
    await new Promise(resolve => setTimeout(resolve, 1_000))
  }
  throw new Error(
    `GFS Upload v2 runtime flag did not converge to ${expectedValue}; observed ${lastValue || '<empty>'}`
  )
}

async function waitForHccProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {
  const deadline = Date.now() + 240_000
  let lastValue: GfsUploadProductMaxInputs | undefined
  while (Date.now() < deadline) {
    lastValue = await readHccProductMaxInputs()
    if (sameGfsUploadProductEnvPair(lastValue, expected)) return
    await new Promise(resolve => setTimeout(resolve, 1_000))
  }
  throw new Error(
    `HCC GFS Upload v2 product inputs did not converge to ${JSON.stringify(expected)}; ` +
      `observed ${JSON.stringify(lastValue)}`
  )
}

async function waitForWriterProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {
  const deadline = Date.now() + 240_000
  let lastValue: GfsUploadProductMaxInputs | undefined
  while (Date.now() < deadline) {
    lastValue = await readWriterProductMaxInputs()
    if (sameGfsUploadProductEnvPair(lastValue, expected)) return
    await new Promise(resolve => setTimeout(resolve, 1_000))
  }
  throw new Error(
    `GFSC writer product inputs did not converge to ${JSON.stringify(expected)}; ` +
      `observed ${JSON.stringify(lastValue)}`
  )
}

export function validateGfsUploadProductMaxBytes(value: unknown): number {
  if (
    !Number.isSafeInteger(value) ||
    (value as number) < 1 ||
    (value as number) > GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES
  ) {
    throw new Error(
      `invalid GFS Upload v2 product maximum; expected an integer from 1 through ${GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES}`
    )
  }
  return value as number
}

async function waitForDeployment(deployment: string): Promise<void> {
  await kubectl([
    '-n',
    GFS_NAMESPACE,
    'rollout',
    'status',
    `deployment/${deployment}`,
    '--timeout=240s',
  ])
}

/**
 * Changes the HCC-owned source flag and waits for the generated writer to
 * converge. The caller must restore the original value in a finally block.
 */
export async function setGfsUploadV2Enabled(enabled: boolean): Promise<void> {
  await assertOwnedRuntimeContext()
  await kubectl([
    '-n',
    HCC_NAMESPACE,
    'set',
    'env',
    `deployment/${HCC_DEPLOYMENT}`,
    `${GFS_V2_HCC_ENV}=${String(enabled)}`,
  ])
  await kubectl([
    '-n',
    HCC_NAMESPACE,
    'rollout',
    'status',
    `deployment/${HCC_DEPLOYMENT}`,
    '--timeout=240s',
  ])
  await waitForRenderedWriterFlag(enabled)
  await waitForDeployment(GFS_WRITER_DEPLOYMENT)
  await waitForDeployment(GFS_READER_DEPLOYMENT)
}

export async function readGfsUploadV2Enabled(): Promise<boolean> {
  const value = await renderedWriterFlag()
  if (value !== 'true' && value !== 'false') {
    throw new Error(
      `GFS writer rendered an invalid ${GFS_V2_WRITER_ENV} value: ${value || '<empty>'}`
    )
  }
  return value === 'true'
}

/**
 * Changes the HCC-owned product policy and waits for the generated writer and
 * reader to converge. The caller must restore the original value in a finally
 * block. This changes policy only; the compiled protocol geometry is untouched.
 */
export function buildGfsUploadProductMaxEnvArgs(
  state: GfsUploadProductMaxState | number
): string[] {
  if (typeof state === 'number') {
    const validated = validateGfsUploadProductMaxBytes(state)
    return [
      `${GFS_PRODUCT_MAX_HCC_ENV}=${String(validated)}`,
      `${GFS_PRODUCT_MAX_HCC_ALIAS_ENV}=${String(validated)}`,
    ]
  }
  if (state.kind === 'absent') {
    return [`${GFS_PRODUCT_MAX_HCC_ENV}-`, `${GFS_PRODUCT_MAX_HCC_ALIAS_ENV}-`]
  }
  if (state.kind === 'explicit') {
    const validated = validateGfsUploadProductMaxBytes(state.value)
    return [
      `${GFS_PRODUCT_MAX_HCC_ENV}=${String(validated)}`,
      `${GFS_PRODUCT_MAX_HCC_ALIAS_ENV}=${String(validated)}`,
    ]
  }
  throw new Error('invalid GFS Upload v2 product maximum restoration state')
}

async function applyHccProductMaxInputs(pair: GfsUploadProductMaxInputs): Promise<void> {
  const envArgs = buildGfsUploadProductMaxInputEnvArgs(pair)
  await kubectl(['-n', HCC_NAMESPACE, 'set', 'env', `deployment/${HCC_DEPLOYMENT}`, ...envArgs])
  await kubectl([
    '-n',
    HCC_NAMESPACE,
    'rollout',
    'status',
    `deployment/${HCC_DEPLOYMENT}`,
    '--timeout=240s',
  ])
  await waitForHccProductMaxInputs(pair)
  await waitForWriterProductMaxInputs(pair)
  await waitForDeployment(GFS_WRITER_DEPLOYMENT)
  await waitForDeployment(GFS_READER_DEPLOYMENT)
}

interface ProductMutationMarkerResource {
  metadata?: { uid?: string }
  immutable?: boolean
  data?: Record<string, string | undefined>
}

async function readProductMutationMarker(): Promise<GfsUploadProductMutationLease | undefined> {
  const raw = (
    await kubectl([
      '-n',
      HCC_NAMESPACE,
      'get',
      `configmap/${GFS_PRODUCT_MUTATION_MARKER}`,
      '--ignore-not-found',
      '-o',
      'json',
    ])
  ).trim()
  if (!raw) return undefined
  let resource: ProductMutationMarkerResource
  try {
    resource = JSON.parse(raw) as ProductMutationMarkerResource
  } catch {
    throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap JSON')
  }
  const uid = resource.metadata?.uid
  const markerRaw = resource.data?.[GFS_PRODUCT_MUTATION_MARKER_KEY]
  if (!uid || resource.immutable !== true || !markerRaw) {
    throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap shape')
  }
  return { uid, marker: parseGfsUploadProductMutationMarker(markerRaw) }
}

async function createProductMutationMarker(
  marker: GfsUploadProductMutationMarker
): Promise<{ uid: string }> {
  const raw = await kubectlWithInput(
    ['-n', HCC_NAMESPACE, 'create', '-f', '-', '-o', 'json'],
    JSON.stringify({
      apiVersion: 'v1',
      kind: 'ConfigMap',
      metadata: {
        name: GFS_PRODUCT_MUTATION_MARKER,
        namespace: HCC_NAMESPACE,
        labels: { 'e2e.clerum.io/coordination': 'gfs-upload-product-limit' },
      },
      immutable: true,
      data: {
        [GFS_PRODUCT_MUTATION_MARKER_KEY]: serializeGfsUploadProductMutationMarker(marker),
      },
    })
  )
  const created = JSON.parse(raw) as ProductMutationMarkerResource
  if (!created.metadata?.uid || created.immutable !== true) {
    throw new Error('created GFS Upload v2 product-limit recovery marker has no immutable UID')
  }
  return { uid: created.metadata.uid }
}

export function buildGfsUploadProductMarkerDeleteRequest(uid: string): {
  args: string[]
  input: string
} {
  if (!uid) throw new Error('refusing GFS product-limit marker deletion without a UID')
  return {
    args: [
      '-n',
      HCC_NAMESPACE,
      'delete',
      '--raw',
      `/api/v1/namespaces/${HCC_NAMESPACE}/configmaps/${GFS_PRODUCT_MUTATION_MARKER}`,
      '-f',
      '-',
      '--wait=true',
    ],
    input: JSON.stringify({
      apiVersion: 'v1',
      kind: 'DeleteOptions',
      preconditions: { uid },
    }),
  }
}

async function deleteProductMutationMarker(lease: GfsUploadProductMutationLease): Promise<void> {
  const request = buildGfsUploadProductMarkerDeleteRequest(lease.uid)
  await kubectlWithInput(request.args, request.input)
}

const productMutationAdapter: GfsUploadProductMutationAdapter = {
  authorizeNewMutation: authorizeNewProductMutation,
  authorizeRecovery: assertOwnedRecoveryContext,
  readBaseline: readHccProductMaxInputs,
  readMarker: readProductMutationMarker,
  createMarker: createProductMutationMarker,
  applyPair: applyHccProductMaxInputs,
  verifyPair: async pair => {
    const actual = await readHccProductMaxInputs()
    if (!sameGfsUploadProductEnvPair(actual, pair)) {
      throw new Error(
        `HCC GFS product inputs do not match the recorded state: ${JSON.stringify(actual)}`
      )
    }
    await waitForWriterProductMaxInputs(pair)
  },
  deleteMarker: deleteProductMutationMarker,
  newHolder: randomUUID,
  now: () => new Date().toISOString(),
  waitBeforeRetry: async attempt => {
    await new Promise(resolve => setTimeout(resolve, attempt * 1_000))
  },
}

/**
 * Starts a branch-owned mutation transaction. The durable immutable marker is
 * created and read back before this function returns, so callers cannot change
 * HCC policy without first recording both original HCC inputs.
 */
export async function beginGfsUploadProductMaxMutation(): Promise<GfsUploadProductMutationLease> {
  return beginGfsUploadProductMutation(productMutationAdapter)
}

export async function setGfsUploadProductMaxBytes(
  lease: GfsUploadProductMutationLease,
  value: number
): Promise<void> {
  await applyGfsUploadProductMutation(productMutationAdapter, lease, productMaxPairForValue(value))
}

/** Restores the exact recorded HCC input pair without re-running new-mutation Git checks. */
export async function restoreGfsUploadProductMaxBytes(
  lease: GfsUploadProductMutationLease
): Promise<void> {
  await restoreGfsUploadProductMutation(productMutationAdapter, lease)
}

/**
 * Recovers an interrupted transaction from its cluster marker. This path still
 * verifies the branch context and cluster lease, but intentionally does not
 * require the originating worktree or its cleanliness.
 */
export async function recoverGfsUploadProductMaxBytes(expected: {
  uid: string
  holder: string
}): Promise<boolean> {
  return recoverGfsUploadProductMutation(productMutationAdapter, expected)
}

export async function readGfsUploadProductMaxBytes(): Promise<GfsUploadProductMaxState> {
  const value = await renderedWriterProductMax()
  if (value === undefined) return { kind: 'absent' }
  return {
    kind: 'explicit',
    value: validateGfsUploadProductMaxBytes(parseCanonicalProductMax(value)),
  }
}

export async function restartGfsWriter(): Promise<void> {
  await assertOwnedRuntimeContext()
  // Restart the real writer deployment while the upload is visibly active.
  // A rollout gives the relay a bounded termination path; deleting the pod
  // directly can leave an already-streaming PUT socket open until the full
  // five-minute part timeout, turning this journey into a transport-timeout
  // test instead of a restart/recovery test.
  await kubectl(['-n', GFS_NAMESPACE, 'rollout', 'restart', `deployment/${GFS_WRITER_DEPLOYMENT}`])
  await waitForDeployment(GFS_WRITER_DEPLOYMENT)
}

/** Revoke only the exact seeded user grant used by the Desktop E2E. */
export async function revokeGfsUserWriteGrant(resourceId: string, userId: string): Promise<void> {
  await assertOwnedRuntimeContext()
  runControlPostgresSql(`
    DELETE FROM gfs_grants
     WHERE drive = 'main'
       AND resource_id = ${sqlLiteral(resourceId)}::uuid
       AND subject_type = 'user'
       AND subject_id = ${sqlLiteral(userId)};
  `)
}

/**
 * Legacy fallback must not create an indexed Upload v2 session. Scope the
 * assertion to the exact seeded parent/name/owner tuple so unrelated active
 * sessions cannot make the negative journey flaky.
 */
export function countGfsCreateSessions(
  parentResourceId: string,
  resourceName: string,
  ownerSubject: string
): number {
  const value = firstDataLine(
    runControlPostgresSql(`
      SELECT COUNT(*)::int
        FROM gfs_upload_sessions
       WHERE drive = 'main'
         AND owner_subject = ${sqlLiteral(ownerSubject)}
         AND operation = 'create'
         AND parent_rid = ${sqlLiteral(parentResourceId)}
         AND resource_name = ${sqlLiteral(resourceName)};
    `)
  )
  const count = Number(value)
  if (!Number.isSafeInteger(count) || count < 0) {
    throw new Error(`invalid GFS v2 session count: ${value || '<empty>'}`)
  }
  return count
}