Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(e2e): r4-h3 make upload policy mutation recoverable

Josuéc777c4255184
diff --git a/tests/e2e/gfsUploadProductMutation.ts b/tests/e2e/gfsUploadProductMutation.ts
new file mode 100644
index 000000000..87f580ec1
--- /dev/null
+++ b/tests/e2e/gfsUploadProductMutation.ts
@@ -0,0 +1,219 @@
+export type GfsUploadProductEnvState = { kind: 'absent' } | { kind: 'explicit'; value: number }
+
+export interface GfsUploadProductEnvPair {
+ canonical: GfsUploadProductEnvState
+ alias: GfsUploadProductEnvState
+}
+
+export interface GfsUploadProductMutationOwnership {
+ context: string
+ branch: string
+ worktreeId: string
+ gitHead: string
+ clusterFingerprint: string
+ preGateMarkerUid: string
+}
+
+export interface GfsUploadProductMutationMarker {
+ version: 1
+ state: 'recovery-required'
+ holder: string
+ startedAt: string
+ ownership: GfsUploadProductMutationOwnership
+ baseline: GfsUploadProductEnvPair
+}
+
+export interface GfsUploadProductMutationLease {
+ uid: string
+ marker: GfsUploadProductMutationMarker
+}
+
+export interface GfsUploadProductMutationAdapter {
+ authorizeNewMutation(): Promise<GfsUploadProductMutationOwnership>
+ authorizeRecovery(marker: GfsUploadProductMutationMarker): Promise<void>
+ readBaseline(): Promise<GfsUploadProductEnvPair>
+ readMarker(): Promise<GfsUploadProductMutationLease | undefined>
+ createMarker(marker: GfsUploadProductMutationMarker): Promise<{ uid: string }>
+ applyPair(pair: GfsUploadProductEnvPair): Promise<void>
+ verifyPair(pair: GfsUploadProductEnvPair): Promise<void>
+ deleteMarker(lease: GfsUploadProductMutationLease): Promise<void>
+ newHolder(): string
+ now(): string
+ waitBeforeRetry(attempt: number): Promise<void>
+}
+
+const RESTORE_ATTEMPTS = 3
+
+function sameState(left: GfsUploadProductEnvState, right: GfsUploadProductEnvState): boolean {
+ return (
+ left.kind === right.kind &&
+ (left.kind === 'absent' || (right.kind === 'explicit' && left.value === right.value))
+ )
+}
+
+export function sameGfsUploadProductEnvPair(
+ left: GfsUploadProductEnvPair,
+ right: GfsUploadProductEnvPair
+): boolean {
+ return sameState(left.canonical, right.canonical) && sameState(left.alias, right.alias)
+}
+
+export function serializeGfsUploadProductMutationMarker(
+ marker: GfsUploadProductMutationMarker
+): string {
+ return JSON.stringify(marker)
+}
+
+export function parseGfsUploadProductMutationMarker(raw: string): GfsUploadProductMutationMarker {
+ let value: unknown
+ try {
+ value = JSON.parse(raw)
+ } catch {
+ throw new Error('invalid GFS Upload v2 product-limit recovery marker JSON')
+ }
+ if (!value || typeof value !== 'object') {
+ throw new Error('invalid GFS Upload v2 product-limit recovery marker shape')
+ }
+ const marker = value as Partial<GfsUploadProductMutationMarker>
+ const ownership = marker.ownership as Partial<GfsUploadProductMutationOwnership> | undefined
+ if (
+ marker.version !== 1 ||
+ marker.state !== 'recovery-required' ||
+ typeof marker.holder !== 'string' ||
+ marker.holder.length === 0 ||
+ typeof marker.startedAt !== 'string' ||
+ marker.startedAt.length === 0 ||
+ !ownership ||
+ typeof ownership.context !== 'string' ||
+ typeof ownership.branch !== 'string' ||
+ typeof ownership.worktreeId !== 'string' ||
+ typeof ownership.gitHead !== 'string' ||
+ typeof ownership.clusterFingerprint !== 'string' ||
+ typeof ownership.preGateMarkerUid !== 'string' ||
+ !marker.baseline ||
+ !isEnvState(marker.baseline.canonical) ||
+ !isEnvState(marker.baseline.alias)
+ ) {
+ throw new Error('invalid GFS Upload v2 product-limit recovery marker shape')
+ }
+ return marker as GfsUploadProductMutationMarker
+}
+
+function isEnvState(value: unknown): value is GfsUploadProductEnvState {
+ if (!value || typeof value !== 'object') return false
+ const state = value as Partial<GfsUploadProductEnvState>
+ return (
+ state.kind === 'absent' ||
+ (state.kind === 'explicit' && Number.isSafeInteger(state.value) && (state.value as number) >= 1)
+ )
+}
+
+function assertSameLease(
+ expected: GfsUploadProductMutationLease,
+ actual: GfsUploadProductMutationLease | undefined
+): void {
+ if (
+ !actual ||
+ actual.uid !== expected.uid ||
+ actual.marker.holder !== expected.marker.holder ||
+ serializeGfsUploadProductMutationMarker(actual.marker) !==
+ serializeGfsUploadProductMutationMarker(expected.marker)
+ ) {
+ throw new Error(
+ 'GFS Upload v2 product-limit recovery marker changed; refusing mutation or cleanup'
+ )
+ }
+}
+
+export async function beginGfsUploadProductMutation(
+ adapter: GfsUploadProductMutationAdapter
+): Promise<GfsUploadProductMutationLease> {
+ const ownership = await adapter.authorizeNewMutation()
+ const existing = await adapter.readMarker()
+ if (existing) {
+ throw new Error(
+ `GFS Upload v2 product-limit recovery is required for uid ${existing.uid}, ` +
+ `holder ${existing.marker.holder}; explicitly confirm both lease values before restoring ` +
+ 'the recorded baseline or starting another mutation'
+ )
+ }
+ const baseline = await adapter.readBaseline()
+ const marker: GfsUploadProductMutationMarker = {
+ version: 1,
+ state: 'recovery-required',
+ holder: adapter.newHolder(),
+ startedAt: adapter.now(),
+ ownership,
+ baseline,
+ }
+ const created = await adapter.createMarker(marker)
+ if (!created.uid) {
+ throw new Error('GFS Upload v2 product-limit recovery marker has no immutable UID')
+ }
+ const lease = { uid: created.uid, marker }
+ await adapter.authorizeRecovery(marker)
+ assertSameLease(lease, await adapter.readMarker())
+ return lease
+}
+
+export async function applyGfsUploadProductMutation(
+ adapter: GfsUploadProductMutationAdapter,
+ lease: GfsUploadProductMutationLease,
+ pair: GfsUploadProductEnvPair
+): Promise<void> {
+ await adapter.authorizeRecovery(lease.marker)
+ assertSameLease(lease, await adapter.readMarker())
+ await adapter.applyPair(pair)
+ await adapter.verifyPair(pair)
+ await adapter.authorizeRecovery(lease.marker)
+ assertSameLease(lease, await adapter.readMarker())
+}
+
+async function restoreWithRetries(
+ adapter: GfsUploadProductMutationAdapter,
+ lease: GfsUploadProductMutationLease
+): Promise<void> {
+ let lastError: unknown
+ for (let attempt = 1; attempt <= RESTORE_ATTEMPTS; attempt += 1) {
+ try {
+ await adapter.authorizeRecovery(lease.marker)
+ assertSameLease(lease, await adapter.readMarker())
+ await adapter.applyPair(lease.marker.baseline)
+ await adapter.verifyPair(lease.marker.baseline)
+ await adapter.authorizeRecovery(lease.marker)
+ assertSameLease(lease, await adapter.readMarker())
+ await adapter.deleteMarker(lease)
+ return
+ } catch (error) {
+ lastError = error
+ if (attempt < RESTORE_ATTEMPTS) await adapter.waitBeforeRetry(attempt)
+ }
+ }
+ throw new Error(
+ `GFS Upload v2 product-limit recovery remains required after ${RESTORE_ATTEMPTS} attempts; ` +
+ `holder=${lease.marker.holder}; baseline=${JSON.stringify(lease.marker.baseline)}; ` +
+ `cause=${lastError instanceof Error ? lastError.message : String(lastError)}`
+ )
+}
+
+export async function restoreGfsUploadProductMutation(
+ adapter: GfsUploadProductMutationAdapter,
+ lease: GfsUploadProductMutationLease
+): Promise<void> {
+ await adapter.authorizeRecovery(lease.marker)
+ await restoreWithRetries(adapter, lease)
+}
+
+export async function recoverGfsUploadProductMutation(
+ adapter: GfsUploadProductMutationAdapter,
+ expected: { uid: string; holder: string }
+): Promise<boolean> {
+ const lease = await adapter.readMarker()
+ if (!lease) return false
+ if (lease.uid !== expected.uid || lease.marker.holder !== expected.holder) {
+ throw new Error('GFS Upload v2 product-limit recovery takeover evidence does not match')
+ }
+ await adapter.authorizeRecovery(lease.marker)
+ await restoreWithRetries(adapter, lease)
+ return true
+}
diff --git a/tests/e2e/gfsUploadV2Runtime.ts b/tests/e2e/gfsUploadV2Runtime.ts
index 819a28b3f..3a4c9404f 100644
--- a/tests/e2e/gfsUploadV2Runtime.ts
+++ b/tests/e2e/gfsUploadV2Runtime.ts
@@ -1,10 +1,25 @@
-import { execFile, execFileSync } from 'node:child_process'
-import { createHash } from 'node:crypto'
+import { execFile, execFileSync, spawn } from 'node:child_process'
+import { createHash, randomUUID } from 'node:crypto'
import { existsSync, readFileSync } from 'node:fs'
import { homedir } from 'node:os'
import { resolve } from 'node:path'
import { promisify } from 'node:util'
import { firstDataLine, kubectlContext, runControlPostgresSql, sqlLiteral } from './gfsFixtureCore'
+import {
+ type GfsUploadProductEnvPair,
+ type GfsUploadProductEnvState,
+ type GfsUploadProductMutationAdapter,
+ type GfsUploadProductMutationLease,
+ type GfsUploadProductMutationMarker,
+ type GfsUploadProductMutationOwnership,
+ applyGfsUploadProductMutation,
+ beginGfsUploadProductMutation,
+ parseGfsUploadProductMutationMarker,
+ recoverGfsUploadProductMutation,
+ restoreGfsUploadProductMutation,
+ sameGfsUploadProductEnvPair,
+ serializeGfsUploadProductMutationMarker,
+} from './gfsUploadProductMutation'
import { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'
export { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'
@@ -20,6 +35,9 @@ const GFS_V2_WRITER_ENV = 'GFS_UPLOAD_V2_ENABLED'
const GFS_PRODUCT_MAX_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_PRODUCT_MAX_FILE_BYTES'
const GFS_PRODUCT_MAX_HCC_ALIAS_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_MAX_FILE_BYTES'
const GFS_PRODUCT_MAX_WRITER_ENV = 'GFS_UPLOAD_PRODUCT_MAX_FILE_BYTES'
+const GFS_PRODUCT_MAX_WRITER_ALIAS_ENV = 'GFS_UPLOAD_MAX_FILE_BYTES'
+const GFS_PRODUCT_MUTATION_MARKER = 'gfs-upload-product-limit-recovery'
+const GFS_PRODUCT_MUTATION_MARKER_KEY = 'transaction.json'
const BRANCH_CONTEXT_RE = /^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/
export interface GfsRuntimeImageMarker {
@@ -36,7 +54,8 @@ export interface ExpectedGfsRuntimeImageMarker {
imagesGeneratedAt?: string
}
-export type GfsUploadProductMaxState = { kind: 'absent' } | { kind: 'explicit'; value: number }
+export type GfsUploadProductMaxState = GfsUploadProductEnvState
+export type GfsUploadProductMaxInputs = GfsUploadProductEnvPair
/**
* Local Minikube image mode intentionally has no registry tag. GHCR mode must
@@ -113,7 +132,21 @@ function normalizeUrl(value: string): string {
* name alone is not ownership proof: another healthy Minikube profile could
* otherwise receive an HCC mutation or writer restart.
*/
-async function assertOwnedRuntimeContext(): Promise<void> {
+interface PreGateMarkerData {
+ worktreeId?: string
+ gitHead?: string
+ clusterFingerprint?: string
+ imageSource?: string
+ imageTag?: string
+ imagesGeneratedAt?: string
+}
+
+interface PreGateMarkerResource {
+ metadata?: { uid?: string }
+ data?: PreGateMarkerData
+}
+
+async function assertOwnedRuntimeContext(): Promise<GfsUploadProductMutationOwnership> {
const context = kubectlContext()
if (/prod|production/i.test(context)) {
throw new Error(`refusing GFS E2E mutation on production-like context ${context}`)
@@ -171,6 +204,13 @@ async function assertOwnedRuntimeContext(): Promise<void> {
cwd: repoRoot,
encoding: 'utf8',
}).trim()
+ const branch = execFileSync('git', ['branch', '--show-current'], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ }).trim()
+ if (!branch) {
+ throw new Error('refusing GFS E2E mutation from a detached HEAD')
+ }
const marker = JSON.parse(
await kubectl([
'-n',
@@ -180,16 +220,7 @@ async function assertOwnedRuntimeContext(): Promise<void> {
'-o',
'json',
])
- ) as {
- data?: {
- worktreeId?: string
- gitHead?: string
- clusterFingerprint?: string
- imageSource?: string
- imageTag?: string
- imagesGeneratedAt?: string
- }
- }
+ ) as PreGateMarkerResource
if (marker.data?.worktreeId !== expectedWorktreeId) {
throw new Error(
'refusing GFS E2E mutation: cluster ownership marker does not match this worktree; run pre-gate-sync first'
@@ -212,6 +243,71 @@ async function assertOwnedRuntimeContext(): Promise<void> {
...(expectedClusterFingerprint ? { clusterFingerprint: expectedClusterFingerprint } : {}),
...(expectedImagesGeneratedAt ? { imagesGeneratedAt: expectedImagesGeneratedAt } : {}),
})
+ if (!marker.metadata?.uid) {
+ throw new Error('refusing GFS E2E mutation: pre-gate marker has no immutable UID')
+ }
+ return {
+ context,
+ branch,
+ worktreeId: expectedWorktreeId,
+ gitHead: expectedHead,
+ clusterFingerprint: markerData?.clusterFingerprint ?? '',
+ preGateMarkerUid: marker.metadata.uid,
+ }
+}
+
+async function assertOwnedRecoveryContext(marker: GfsUploadProductMutationMarker): Promise<void> {
+ const context = kubectlContext()
+ if (/prod|production/i.test(context) || !BRANCH_CONTEXT_RE.test(context)) {
+ throw new Error(`refusing GFS E2E recovery on non-branch or production-like context ${context}`)
+ }
+ if (context !== marker.ownership.context) {
+ throw new Error(
+ `refusing GFS E2E recovery: marker context ${marker.ownership.context} does not match ${context}`
+ )
+ }
+ const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {
+ cwd: process.cwd(),
+ encoding: 'utf8',
+ }).trim()
+ const branch = execFileSync('git', ['branch', '--show-current'], {
+ cwd: repoRoot,
+ encoding: 'utf8',
+ }).trim()
+ if (!branch || branch !== marker.ownership.branch) {
+ throw new Error(
+ `refusing GFS E2E recovery: marker branch ${marker.ownership.branch} does not match ` +
+ `${branch || '<detached>'}`
+ )
+ }
+ const preGate = JSON.parse(
+ await kubectl([
+ '-n',
+ HCC_NAMESPACE,
+ 'get',
+ 'configmap/clerum-pre-gate-sync-state',
+ '-o',
+ 'json',
+ ])
+ ) as PreGateMarkerResource
+ if (
+ preGate.metadata?.uid !== marker.ownership.preGateMarkerUid ||
+ preGate.data?.clusterFingerprint !== marker.ownership.clusterFingerprint
+ ) {
+ throw new Error(
+ 'refusing GFS E2E recovery: cluster ownership evidence no longer matches the recorded lease'
+ )
+ }
+}
+
+async function authorizeNewProductMutation(): Promise<GfsUploadProductMutationOwnership> {
+ const ownership = await assertOwnedRuntimeContext()
+ if (!BRANCH_CONTEXT_RE.test(ownership.context)) {
+ throw new Error(
+ `refusing GFS product-limit mutation on non-branch-owned context ${ownership.context}`
+ )
+ }
+ return ownership
}
async function kubectl(args: string[], timeout = 240_000): Promise<string> {
@@ -223,6 +319,35 @@ async function kubectl(args: string[], timeout = 240_000): Promise<string> {
return result.stdout
}
+async function kubectlWithInput(args: string[], input: string, timeout = 240_000): Promise<string> {
+ return new Promise((resolvePromise, reject) => {
+ const child = spawn('kubectl', ['--context', kubectlContext(), ...args], {
+ stdio: ['pipe', 'pipe', 'pipe'],
+ })
+ let stdout = ''
+ let stderr = ''
+ const timer = setTimeout(() => child.kill('SIGTERM'), timeout)
+ child.stdout.setEncoding('utf8')
+ child.stderr.setEncoding('utf8')
+ child.stdout.on('data', chunk => {
+ stdout += chunk
+ })
+ child.stderr.on('data', chunk => {
+ stderr += chunk
+ })
+ child.on('error', error => {
+ clearTimeout(timer)
+ reject(error)
+ })
+ child.on('close', code => {
+ clearTimeout(timer)
+ if (code === 0) resolvePromise(stdout)
+ else reject(new Error(`kubectl exited with ${code}: ${stderr.trim() || '<no stderr>'}`))
+ })
+ child.stdin.end(input)
+ })
+}
+
async function renderedWriterFlag(): Promise<string> {
return (
await kubectl([
@@ -251,6 +376,104 @@ interface RenderedWriterDeployment {
}
}
+function parseDeploymentEnvState(
+ containers: Array<{ env?: RenderedWriterEnv[] }>,
+ envName: string,
+ owner: string
+): GfsUploadProductMaxState {
+ const matches = containers.flatMap(container => {
+ if (container.env === undefined) return []
+ if (!Array.isArray(container.env)) {
+ throw new Error(`invalid ${envName} ${owner} deployment env shape`)
+ }
+ return container.env.filter(env => env?.name === envName)
+ })
+ if (matches.length === 0) return { kind: 'absent' }
+ if (matches.length !== 1 || typeof matches[0]?.value !== 'string') {
+ throw new Error(`invalid ${envName} ${owner} deployment value or duplicate authority`)
+ }
+ return {
+ kind: 'explicit',
+ value: validateGfsUploadProductMaxBytes(parseCanonicalProductMax(matches[0].value)),
+ }
+}
+
+export function parseGfsUploadProductMaxInputs(raw: string): GfsUploadProductMaxInputs {
+ let deployment: RenderedWriterDeployment
+ try {
+ deployment = JSON.parse(raw) as RenderedWriterDeployment
+ } catch {
+ throw new Error('invalid HCC product-limit deployment JSON')
+ }
+ const containers = deployment.spec?.template?.spec?.containers
+ if (
+ !Array.isArray(containers) ||
+ containers.some(container => !container || typeof container !== 'object')
+ ) {
+ throw new Error('invalid HCC product-limit deployment shape')
+ }
+ const pair = {
+ canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ENV, 'HCC'),
+ alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ALIAS_ENV, 'HCC'),
+ }
+ if (
+ pair.canonical.kind === 'explicit' &&
+ pair.alias.kind === 'explicit' &&
+ pair.canonical.value !== pair.alias.value
+ ) {
+ throw new Error('conflicting canonical and deprecated HCC GFS product-limit values')
+ }
+ return pair
+}
+
+function parseRenderedWriterProductMaxInputs(raw: string): GfsUploadProductMaxInputs {
+ let deployment: RenderedWriterDeployment
+ try {
+ deployment = JSON.parse(raw) as RenderedWriterDeployment
+ } catch {
+ throw new Error('invalid GFSC writer product-limit deployment JSON')
+ }
+ const containers = deployment.spec?.template?.spec?.containers
+ if (!Array.isArray(containers)) {
+ throw new Error('invalid GFSC writer product-limit deployment shape')
+ }
+ return {
+ canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ENV, 'GFSC writer'),
+ alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ALIAS_ENV, 'GFSC writer'),
+ }
+}
+
+export function buildGfsUploadProductMaxInputEnvArgs(pair: GfsUploadProductMaxInputs): string[] {
+ const arg = (name: string, state: GfsUploadProductMaxState): string =>
+ state.kind === 'absent'
+ ? `${name}-`
+ : `${name}=${validateGfsUploadProductMaxBytes(state.value)}`
+ return [
+ arg(GFS_PRODUCT_MAX_HCC_ENV, pair.canonical),
+ arg(GFS_PRODUCT_MAX_HCC_ALIAS_ENV, pair.alias),
+ ]
+}
+
+function productMaxPairForValue(value: number): GfsUploadProductMaxInputs {
+ const validated = validateGfsUploadProductMaxBytes(value)
+ return {
+ canonical: { kind: 'explicit', value: validated },
+ alias: { kind: 'explicit', value: validated },
+ }
+}
+
+async function readHccProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {
+ return parseGfsUploadProductMaxInputs(
+ await kubectl(['-n', HCC_NAMESPACE, 'get', `deployment/${HCC_DEPLOYMENT}`, '-o', 'json'])
+ )
+}
+
+async function readWriterProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {
+ return parseRenderedWriterProductMaxInputs(
+ await kubectl(['-n', GFS_NAMESPACE, 'get', `deployment/${GFS_WRITER_DEPLOYMENT}`, '-o', 'json'])
+ )
+}
+
/**
* Reads the complete deployment object so an absent env can be distinguished
* from a present-but-malformed value. A malformed deployment or duplicate
@@ -323,17 +546,31 @@ async function waitForRenderedWriterFlag(expected: boolean): Promise<void> {
)
}
-async function waitForRenderedWriterProductMax(expected: number | undefined): Promise<void> {
- const expectedValue = expected === undefined ? '<absent>' : String(expected)
+async function waitForHccProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {
const deadline = Date.now() + 240_000
- let lastValue: string | undefined
+ let lastValue: GfsUploadProductMaxInputs | undefined
while (Date.now() < deadline) {
- lastValue = await renderedWriterProductMax()
- if (lastValue === (expected === undefined ? undefined : expectedValue)) return
+ lastValue = await readHccProductMaxInputs()
+ if (sameGfsUploadProductEnvPair(lastValue, expected)) return
await new Promise(resolve => setTimeout(resolve, 1_000))
}
throw new Error(
- `GFS Upload v2 product maximum did not converge to ${expectedValue}; observed ${lastValue || '<absent>'}`
+ `HCC GFS Upload v2 product inputs did not converge to ${JSON.stringify(expected)}; ` +
+ `observed ${JSON.stringify(lastValue)}`
+ )
+}
+
+async function waitForWriterProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {
+ const deadline = Date.now() + 240_000
+ let lastValue: GfsUploadProductMaxInputs | undefined
+ while (Date.now() < deadline) {
+ lastValue = await readWriterProductMaxInputs()
+ if (sameGfsUploadProductEnvPair(lastValue, expected)) return
+ await new Promise(resolve => setTimeout(resolve, 1_000))
+ }
+ throw new Error(
+ `GFSC writer product inputs did not converge to ${JSON.stringify(expected)}; ` +
+ `observed ${JSON.stringify(lastValue)}`
)
}
@@ -426,13 +663,8 @@ export function buildGfsUploadProductMaxEnvArgs(
throw new Error('invalid GFS Upload v2 product maximum restoration state')
}
-export async function setGfsUploadProductMaxBytes(
- state: GfsUploadProductMaxState | number
-): Promise<void> {
- const envArgs = buildGfsUploadProductMaxEnvArgs(state)
- const expected =
- typeof state === 'number' ? state : state.kind === 'explicit' ? state.value : undefined
- await assertOwnedRuntimeContext()
+async function applyHccProductMaxInputs(pair: GfsUploadProductMaxInputs): Promise<void> {
+ const envArgs = buildGfsUploadProductMaxInputEnvArgs(pair)
await kubectl(['-n', HCC_NAMESPACE, 'set', 'env', `deployment/${HCC_DEPLOYMENT}`, ...envArgs])
await kubectl([
'-n',
@@ -442,11 +674,159 @@ export async function setGfsUploadProductMaxBytes(
`deployment/${HCC_DEPLOYMENT}`,
'--timeout=240s',
])
- await waitForRenderedWriterProductMax(expected)
+ await waitForHccProductMaxInputs(pair)
+ await waitForWriterProductMaxInputs(pair)
await waitForDeployment(GFS_WRITER_DEPLOYMENT)
await waitForDeployment(GFS_READER_DEPLOYMENT)
}
+interface ProductMutationMarkerResource {
+ metadata?: { uid?: string }
+ immutable?: boolean
+ data?: Record<string, string | undefined>
+}
+
+async function readProductMutationMarker(): Promise<GfsUploadProductMutationLease | undefined> {
+ const raw = (
+ await kubectl([
+ '-n',
+ HCC_NAMESPACE,
+ 'get',
+ `configmap/${GFS_PRODUCT_MUTATION_MARKER}`,
+ '--ignore-not-found',
+ '-o',
+ 'json',
+ ])
+ ).trim()
+ if (!raw) return undefined
+ let resource: ProductMutationMarkerResource
+ try {
+ resource = JSON.parse(raw) as ProductMutationMarkerResource
+ } catch {
+ throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap JSON')
+ }
+ const uid = resource.metadata?.uid
+ const markerRaw = resource.data?.[GFS_PRODUCT_MUTATION_MARKER_KEY]
+ if (!uid || resource.immutable !== true || !markerRaw) {
+ throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap shape')
+ }
+ return { uid, marker: parseGfsUploadProductMutationMarker(markerRaw) }
+}
+
+async function createProductMutationMarker(
+ marker: GfsUploadProductMutationMarker
+): Promise<{ uid: string }> {
+ const raw = await kubectlWithInput(
+ ['-n', HCC_NAMESPACE, 'create', '-f', '-', '-o', 'json'],
+ JSON.stringify({
+ apiVersion: 'v1',
+ kind: 'ConfigMap',
+ metadata: {
+ name: GFS_PRODUCT_MUTATION_MARKER,
+ namespace: HCC_NAMESPACE,
+ labels: { 'e2e.clerum.io/coordination': 'gfs-upload-product-limit' },
+ },
+ immutable: true,
+ data: {
+ [GFS_PRODUCT_MUTATION_MARKER_KEY]: serializeGfsUploadProductMutationMarker(marker),
+ },
+ })
+ )
+ const created = JSON.parse(raw) as ProductMutationMarkerResource
+ if (!created.metadata?.uid || created.immutable !== true) {
+ throw new Error('created GFS Upload v2 product-limit recovery marker has no immutable UID')
+ }
+ return { uid: created.metadata.uid }
+}
+
+export function buildGfsUploadProductMarkerDeleteRequest(uid: string): {
+ args: string[]
+ input: string
+} {
+ if (!uid) throw new Error('refusing GFS product-limit marker deletion without a UID')
+ return {
+ args: [
+ '-n',
+ HCC_NAMESPACE,
+ 'delete',
+ '--raw',
+ `/api/v1/namespaces/${HCC_NAMESPACE}/configmaps/${GFS_PRODUCT_MUTATION_MARKER}`,
+ '-f',
+ '-',
+ '--wait=true',
+ ],
+ input: JSON.stringify({
+ apiVersion: 'v1',
+ kind: 'DeleteOptions',
+ preconditions: { uid },
+ }),
+ }
+}
+
+async function deleteProductMutationMarker(lease: GfsUploadProductMutationLease): Promise<void> {
+ const request = buildGfsUploadProductMarkerDeleteRequest(lease.uid)
+ await kubectlWithInput(request.args, request.input)
+}
+
+const productMutationAdapter: GfsUploadProductMutationAdapter = {
+ authorizeNewMutation: authorizeNewProductMutation,
+ authorizeRecovery: assertOwnedRecoveryContext,
+ readBaseline: readHccProductMaxInputs,
+ readMarker: readProductMutationMarker,
+ createMarker: createProductMutationMarker,
+ applyPair: applyHccProductMaxInputs,
+ verifyPair: async pair => {
+ const actual = await readHccProductMaxInputs()
+ if (!sameGfsUploadProductEnvPair(actual, pair)) {
+ throw new Error(
+ `HCC GFS product inputs do not match the recorded state: ${JSON.stringify(actual)}`
+ )
+ }
+ await waitForWriterProductMaxInputs(pair)
+ },
+ deleteMarker: deleteProductMutationMarker,
+ newHolder: randomUUID,
+ now: () => new Date().toISOString(),
+ waitBeforeRetry: async attempt => {
+ await new Promise(resolve => setTimeout(resolve, attempt * 1_000))
+ },
+}
+
+/**
+ * Starts a branch-owned mutation transaction. The durable immutable marker is
+ * created and read back before this function returns, so callers cannot change
+ * HCC policy without first recording both original HCC inputs.
+ */
+export async function beginGfsUploadProductMaxMutation(): Promise<GfsUploadProductMutationLease> {
+ return beginGfsUploadProductMutation(productMutationAdapter)
+}
+
+export async function setGfsUploadProductMaxBytes(
+ lease: GfsUploadProductMutationLease,
+ value: number
+): Promise<void> {
+ await applyGfsUploadProductMutation(productMutationAdapter, lease, productMaxPairForValue(value))
+}
+
+/** Restores the exact recorded HCC input pair without re-running new-mutation Git checks. */
+export async function restoreGfsUploadProductMaxBytes(
+ lease: GfsUploadProductMutationLease
+): Promise<void> {
+ await restoreGfsUploadProductMutation(productMutationAdapter, lease)
+}
+
+/**
+ * Recovers an interrupted transaction from its cluster marker. This path still
+ * verifies the branch context and cluster lease, but intentionally does not
+ * require the originating worktree or its cleanliness.
+ */
+export async function recoverGfsUploadProductMaxBytes(expected: {
+ uid: string
+ holder: string
+}): Promise<boolean> {
+ return recoverGfsUploadProductMutation(productMutationAdapter, expected)
+}
+
export async function readGfsUploadProductMaxBytes(): Promise<GfsUploadProductMaxState> {
const value = await renderedWriterProductMax()
if (value === undefined) return { kind: 'absent' }