Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
fix(e2e): r4-h3 make upload policy mutation recoverable
Josuéc777c4255184
diff --git a/tests/e2e/gfsUploadProductMutation.ts b/tests/e2e/gfsUploadProductMutation.tsnew file mode 100644index 000000000..87f580ec1--- /dev/null+++ b/tests/e2e/gfsUploadProductMutation.ts@@ -0,0 +1,219 @@+export type GfsUploadProductEnvState = { kind: 'absent' } | { kind: 'explicit'; value: number }++export interface GfsUploadProductEnvPair {+ canonical: GfsUploadProductEnvState+ alias: GfsUploadProductEnvState+}++export interface GfsUploadProductMutationOwnership {+ context: string+ branch: string+ worktreeId: string+ gitHead: string+ clusterFingerprint: string+ preGateMarkerUid: string+}++export interface GfsUploadProductMutationMarker {+ version: 1+ state: 'recovery-required'+ holder: string+ startedAt: string+ ownership: GfsUploadProductMutationOwnership+ baseline: GfsUploadProductEnvPair+}++export interface GfsUploadProductMutationLease {+ uid: string+ marker: GfsUploadProductMutationMarker+}++export interface GfsUploadProductMutationAdapter {+ authorizeNewMutation(): Promise<GfsUploadProductMutationOwnership>+ authorizeRecovery(marker: GfsUploadProductMutationMarker): Promise<void>+ readBaseline(): Promise<GfsUploadProductEnvPair>+ readMarker(): Promise<GfsUploadProductMutationLease | undefined>+ createMarker(marker: GfsUploadProductMutationMarker): Promise<{ uid: string }>+ applyPair(pair: GfsUploadProductEnvPair): Promise<void>+ verifyPair(pair: GfsUploadProductEnvPair): Promise<void>+ deleteMarker(lease: GfsUploadProductMutationLease): Promise<void>+ newHolder(): string+ now(): string+ waitBeforeRetry(attempt: number): Promise<void>+}++const RESTORE_ATTEMPTS = 3++function sameState(left: GfsUploadProductEnvState, right: GfsUploadProductEnvState): boolean {+ return (+ left.kind === right.kind &&+ (left.kind === 'absent' || (right.kind === 'explicit' && left.value === right.value))+ )+}++export function sameGfsUploadProductEnvPair(+ left: GfsUploadProductEnvPair,+ right: GfsUploadProductEnvPair+): boolean {+ return sameState(left.canonical, right.canonical) && sameState(left.alias, right.alias)+}++export function serializeGfsUploadProductMutationMarker(+ marker: GfsUploadProductMutationMarker+): string {+ return JSON.stringify(marker)+}++export function parseGfsUploadProductMutationMarker(raw: string): GfsUploadProductMutationMarker {+ let value: unknown+ try {+ value = JSON.parse(raw)+ } catch {+ throw new Error('invalid GFS Upload v2 product-limit recovery marker JSON')+ }+ if (!value || typeof value !== 'object') {+ throw new Error('invalid GFS Upload v2 product-limit recovery marker shape')+ }+ const marker = value as Partial<GfsUploadProductMutationMarker>+ const ownership = marker.ownership as Partial<GfsUploadProductMutationOwnership> | undefined+ if (+ marker.version !== 1 ||+ marker.state !== 'recovery-required' ||+ typeof marker.holder !== 'string' ||+ marker.holder.length === 0 ||+ typeof marker.startedAt !== 'string' ||+ marker.startedAt.length === 0 ||+ !ownership ||+ typeof ownership.context !== 'string' ||+ typeof ownership.branch !== 'string' ||+ typeof ownership.worktreeId !== 'string' ||+ typeof ownership.gitHead !== 'string' ||+ typeof ownership.clusterFingerprint !== 'string' ||+ typeof ownership.preGateMarkerUid !== 'string' ||+ !marker.baseline ||+ !isEnvState(marker.baseline.canonical) ||+ !isEnvState(marker.baseline.alias)+ ) {+ throw new Error('invalid GFS Upload v2 product-limit recovery marker shape')+ }+ return marker as GfsUploadProductMutationMarker+}++function isEnvState(value: unknown): value is GfsUploadProductEnvState {+ if (!value || typeof value !== 'object') return false+ const state = value as Partial<GfsUploadProductEnvState>+ return (+ state.kind === 'absent' ||+ (state.kind === 'explicit' && Number.isSafeInteger(state.value) && (state.value as number) >= 1)+ )+}++function assertSameLease(+ expected: GfsUploadProductMutationLease,+ actual: GfsUploadProductMutationLease | undefined+): void {+ if (+ !actual ||+ actual.uid !== expected.uid ||+ actual.marker.holder !== expected.marker.holder ||+ serializeGfsUploadProductMutationMarker(actual.marker) !==+ serializeGfsUploadProductMutationMarker(expected.marker)+ ) {+ throw new Error(+ 'GFS Upload v2 product-limit recovery marker changed; refusing mutation or cleanup'+ )+ }+}++export async function beginGfsUploadProductMutation(+ adapter: GfsUploadProductMutationAdapter+): Promise<GfsUploadProductMutationLease> {+ const ownership = await adapter.authorizeNewMutation()+ const existing = await adapter.readMarker()+ if (existing) {+ throw new Error(+ `GFS Upload v2 product-limit recovery is required for uid ${existing.uid}, ` ++ `holder ${existing.marker.holder}; explicitly confirm both lease values before restoring ` ++ 'the recorded baseline or starting another mutation'+ )+ }+ const baseline = await adapter.readBaseline()+ const marker: GfsUploadProductMutationMarker = {+ version: 1,+ state: 'recovery-required',+ holder: adapter.newHolder(),+ startedAt: adapter.now(),+ ownership,+ baseline,+ }+ const created = await adapter.createMarker(marker)+ if (!created.uid) {+ throw new Error('GFS Upload v2 product-limit recovery marker has no immutable UID')+ }+ const lease = { uid: created.uid, marker }+ await adapter.authorizeRecovery(marker)+ assertSameLease(lease, await adapter.readMarker())+ return lease+}++export async function applyGfsUploadProductMutation(+ adapter: GfsUploadProductMutationAdapter,+ lease: GfsUploadProductMutationLease,+ pair: GfsUploadProductEnvPair+): Promise<void> {+ await adapter.authorizeRecovery(lease.marker)+ assertSameLease(lease, await adapter.readMarker())+ await adapter.applyPair(pair)+ await adapter.verifyPair(pair)+ await adapter.authorizeRecovery(lease.marker)+ assertSameLease(lease, await adapter.readMarker())+}++async function restoreWithRetries(+ adapter: GfsUploadProductMutationAdapter,+ lease: GfsUploadProductMutationLease+): Promise<void> {+ let lastError: unknown+ for (let attempt = 1; attempt <= RESTORE_ATTEMPTS; attempt += 1) {+ try {+ await adapter.authorizeRecovery(lease.marker)+ assertSameLease(lease, await adapter.readMarker())+ await adapter.applyPair(lease.marker.baseline)+ await adapter.verifyPair(lease.marker.baseline)+ await adapter.authorizeRecovery(lease.marker)+ assertSameLease(lease, await adapter.readMarker())+ await adapter.deleteMarker(lease)+ return+ } catch (error) {+ lastError = error+ if (attempt < RESTORE_ATTEMPTS) await adapter.waitBeforeRetry(attempt)+ }+ }+ throw new Error(+ `GFS Upload v2 product-limit recovery remains required after ${RESTORE_ATTEMPTS} attempts; ` ++ `holder=${lease.marker.holder}; baseline=${JSON.stringify(lease.marker.baseline)}; ` ++ `cause=${lastError instanceof Error ? lastError.message : String(lastError)}`+ )+}++export async function restoreGfsUploadProductMutation(+ adapter: GfsUploadProductMutationAdapter,+ lease: GfsUploadProductMutationLease+): Promise<void> {+ await adapter.authorizeRecovery(lease.marker)+ await restoreWithRetries(adapter, lease)+}++export async function recoverGfsUploadProductMutation(+ adapter: GfsUploadProductMutationAdapter,+ expected: { uid: string; holder: string }+): Promise<boolean> {+ const lease = await adapter.readMarker()+ if (!lease) return false+ if (lease.uid !== expected.uid || lease.marker.holder !== expected.holder) {+ throw new Error('GFS Upload v2 product-limit recovery takeover evidence does not match')+ }+ await adapter.authorizeRecovery(lease.marker)+ await restoreWithRetries(adapter, lease)+ return true+}diff --git a/tests/e2e/gfsUploadV2Runtime.ts b/tests/e2e/gfsUploadV2Runtime.tsindex 819a28b3f..3a4c9404f 100644--- a/tests/e2e/gfsUploadV2Runtime.ts+++ b/tests/e2e/gfsUploadV2Runtime.ts@@ -1,10 +1,25 @@-import { execFile, execFileSync } from 'node:child_process'-import { createHash } from 'node:crypto'+import { execFile, execFileSync, spawn } from 'node:child_process'+import { createHash, randomUUID } from 'node:crypto'import { existsSync, readFileSync } from 'node:fs'import { homedir } from 'node:os'import { resolve } from 'node:path'import { promisify } from 'node:util'import { firstDataLine, kubectlContext, runControlPostgresSql, sqlLiteral } from './gfsFixtureCore'+import {+ type GfsUploadProductEnvPair,+ type GfsUploadProductEnvState,+ type GfsUploadProductMutationAdapter,+ type GfsUploadProductMutationLease,+ type GfsUploadProductMutationMarker,+ type GfsUploadProductMutationOwnership,+ applyGfsUploadProductMutation,+ beginGfsUploadProductMutation,+ parseGfsUploadProductMutationMarker,+ recoverGfsUploadProductMutation,+ restoreGfsUploadProductMutation,+ sameGfsUploadProductEnvPair,+ serializeGfsUploadProductMutationMarker,+} from './gfsUploadProductMutation'import { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'export { GFS_UPLOAD_V2_PROTOCOL_MAX_BYTES } from './gfsUploadV2Fixtures'@@ -20,6 +35,9 @@ const GFS_V2_WRITER_ENV = 'GFS_UPLOAD_V2_ENABLED'const GFS_PRODUCT_MAX_HCC_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_PRODUCT_MAX_FILE_BYTES'const GFS_PRODUCT_MAX_HCC_ALIAS_ENV = 'CONTEXT_MAPPER_GFSC_UPLOAD_MAX_FILE_BYTES'const GFS_PRODUCT_MAX_WRITER_ENV = 'GFS_UPLOAD_PRODUCT_MAX_FILE_BYTES'+const GFS_PRODUCT_MAX_WRITER_ALIAS_ENV = 'GFS_UPLOAD_MAX_FILE_BYTES'+const GFS_PRODUCT_MUTATION_MARKER = 'gfs-upload-product-limit-recovery'+const GFS_PRODUCT_MUTATION_MARKER_KEY = 'transaction.json'const BRANCH_CONTEXT_RE = /^clerum-[a-z0-9][a-z0-9-]*-[0-9a-f]{8}$/export interface GfsRuntimeImageMarker {@@ -36,7 +54,8 @@ export interface ExpectedGfsRuntimeImageMarker {imagesGeneratedAt?: string}-export type GfsUploadProductMaxState = { kind: 'absent' } | { kind: 'explicit'; value: number }+export type GfsUploadProductMaxState = GfsUploadProductEnvState+export type GfsUploadProductMaxInputs = GfsUploadProductEnvPair/*** Local Minikube image mode intentionally has no registry tag. GHCR mode must@@ -113,7 +132,21 @@ function normalizeUrl(value: string): string {* name alone is not ownership proof: another healthy Minikube profile could* otherwise receive an HCC mutation or writer restart.*/-async function assertOwnedRuntimeContext(): Promise<void> {+interface PreGateMarkerData {+ worktreeId?: string+ gitHead?: string+ clusterFingerprint?: string+ imageSource?: string+ imageTag?: string+ imagesGeneratedAt?: string+}++interface PreGateMarkerResource {+ metadata?: { uid?: string }+ data?: PreGateMarkerData+}++async function assertOwnedRuntimeContext(): Promise<GfsUploadProductMutationOwnership> {const context = kubectlContext()if (/prod|production/i.test(context)) {throw new Error(`refusing GFS E2E mutation on production-like context ${context}`)@@ -171,6 +204,13 @@ async function assertOwnedRuntimeContext(): Promise<void> {cwd: repoRoot,encoding: 'utf8',}).trim()+ const branch = execFileSync('git', ['branch', '--show-current'], {+ cwd: repoRoot,+ encoding: 'utf8',+ }).trim()+ if (!branch) {+ throw new Error('refusing GFS E2E mutation from a detached HEAD')+ }const marker = JSON.parse(await kubectl(['-n',@@ -180,16 +220,7 @@ async function assertOwnedRuntimeContext(): Promise<void> {'-o','json',])- ) as {- data?: {- worktreeId?: string- gitHead?: string- clusterFingerprint?: string- imageSource?: string- imageTag?: string- imagesGeneratedAt?: string- }- }+ ) as PreGateMarkerResourceif (marker.data?.worktreeId !== expectedWorktreeId) {throw new Error('refusing GFS E2E mutation: cluster ownership marker does not match this worktree; run pre-gate-sync first'@@ -212,6 +243,71 @@ async function assertOwnedRuntimeContext(): Promise<void> {...(expectedClusterFingerprint ? { clusterFingerprint: expectedClusterFingerprint } : {}),...(expectedImagesGeneratedAt ? { imagesGeneratedAt: expectedImagesGeneratedAt } : {}),})+ if (!marker.metadata?.uid) {+ throw new Error('refusing GFS E2E mutation: pre-gate marker has no immutable UID')+ }+ return {+ context,+ branch,+ worktreeId: expectedWorktreeId,+ gitHead: expectedHead,+ clusterFingerprint: markerData?.clusterFingerprint ?? '',+ preGateMarkerUid: marker.metadata.uid,+ }+}++async function assertOwnedRecoveryContext(marker: GfsUploadProductMutationMarker): Promise<void> {+ const context = kubectlContext()+ if (/prod|production/i.test(context) || !BRANCH_CONTEXT_RE.test(context)) {+ throw new Error(`refusing GFS E2E recovery on non-branch or production-like context ${context}`)+ }+ if (context !== marker.ownership.context) {+ throw new Error(+ `refusing GFS E2E recovery: marker context ${marker.ownership.context} does not match ${context}`+ )+ }+ const repoRoot = execFileSync('git', ['rev-parse', '--show-toplevel'], {+ cwd: process.cwd(),+ encoding: 'utf8',+ }).trim()+ const branch = execFileSync('git', ['branch', '--show-current'], {+ cwd: repoRoot,+ encoding: 'utf8',+ }).trim()+ if (!branch || branch !== marker.ownership.branch) {+ throw new Error(+ `refusing GFS E2E recovery: marker branch ${marker.ownership.branch} does not match ` ++ `${branch || '<detached>'}`+ )+ }+ const preGate = JSON.parse(+ await kubectl([+ '-n',+ HCC_NAMESPACE,+ 'get',+ 'configmap/clerum-pre-gate-sync-state',+ '-o',+ 'json',+ ])+ ) as PreGateMarkerResource+ if (+ preGate.metadata?.uid !== marker.ownership.preGateMarkerUid ||+ preGate.data?.clusterFingerprint !== marker.ownership.clusterFingerprint+ ) {+ throw new Error(+ 'refusing GFS E2E recovery: cluster ownership evidence no longer matches the recorded lease'+ )+ }+}++async function authorizeNewProductMutation(): Promise<GfsUploadProductMutationOwnership> {+ const ownership = await assertOwnedRuntimeContext()+ if (!BRANCH_CONTEXT_RE.test(ownership.context)) {+ throw new Error(+ `refusing GFS product-limit mutation on non-branch-owned context ${ownership.context}`+ )+ }+ return ownership}async function kubectl(args: string[], timeout = 240_000): Promise<string> {@@ -223,6 +319,35 @@ async function kubectl(args: string[], timeout = 240_000): Promise<string> {return result.stdout}+async function kubectlWithInput(args: string[], input: string, timeout = 240_000): Promise<string> {+ return new Promise((resolvePromise, reject) => {+ const child = spawn('kubectl', ['--context', kubectlContext(), ...args], {+ stdio: ['pipe', 'pipe', 'pipe'],+ })+ let stdout = ''+ let stderr = ''+ const timer = setTimeout(() => child.kill('SIGTERM'), timeout)+ child.stdout.setEncoding('utf8')+ child.stderr.setEncoding('utf8')+ child.stdout.on('data', chunk => {+ stdout += chunk+ })+ child.stderr.on('data', chunk => {+ stderr += chunk+ })+ child.on('error', error => {+ clearTimeout(timer)+ reject(error)+ })+ child.on('close', code => {+ clearTimeout(timer)+ if (code === 0) resolvePromise(stdout)+ else reject(new Error(`kubectl exited with ${code}: ${stderr.trim() || '<no stderr>'}`))+ })+ child.stdin.end(input)+ })+}+async function renderedWriterFlag(): Promise<string> {return (await kubectl([@@ -251,6 +376,104 @@ interface RenderedWriterDeployment {}}+function parseDeploymentEnvState(+ containers: Array<{ env?: RenderedWriterEnv[] }>,+ envName: string,+ owner: string+): GfsUploadProductMaxState {+ const matches = containers.flatMap(container => {+ if (container.env === undefined) return []+ if (!Array.isArray(container.env)) {+ throw new Error(`invalid ${envName} ${owner} deployment env shape`)+ }+ return container.env.filter(env => env?.name === envName)+ })+ if (matches.length === 0) return { kind: 'absent' }+ if (matches.length !== 1 || typeof matches[0]?.value !== 'string') {+ throw new Error(`invalid ${envName} ${owner} deployment value or duplicate authority`)+ }+ return {+ kind: 'explicit',+ value: validateGfsUploadProductMaxBytes(parseCanonicalProductMax(matches[0].value)),+ }+}++export function parseGfsUploadProductMaxInputs(raw: string): GfsUploadProductMaxInputs {+ let deployment: RenderedWriterDeployment+ try {+ deployment = JSON.parse(raw) as RenderedWriterDeployment+ } catch {+ throw new Error('invalid HCC product-limit deployment JSON')+ }+ const containers = deployment.spec?.template?.spec?.containers+ if (+ !Array.isArray(containers) ||+ containers.some(container => !container || typeof container !== 'object')+ ) {+ throw new Error('invalid HCC product-limit deployment shape')+ }+ const pair = {+ canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ENV, 'HCC'),+ alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_HCC_ALIAS_ENV, 'HCC'),+ }+ if (+ pair.canonical.kind === 'explicit' &&+ pair.alias.kind === 'explicit' &&+ pair.canonical.value !== pair.alias.value+ ) {+ throw new Error('conflicting canonical and deprecated HCC GFS product-limit values')+ }+ return pair+}++function parseRenderedWriterProductMaxInputs(raw: string): GfsUploadProductMaxInputs {+ let deployment: RenderedWriterDeployment+ try {+ deployment = JSON.parse(raw) as RenderedWriterDeployment+ } catch {+ throw new Error('invalid GFSC writer product-limit deployment JSON')+ }+ const containers = deployment.spec?.template?.spec?.containers+ if (!Array.isArray(containers)) {+ throw new Error('invalid GFSC writer product-limit deployment shape')+ }+ return {+ canonical: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ENV, 'GFSC writer'),+ alias: parseDeploymentEnvState(containers, GFS_PRODUCT_MAX_WRITER_ALIAS_ENV, 'GFSC writer'),+ }+}++export function buildGfsUploadProductMaxInputEnvArgs(pair: GfsUploadProductMaxInputs): string[] {+ const arg = (name: string, state: GfsUploadProductMaxState): string =>+ state.kind === 'absent'+ ? `${name}-`+ : `${name}=${validateGfsUploadProductMaxBytes(state.value)}`+ return [+ arg(GFS_PRODUCT_MAX_HCC_ENV, pair.canonical),+ arg(GFS_PRODUCT_MAX_HCC_ALIAS_ENV, pair.alias),+ ]+}++function productMaxPairForValue(value: number): GfsUploadProductMaxInputs {+ const validated = validateGfsUploadProductMaxBytes(value)+ return {+ canonical: { kind: 'explicit', value: validated },+ alias: { kind: 'explicit', value: validated },+ }+}++async function readHccProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {+ return parseGfsUploadProductMaxInputs(+ await kubectl(['-n', HCC_NAMESPACE, 'get', `deployment/${HCC_DEPLOYMENT}`, '-o', 'json'])+ )+}++async function readWriterProductMaxInputs(): Promise<GfsUploadProductMaxInputs> {+ return parseRenderedWriterProductMaxInputs(+ await kubectl(['-n', GFS_NAMESPACE, 'get', `deployment/${GFS_WRITER_DEPLOYMENT}`, '-o', 'json'])+ )+}+/*** Reads the complete deployment object so an absent env can be distinguished* from a present-but-malformed value. A malformed deployment or duplicate@@ -323,17 +546,31 @@ async function waitForRenderedWriterFlag(expected: boolean): Promise<void> {)}-async function waitForRenderedWriterProductMax(expected: number | undefined): Promise<void> {- const expectedValue = expected === undefined ? '<absent>' : String(expected)+async function waitForHccProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {const deadline = Date.now() + 240_000- let lastValue: string | undefined+ let lastValue: GfsUploadProductMaxInputs | undefinedwhile (Date.now() < deadline) {- lastValue = await renderedWriterProductMax()- if (lastValue === (expected === undefined ? undefined : expectedValue)) return+ lastValue = await readHccProductMaxInputs()+ if (sameGfsUploadProductEnvPair(lastValue, expected)) returnawait new Promise(resolve => setTimeout(resolve, 1_000))}throw new Error(- `GFS Upload v2 product maximum did not converge to ${expectedValue}; observed ${lastValue || '<absent>'}`+ `HCC GFS Upload v2 product inputs did not converge to ${JSON.stringify(expected)}; ` ++ `observed ${JSON.stringify(lastValue)}`+ )+}++async function waitForWriterProductMaxInputs(expected: GfsUploadProductMaxInputs): Promise<void> {+ const deadline = Date.now() + 240_000+ let lastValue: GfsUploadProductMaxInputs | undefined+ while (Date.now() < deadline) {+ lastValue = await readWriterProductMaxInputs()+ if (sameGfsUploadProductEnvPair(lastValue, expected)) return+ await new Promise(resolve => setTimeout(resolve, 1_000))+ }+ throw new Error(+ `GFSC writer product inputs did not converge to ${JSON.stringify(expected)}; ` ++ `observed ${JSON.stringify(lastValue)}`)}@@ -426,13 +663,8 @@ export function buildGfsUploadProductMaxEnvArgs(throw new Error('invalid GFS Upload v2 product maximum restoration state')}-export async function setGfsUploadProductMaxBytes(- state: GfsUploadProductMaxState | number-): Promise<void> {- const envArgs = buildGfsUploadProductMaxEnvArgs(state)- const expected =- typeof state === 'number' ? state : state.kind === 'explicit' ? state.value : undefined- await assertOwnedRuntimeContext()+async function applyHccProductMaxInputs(pair: GfsUploadProductMaxInputs): Promise<void> {+ const envArgs = buildGfsUploadProductMaxInputEnvArgs(pair)await kubectl(['-n', HCC_NAMESPACE, 'set', 'env', `deployment/${HCC_DEPLOYMENT}`, ...envArgs])await kubectl(['-n',@@ -442,11 +674,159 @@ export async function setGfsUploadProductMaxBytes(`deployment/${HCC_DEPLOYMENT}`,'--timeout=240s',])- await waitForRenderedWriterProductMax(expected)+ await waitForHccProductMaxInputs(pair)+ await waitForWriterProductMaxInputs(pair)await waitForDeployment(GFS_WRITER_DEPLOYMENT)await waitForDeployment(GFS_READER_DEPLOYMENT)}+interface ProductMutationMarkerResource {+ metadata?: { uid?: string }+ immutable?: boolean+ data?: Record<string, string | undefined>+}++async function readProductMutationMarker(): Promise<GfsUploadProductMutationLease | undefined> {+ const raw = (+ await kubectl([+ '-n',+ HCC_NAMESPACE,+ 'get',+ `configmap/${GFS_PRODUCT_MUTATION_MARKER}`,+ '--ignore-not-found',+ '-o',+ 'json',+ ])+ ).trim()+ if (!raw) return undefined+ let resource: ProductMutationMarkerResource+ try {+ resource = JSON.parse(raw) as ProductMutationMarkerResource+ } catch {+ throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap JSON')+ }+ const uid = resource.metadata?.uid+ const markerRaw = resource.data?.[GFS_PRODUCT_MUTATION_MARKER_KEY]+ if (!uid || resource.immutable !== true || !markerRaw) {+ throw new Error('invalid GFS Upload v2 product-limit recovery ConfigMap shape')+ }+ return { uid, marker: parseGfsUploadProductMutationMarker(markerRaw) }+}++async function createProductMutationMarker(+ marker: GfsUploadProductMutationMarker+): Promise<{ uid: string }> {+ const raw = await kubectlWithInput(+ ['-n', HCC_NAMESPACE, 'create', '-f', '-', '-o', 'json'],+ JSON.stringify({+ apiVersion: 'v1',+ kind: 'ConfigMap',+ metadata: {+ name: GFS_PRODUCT_MUTATION_MARKER,+ namespace: HCC_NAMESPACE,+ labels: { 'e2e.clerum.io/coordination': 'gfs-upload-product-limit' },+ },+ immutable: true,+ data: {+ [GFS_PRODUCT_MUTATION_MARKER_KEY]: serializeGfsUploadProductMutationMarker(marker),+ },+ })+ )+ const created = JSON.parse(raw) as ProductMutationMarkerResource+ if (!created.metadata?.uid || created.immutable !== true) {+ throw new Error('created GFS Upload v2 product-limit recovery marker has no immutable UID')+ }+ return { uid: created.metadata.uid }+}++export function buildGfsUploadProductMarkerDeleteRequest(uid: string): {+ args: string[]+ input: string+} {+ if (!uid) throw new Error('refusing GFS product-limit marker deletion without a UID')+ return {+ args: [+ '-n',+ HCC_NAMESPACE,+ 'delete',+ '--raw',+ `/api/v1/namespaces/${HCC_NAMESPACE}/configmaps/${GFS_PRODUCT_MUTATION_MARKER}`,+ '-f',+ '-',+ '--wait=true',+ ],+ input: JSON.stringify({+ apiVersion: 'v1',+ kind: 'DeleteOptions',+ preconditions: { uid },+ }),+ }+}++async function deleteProductMutationMarker(lease: GfsUploadProductMutationLease): Promise<void> {+ const request = buildGfsUploadProductMarkerDeleteRequest(lease.uid)+ await kubectlWithInput(request.args, request.input)+}++const productMutationAdapter: GfsUploadProductMutationAdapter = {+ authorizeNewMutation: authorizeNewProductMutation,+ authorizeRecovery: assertOwnedRecoveryContext,+ readBaseline: readHccProductMaxInputs,+ readMarker: readProductMutationMarker,+ createMarker: createProductMutationMarker,+ applyPair: applyHccProductMaxInputs,+ verifyPair: async pair => {+ const actual = await readHccProductMaxInputs()+ if (!sameGfsUploadProductEnvPair(actual, pair)) {+ throw new Error(+ `HCC GFS product inputs do not match the recorded state: ${JSON.stringify(actual)}`+ )+ }+ await waitForWriterProductMaxInputs(pair)+ },+ deleteMarker: deleteProductMutationMarker,+ newHolder: randomUUID,+ now: () => new Date().toISOString(),+ waitBeforeRetry: async attempt => {+ await new Promise(resolve => setTimeout(resolve, attempt * 1_000))+ },+}++/**+ * Starts a branch-owned mutation transaction. The durable immutable marker is+ * created and read back before this function returns, so callers cannot change+ * HCC policy without first recording both original HCC inputs.+ */+export async function beginGfsUploadProductMaxMutation(): Promise<GfsUploadProductMutationLease> {+ return beginGfsUploadProductMutation(productMutationAdapter)+}++export async function setGfsUploadProductMaxBytes(+ lease: GfsUploadProductMutationLease,+ value: number+): Promise<void> {+ await applyGfsUploadProductMutation(productMutationAdapter, lease, productMaxPairForValue(value))+}++/** Restores the exact recorded HCC input pair without re-running new-mutation Git checks. */+export async function restoreGfsUploadProductMaxBytes(+ lease: GfsUploadProductMutationLease+): Promise<void> {+ await restoreGfsUploadProductMutation(productMutationAdapter, lease)+}++/**+ * Recovers an interrupted transaction from its cluster marker. This path still+ * verifies the branch context and cluster lease, but intentionally does not+ * require the originating worktree or its cleanliness.+ */+export async function recoverGfsUploadProductMaxBytes(expected: {+ uid: string+ holder: string+}): Promise<boolean> {+ return recoverGfsUploadProductMutation(productMutationAdapter, expected)+}+export async function readGfsUploadProductMaxBytes(): Promise<GfsUploadProductMaxState> {const value = await renderedWriterProductMax()if (value === undefined) return { kind: 'absent' }