Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(llm): complete isolated fixture orchestration and cleanup

Alfredo Lopeza6d10b65afff
diff --git a/Makefile b/Makefile
index cd95dd7d0..4bb5a87c9 100644
--- a/Makefile
+++ b/Makefile
@@ -1576,6 +1576,24 @@ minikube-control-api-authorize-memory-body:
@T2_SKIP_LOCK=true node scripts/tests/measure-control-api-authorize-memory.mjs --config "$$CONTROL_API_MEMORY_CONFIG"
# Isolated subscription image journeys use the existing branch-profile mutation lease.
+.PHONY: minikube-subscription-image-coordinated
+minikube-subscription-image-coordinated:
+ @test -n "$${SUBSCRIPTION_IMAGE_MEMORY_CONFIG:-}" || { echo "SUBSCRIPTION_IMAGE_MEMORY_CONFIG required"; exit 1; }
+ @test -n "$${SUBSCRIPTION_IMAGE_RUN_ID:-}" || { echo "SUBSCRIPTION_IMAGE_RUN_ID required"; exit 1; }
+ @test -n "$${SUBSCRIPTION_IMAGE_SUITE:-}" || { echo "SUBSCRIPTION_IMAGE_SUITE required"; exit 1; }
+ @test -n "$${SUBSCRIPTION_IMAGE_SCRATCH:-}" || { echo "SUBSCRIPTION_IMAGE_SCRATCH required"; exit 1; }
+ @test -n "$${SUBSCRIPTION_IMAGE_RED_DETECTOR:-}" || { echo "SUBSCRIPTION_IMAGE_RED_DETECTOR required"; exit 1; }
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(CONTROL_API_REAL_PG_CONTEXT)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- \
+ node scripts/e2e/coordinate-subscription-image-journeys.mjs \
+ --memory-config "$$SUBSCRIPTION_IMAGE_MEMORY_CONFIG" \
+ --suite "$$SUBSCRIPTION_IMAGE_SUITE" --mode fixture \
+ --profile "$$MINIKUBE_PROFILE" --run-id "$$SUBSCRIPTION_IMAGE_RUN_ID" \
+ --scratch "$$SUBSCRIPTION_IMAGE_SCRATCH" --frames - \
+ --red-detector "$$SUBSCRIPTION_IMAGE_RED_DETECTOR" \
+ $${SUBSCRIPTION_IMAGE_SFW_ARTIFACT:+--sfw-artifact "$$SUBSCRIPTION_IMAGE_SFW_ARTIFACT"}
+
.PHONY: minikube-subscription-image-journeys
minikube-subscription-image-journeys:
@test -n "$${MINIKUBE_PROFILE:-}" || { echo "MINIKUBE_PROFILE required"; exit 1; }
diff --git a/control-api/package-lock.json b/control-api/package-lock.json
index 975a60a2e..d8f3c3191 100644
--- a/control-api/package-lock.json
+++ b/control-api/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "clerum-control-api",
- "version": "0.1.712",
+ "version": "0.1.713",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "clerum-control-api",
- "version": "0.1.712",
+ "version": "0.1.713",
"dependencies": {
"@clerum/display-field": "file:../packages/display-field",
"@clerum/grok-provider-attempt-contract": "file:../packages/grok-provider-attempt-contract",
diff --git a/control-api/package.json b/control-api/package.json
index 5efba1d2b..84dc8a398 100644
--- a/control-api/package.json
+++ b/control-api/package.json
@@ -1,6 +1,6 @@
{
"name": "clerum-control-api",
- "version": "0.1.712",
+ "version": "0.1.713",
"description": "Internal Clerum control plane REST API + UI for managing CRDs and secrets",
"main": "dist/main.js",
"engines": {
diff --git a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
index da447aadb..cccda197f 100644
--- a/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
+++ b/control-api/test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts
@@ -343,25 +343,61 @@ describe('createApp retained authorize-body ownership', () => {
const body = JSON.stringify({
request: { provider: `${provider}-subscription`, chunked: true },
})
+ const workStarted = deferred()
+ const permitUnwind = deferred()
+ vi.mocked(authorizer.authorizeLlmProviderAttempt).mockImplementationOnce(
+ async (_claims, _body, deps) => {
+ workStarted.resolve()
+ await permitUnwind.promise
+ deps?.signal?.throwIfAborted()
+ return SUCCESS
+ }
+ )
const sent = send(app.url, {
...headers(`chunked-${provider}-host`, provider),
+ 'x-admission-case': 'held',
'transfer-encoding': 'chunked',
})
+ const firstOutcome = sent.response.catch(error => error)
sent.client.write(body.slice(0, 20))
sent.client.end(body.slice(20))
- const response = await sent.response
- expect(response.status).toBe(200)
- expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledExactlyOnceWith(
- expect.objectContaining({ hostRefs: [`chunked-${provider}-host`] }),
- JSON.parse(body),
- expect.objectContaining({
- signal: expect.any(AbortSignal),
- resolveAssignment: expect.any(Function),
+ try {
+ await workStarted.promise
+ expect(app.observations[0]).toMatchObject({ bodyDataSubscriptions: 1 })
+
+ const refused = await send(
+ app.url,
+ headers(`chunked-refused-${provider}-host`, provider),
+ '{}'
+ ).response
+ expect(refused.status).toBe(503)
+ expect(JSON.parse(refused.body)).toEqual({ error: 'authorize_capacity_exceeded' })
+ expect(app.observations[1]).toMatchObject({
+ bodyDataSubscriptions: 0,
+ readBytesBeforeResponse: 0,
})
- )
- expect((app.observations[0].request as Request).body).toBeUndefined()
- const next = await send(app.url, headers(`next-${provider}-host`, provider), '{}').response
- expect(next.status).toBe(200)
+
+ permitUnwind.resolve()
+ const response = await firstOutcome
+ expect(response.status).toBe(200)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ hostRefs: [`chunked-${provider}-host`] }),
+ JSON.parse(body),
+ expect.objectContaining({
+ signal: expect.any(AbortSignal),
+ resolveAssignment: expect.any(Function),
+ })
+ )
+ expect((app.observations[0].request as Request).body).toBeUndefined()
+ const next = await send(app.url, headers(`next-${provider}-host`, provider), '{}')
+ .response
+ expect(next.status).toBe(200)
+ expect(authorizer.authorizeLlmProviderAttempt).toHaveBeenCalledTimes(2)
+ } finally {
+ sent.client.destroy()
+ permitUnwind.resolve()
+ await firstOutcome
+ }
})
})
}
diff --git a/desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts b/desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts
index 3b12c4f9a..9c41f3d80 100644
--- a/desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts
+++ b/desktop-app/test/e2e-playwright/playwright.subscription-image.config.ts
@@ -18,5 +18,7 @@ export default defineConfig({
reporter: [['list']],
outputDir: path.join(run.runRoot, 'playwright-results'),
metadata: { mode: run.mode, runId: run.runId, profile: run.profile },
- use: { screenshot: 'only-on-failure', trace: 'retain-on-failure', video: 'retain-on-failure' },
+ // Playwright action traces retain the plaintext value supplied at visible login.
+ // Keep private credentials in RAM; screenshots/video mask the password input.
+ use: { screenshot: 'only-on-failure', trace: 'off', video: 'retain-on-failure' },
})
diff --git a/desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts b/desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts
index f8ffc5240..abd79c305 100644
--- a/desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts
+++ b/desktop-app/test/e2e-playwright/subscription-admission-recovery.spec.ts
@@ -64,13 +64,15 @@ for (const binding of run.bindings) {
appPage,
}, testInfo) => {
const fixture = receipt.fixtures[binding.provider] as AdmissionFixture
+ // Keep the configured visible-journey budget and separately admit the nine
+ // bounded private commands (including finally release/close on failure).
+ test.setTimeout(testInfo.timeout + 9 * fixture.pressure.commandDeadlineMs)
await openOwnedChat(appPage, binding)
const refusalReceiptId = randomUUID()
const refusalPrompt = `Reply with TEXT_RECEIPT:${refusalReceiptId} only. Receipt: ${refusalReceiptId}`
await appPage.getByTestId('chat-input').fill(refusalPrompt)
const pressure = await openAdmissionPressure({
receiptFile: fixture.pressure.receiptFile,
- deadlineMs: 30_000,
})
let released = false
try {
@@ -81,6 +83,10 @@ for (const binding of run.bindings) {
expect(metadata.podUid).toBe(fixture.controlApiPodUid)
expect(metadata.imageId).toBe(fixture.controlApiImageId)
expect(metadata.maxInFlight).toBe(fixture.maxInFlight)
+ expect(metadata.commandDeadlineMs).toBe(fixture.pressure.commandDeadlineMs)
+ expect(metadata.readDeadlineMs).toBe(fixture.readDeadlineMs)
+ expect(metadata.workDeadlineMs).toBe(fixture.pressure.workDeadlineMs)
+ expect(metadata.closeGraceMs).toBe(fixture.pressure.closeGraceMs)
expect(new Set(metadata.hostRefs)).toEqual(new Set(run.bindings.map(item => item.hostRef)))
const baseline = (await pressure.owners()) as OwnerObservation
expect(baseline.pressureRunId).toBe(metadata.pressureRunId)
diff --git a/desktop-app/test/e2e-playwright/subscriptionRemainingJourneysContract.ts b/desktop-app/test/e2e-playwright/subscriptionRemainingJourneysContract.ts
index ff8d7df85..551ac0025 100644
--- a/desktop-app/test/e2e-playwright/subscriptionRemainingJourneysContract.ts
+++ b/desktop-app/test/e2e-playwright/subscriptionRemainingJourneysContract.ts
@@ -1,5 +1,6 @@
// E2E_GUARDIAN_IPC_FLOW: pure collection admission and fixture shape validation; no launch, network or session access.
import path from 'node:path'
+import { pressureCommandDeadlineMs } from '../../../scripts/e2e/fixtures/subscription-image-admission-pressure.mjs'
import type {
SubscriptionImageProvider,
SubscriptionImageRun,
@@ -45,7 +46,12 @@ export type AdmissionFixture = RuntimeIdentity & {
readDeadlineMs: number
// The private relay client validates its own sealed metadata and physical
// socket ownership. This layer never invents a URL, JWT or IPC command.
- pressure: { receiptFile: string }
+ pressure: {
+ receiptFile: string
+ workDeadlineMs: number
+ closeGraceMs: number
+ commandDeadlineMs: number
+ }
// Main must prove a real eligible fallback exists before testing no-failover.
fallback: { provider: SubscriptionImageProvider; modelId: string }
}
@@ -218,7 +224,6 @@ export function validateRemainingFixture(
fixture.fallback,
'Admission fixture requires an actual eligible fallback'
)
- const other = run.bindings.find(candidate => candidate.provider !== binding.provider)
const pressure = record(
fixture.pressure,
'Admission pressure requires sealed private relay metadata'
@@ -247,8 +252,17 @@ export function validateRemainingFixture(
!Number.isSafeInteger(fixture.readDeadlineMs) ||
Number(fixture.readDeadlineMs) < 1 ||
Number(fixture.readDeadlineMs) > 2_147_483_647 ||
- fallback.provider !== other?.provider ||
- fallback.modelId !== other?.modelId
+ pressure.commandDeadlineMs !==
+ pressureCommandDeadlineMs({
+ readDeadlineMs: fixture.readDeadlineMs as number,
+ workDeadlineMs: pressure.workDeadlineMs as number,
+ closeGraceMs: pressure.closeGraceMs as number,
+ }) ||
+ fallback.provider !== binding.provider ||
+ typeof fallback.modelId !== 'string' ||
+ !fallback.modelId ||
+ fallback.modelId === binding.modelId ||
+ fallback.modelId === binding.unsupportedModelId
) {
throw new Error(
'Admission fixture must bind real owned capacity and an eligible physical fallback'
diff --git a/desktop-app/test/subscriptionRemainingJourneysContract.test.ts b/desktop-app/test/subscriptionRemainingJourneysContract.test.ts
index 27e77370e..928103c93 100644
--- a/desktop-app/test/subscriptionRemainingJourneysContract.test.ts
+++ b/desktop-app/test/subscriptionRemainingJourneysContract.test.ts
@@ -108,10 +108,15 @@ function admissionReceipt() {
readDeadlineMs: 10000,
// This suite tests only outer metadata admission. The physical pressure
// client must reject this unit-only record; it is never used to hold work.
- pressure: { receiptFile: '/runner-admission/unit-pressure-metadata.json' },
+ pressure: {
+ receiptFile: '/runner-admission/unit-pressure-metadata.json',
+ workDeadlineMs: 30000,
+ closeGraceMs: 250,
+ commandDeadlineMs: 40250,
+ },
fallback: {
- provider: run.bindings.find(item => item.provider !== binding.provider)!.provider,
- modelId: run.bindings.find(item => item.provider !== binding.provider)!.modelId,
+ provider: binding.provider,
+ modelId: `${binding.provider}-eligible-alternate`,
},
},
])
@@ -273,6 +278,22 @@ describe('remaining subscription journeys collection and data-preparation gates'
validateRemainingFixture(value, { ...input, suite: 'admission-recovery' }, run, digest)
).toThrow('eligible physical fallback')
})
+ it('admits a sealed command deadline derived from its actual policy', () => {
+ const value = admissionReceipt()
+ expect(
+ validateRemainingFixture(value, { ...input, suite: 'admission-recovery' }, run, digest)
+ ).toBe(value)
+ })
+ it.each([30000, 40249, 40251])(
+ 'refuses a command deadline that does not match policy: %s',
+ deadline => {
+ const value = admissionReceipt()
+ value.fixtures['grok-subscription']!.pressure.commandDeadlineMs = deadline
+ expect(() =>
+ validateRemainingFixture(value, { ...input, suite: 'admission-recovery' }, run, digest)
+ ).toThrow('owned capacity')
+ }
+ )
it('rejects pressure metadata with no actual control-api pod identity', () => {
const value = admissionReceipt()
value.fixtures['grok-subscription']!.controlApiPodUid = 'unknown'
diff --git a/scripts/e2e/build-subscription-image-proxy.sh b/scripts/e2e/build-subscription-image-proxy.sh
new file mode 100755
index 000000000..1f535d0be
--- /dev/null
+++ b/scripts/e2e/build-subscription-image-proxy.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# E2E_GUARDIAN_IPC_FLOW: physical build prerequisite only; no browser journey.
+# Test-only derived images. The documented Make coordinator owns the same
+# mutation lease throughout build, activation, observation and restoration.
+set -euo pipefail
+umask 077
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
+PROJECT_DIR="$(cd -- "$SCRIPT_DIR/../.." && pwd -P)"
+[[ $# == 7 ]] || { printf 'DERIVED_PROXY_BUILD_ARGUMENTS\n' >&2; exit 2; }
+PROFILE="$1"; CONTEXT_DIR="$2"; PROXY_REF="$3"; PROXY_ID="$4"; HOST_ID="$5"; FIXTURE_REF="$6"; HEAD="$7"
+[[ "$PROFILE" =~ ^[a-z0-9][a-z0-9-]{0,62}$ && "$PROFILE" != clerum-test &&
+ "$PROFILE" == "${MINIKUBE_PROFILE:-}" && "$PROFILE" == "${CONTROL_API_REAL_PG_CONTEXT:-}" &&
+ "$PROXY_ID" =~ ^sha256:[a-f0-9]{64}$ && "$HOST_ID" =~ ^sha256:[a-f0-9]{64}$ &&
+ "$HEAD" =~ ^[a-f0-9]{40}$ && "$CONTEXT_DIR" == /* &&
+ "$PROXY_REF" =~ ^[a-zA-Z0-9][a-zA-Z0-9._/:-]*$ &&
+ "$FIXTURE_REF" =~ ^clerum/(grok|codex)-llm-proxy-image-qa:[a-f0-9]{12}$ ]] || {
+ printf 'DERIVED_PROXY_BUILD_IDENTITY\n' >&2; exit 2;
+}
+T2_PROJECT_DIR="$PROJECT_DIR" T2_PROFILE="$PROFILE" T2_CONTEXT="$PROFILE" \
+ bash "$PROJECT_DIR/scripts/minikube/require-t2-mutation-lock.sh"
+source "$PROJECT_DIR/scripts/minikube/docker-cli-env.sh"
+trap docker_cli_env_cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+docker_cli_env_prepare
+# The native helper pins an explicit local endpoint before creating its empty
+# task Docker config. The profile endpoint must independently pass that gate.
+PROFILE_DOCKER_ENV="$(node "$DOCKER_CLI_DEADLINE_RUNNER" --timeout-seconds 30 --heartbeat-seconds 20 \
+ --kill-grace-seconds 5 --label qa-minikube-docker-env -- minikube --profile "$PROFILE" docker-env --shell bash)"
+[[ -n "$PROFILE_DOCKER_ENV" ]] || { printf 'DERIVED_PROXY_DOCKER_ENV_EMPTY\n' >&2; exit 1; }
+PARSED_DOCKER_ENV="$(printf '%s' "$PROFILE_DOCKER_ENV" | node --input-type=module -e '
+let raw="";for await(const chunk of process.stdin)raw+=chunk;
+const allowed=new Set(["DOCKER_HOST","DOCKER_TLS_VERIFY","DOCKER_CERT_PATH","MINIKUBE_ACTIVE_DOCKERD"]),seen=new Set();
+for(const line of raw.split("\n")){if(!line.trim()||line.startsWith("#"))continue;
+ const match=/^export ([A-Z_]+)="([^"\\$`\x00-\x1f]*)"$/.exec(line);
+ if(!match||!allowed.has(match[1])||seen.has(match[1]))throw Error("DERIVED_PROXY_DOCKER_ENV_SYNTAX");
+ seen.add(match[1]);process.stdout.write(match[1]+"\t"+match[2]+"\n");
+}if(!seen.has("DOCKER_HOST")||!seen.has("MINIKUBE_ACTIVE_DOCKERD"))throw Error("DERIVED_PROXY_DOCKER_ENV_INCOMPLETE");')"
+while IFS=$'\t' read -r name value; do export "$name=$value"; done <<< "$PARSED_DOCKER_ENV"
+[[ "$MINIKUBE_ACTIVE_DOCKERD" == "$PROFILE" ]] || { printf 'DERIVED_PROXY_DAEMON_OWNER\n' >&2; exit 1; }
+docker_cli_env_validate_local_endpoint "$DOCKER_HOST"
+unset DOCKER_API_VERSION
+live_proxy="$(docker_cli_run_public qa-proxy-base-id 30 docker image inspect --format '{{.Id}}' "$PROXY_REF")"
+live_host="$(docker_cli_run_public qa-host-base-id 30 docker image inspect --format '{{.Id}}' clerum/mcp-host:test)"
+[[ "$live_proxy" == "$PROXY_ID" && "$live_host" == "$HOST_ID" ]] || { printf 'DERIVED_PROXY_BASE_IMAGE_CHANGED\n' >&2; exit 1; }
+suffix="${FIXTURE_REF##*:}"
+proxy_base="${FIXTURE_REF%:*}-base:$suffix"
+host_base="clerum/subscription-image-host-base:$suffix"
+docker_cli_run_public qa-proxy-base-pin 30 docker image tag "$PROXY_ID" "$proxy_base"
+docker_cli_run_public qa-host-base-pin 30 docker image tag "$HOST_ID" "$host_base"
+docker_cli_run_public qa-proxy-build 300 docker build --pull=false \
+ -f "$CONTEXT_DIR/scripts/e2e/fixtures/subscription-image-proxy.Dockerfile" -t "$FIXTURE_REF" \
+ --build-arg "PROXY_BASE_IMAGE=$proxy_base" --build-arg "HOST_BASE_IMAGE=$host_base" \
+ --build-arg "SOURCE_HEAD=$HEAD" "$CONTEXT_DIR" >&2
+after_proxy="$(docker_cli_run_public qa-proxy-base-after 30 docker image inspect --format '{{.Id}}' "$proxy_base")"
+after_host="$(docker_cli_run_public qa-host-base-after 30 docker image inspect --format '{{.Id}}' "$host_base")"
+[[ "$after_proxy" == "$PROXY_ID" && "$after_host" == "$HOST_ID" ]] || { printf 'DERIVED_PROXY_BASE_IMAGE_CHANGED\n' >&2; exit 1; }
+derived="$(docker_cli_run_public qa-derived-image-id 30 docker image inspect --format '{{.Id}}' "$FIXTURE_REF")"
+[[ "$derived" =~ ^sha256:[a-f0-9]{64}$ ]] || { printf 'DERIVED_PROXY_IMAGE_ID_UNKNOWN\n' >&2; exit 1; }
+printf '{"imageId":"%s","baseImageId":"%s","hostImageId":"%s","gitHead":"%s"}\n' "$derived" "$PROXY_ID" "$HOST_ID" "$HEAD"
diff --git a/scripts/e2e/coordinate-subscription-image-journeys.mjs b/scripts/e2e/coordinate-subscription-image-journeys.mjs
new file mode 100755
index 000000000..af988a087
--- /dev/null
+++ b/scripts/e2e/coordinate-subscription-image-journeys.mjs
@@ -0,0 +1,691 @@
+#!/usr/bin/env node
+// E2E_GUARDIAN_IPC_FLOW: leased QA preparation and external vendor observation.
+// Desktop login, selection, attachment, approval and Send remain visible.
+// Operator material is retained in RAM and crosses child stdin only.
+import fs from 'node:fs'
+import path from 'node:path'
+import { spawn } from 'node:child_process'
+import { Readable } from 'node:stream'
+import { createRequire } from 'node:module'
+import { fileURLToPath, pathToFileURL } from 'node:url'
+import { isDeepStrictEqual } from 'node:util'
+import { prepare, run as command } from './prepare-subscription-image-admission.mjs'
+import { run as seed, parseOptions, Channel, buildAuthorizeMemoryCompanionBundle,
+ openAuthorizeAdmissionPressure } from '../tests/measure-control-api-authorize-memory.mjs'
+import { prepareRemainingFixtures } from './prepare-subscription-remaining-fixtures.prepare.mjs'
+import { pressureCommandDeadlineMs } from './fixtures/subscription-image-admission-pressure.mjs'
+import { admitVendorFrame, digest, RUNNER_RUN_BASE, resolveSuite } from '../tests/lib/subscription-image-runner-contract.mjs'
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
+const require = createRequire(import.meta.url)
+const projection = require('../../packages/codex-catalog-projection/index.cjs')
+const imageCapability = require('../../packages/llm-providers/imageInput.cjs')
+const NAMESPACE = 'control-plane'
+const IMPORT_PATH = '/app/scripts/e2e/fixtures/subscription-image-provider.mjs'
+const PROXY_DOCKERFILE = 'scripts/e2e/fixtures/subscription-image-proxy.Dockerfile'
+const PROVIDERS = ['grok-subscription', 'codex-subscription']
+export const CATALOG_CRON_KEYS = ['LLM_CATALOG_SYNC_CRON_ENABLED', 'SUBSCRIPTION_CATALOG_SYNC_CRON_ENABLED']
+const fail = code => { throw new Error(code) }
+const assert = (value, code) => { if (!value) fail(code) }
+const validImageId = value => /^sha256:[a-f0-9]{64}$/.test(value ?? '')
+const imageId = value => /sha256:[a-f0-9]{64}$/.exec(value ?? '')?.[0]
+const pause = ms => new Promise(resolve => setTimeout(resolve, ms))
+const json = (context, args, input) => JSON.parse(command('kubectl', ['--context', context,
+ '--request-timeout=15s', ...args], { input, timeout: 30_000 }))
+const getDeployment = (context, name) => json(context, ['-n', NAMESPACE, 'get', 'deployment', name, '-o', 'json'])
+const patchDeployment = (context, name, patch) => command('kubectl', ['--context', context,
+ '--request-timeout=15s', '-n', NAMESPACE, 'patch', 'deployment', name, '--type=json',
+ '--patch-file=/dev/stdin'], { input: JSON.stringify(patch), timeout: 30_000 })
+
+async function until(check, accept, timeoutMs, code) {
+ const deadline = Date.now() + timeoutMs
+ for (;;) {
+ const value = await check()
+ if (accept(value)) return value
+ if (Date.now() >= deadline) fail(code)
+ await pause(100)
+ }
+}
+
+/** Snapshot only the controlled fields; retain no unrelated environment values. */
+export function snapshotDeployment(deployment, containerName, names) {
+ const containers = deployment?.spec?.template?.spec?.containers
+ assert(deployment?.spec?.replicas === 1 && !deployment.spec.template.spec.hostNetwork &&
+ typeof deployment.metadata?.uid === 'string' && Array.isArray(containers), 'COORDINATOR_DEPLOYMENT_INVALID')
+ const matches = containers.filter(value => value.name === containerName)
+ assert(matches.length === 1, 'COORDINATOR_CONTAINER_AMBIGUOUS')
+ const container = matches[0], env = {}
+ for (const name of names) {
+ const values = (container.env ?? []).filter(value => value.name === name)
+ assert(values.length <= 1 && values.every(value => !value.valueFrom && typeof value.value === 'string'),
+ 'COORDINATOR_CONTROLLED_ENV_INVALID')
+ if (values.length) env[name] = values[0].value
+ }
+ if (env.NODE_OPTIONS !== undefined) assert(/^--max-old-space-size=\d+$/.test(env.NODE_OPTIONS),
+ 'COORDINATOR_ORIGINAL_NODE_OPTIONS_UNSUPPORTED')
+ assert(names.filter(name => !['NODE_OPTIONS', 'NODE_ENV', ...CATALOG_CRON_KEYS].includes(name)).every(name => !(name in env)),
+ 'COORDINATOR_UNRESTORED_FIXTURE')
+ assert(CATALOG_CRON_KEYS.every(name => env[name] === undefined || ['true', 'false'].includes(env[name])),
+ 'COORDINATOR_CATALOG_CRON_VALUE_UNSUPPORTED')
+ return { uid: deployment.metadata.uid, containerName, image: container.image, env, names: [...names] }
+}
+
+/** UID/RV/image/field CAS in both directions, preserving unrelated fields. */
+export function deploymentMutationPatch(current, original, applied, restore = false) {
+ assert(current?.metadata?.uid === original.uid && typeof current.metadata.resourceVersion === 'string',
+ 'COORDINATOR_DEPLOYMENT_OWNER_CHANGED')
+ const containers = current.spec.template.spec.containers
+ const index = containers.findIndex(value => value.name === original.containerName)
+ assert(index >= 0, 'COORDINATOR_CONTAINER_CHANGED')
+ const container = containers[index], base = `/spec/template/spec/containers/${index}`
+ const expected = restore ? applied : original, desired = restore ? original : applied
+ assert(container.image === expected.image, 'COORDINATOR_IMAGE_RESTORE_CONFLICT')
+ for (const name of original.names) {
+ const values = (container.env ?? []).filter(value => value.name === name)
+ assert(values.length <= 1 && isDeepStrictEqual(values[0], expected.env[name] === undefined ? undefined :
+ { name, value: expected.env[name] }), 'COORDINATOR_ENV_RESTORE_CONFLICT')
+ }
+ const patch = [{ op: 'test', path: '/metadata/uid', value: original.uid },
+ { op: 'test', path: '/metadata/resourceVersion', value: current.metadata.resourceVersion },
+ { op: 'test', path: `${base}/name`, value: original.containerName },
+ { op: 'test', path: `${base}/image`, value: expected.image },
+ { op: 'replace', path: `${base}/image`, value: desired.image }]
+ if (!container.env) patch.push({ op: 'add', path: `${base}/env`, value: [] })
+ const removals = []
+ for (const name of original.names) {
+ const ei = (container.env ?? []).findIndex(value => value.name === name)
+ if (ei >= 0) patch.push({ op: 'test', path: `${base}/env/${ei}`, value: container.env[ei] })
+ if (desired.env[name] === undefined) {
+ if (ei >= 0) removals.push({ op: 'remove', path: `${base}/env/${ei}` })
+ } else patch.push({ op: ei >= 0 ? 'replace' : 'add', path: ei >= 0 ? `${base}/env/${ei}` : `${base}/env/-`,
+ value: { name, value: desired.env[name] } })
+ }
+ removals.sort((a, b) => Number(b.path.split('/').at(-1)) - Number(a.path.split('/').at(-1)))
+ return [...patch, ...removals]
+}
+
+function readyPod(context, name, deploymentUid) {
+ const replicas = json(context, ['-n', NAMESPACE, 'get', 'replicaset', '-o', 'json']).items
+ .filter(row => row.metadata.ownerReferences?.some(owner => owner.uid === deploymentUid && owner.controller === true))
+ const ids = new Set(replicas.map(row => row.metadata.uid))
+ const pods = json(context, ['-n', NAMESPACE, 'get', 'pods', '-o', 'json']).items.filter(row =>
+ !row.metadata.deletionTimestamp && row.status.phase === 'Running' &&
+ row.metadata.ownerReferences?.some(owner => ids.has(owner.uid) && owner.controller === true) &&
+ row.status.containerStatuses?.some(container => container.name === name && container.ready))
+ return pods.length === 1 ? pods[0] : undefined
+}
+
+export async function openControlApiCatalogIsolation(options) {
+ assert(options.profile === options.context && options.profile !== 'clerum-test' &&
+ !/(^|-)(prod|production)(-|$)/i.test(options.profile), 'COORDINATOR_CATALOG_CONTEXT_INVALID')
+ command('bash', [path.join(ROOT, 'scripts/minikube/require-t2-mutation-lock.sh')], { timeout: 10_000 })
+ const resource = getDeployment(options.context, 'control-api')
+ const original = snapshotDeployment(resource, 'control-api', CATALOG_CRON_KEYS)
+ const pod = await until(() => readyPod(options.context, 'control-api', original.uid), Boolean, 120_000,
+ 'COORDINATOR_CATALOG_API_NOT_READY')
+ const baseImageId = imageId(pod.status.containerStatuses.find(value => value.name === 'control-api').imageID)
+ assert(validImageId(baseImageId), 'COORDINATOR_CATALOG_API_IMAGE_UNKNOWN')
+ const applied = { image: original.image, env: Object.fromEntries(CATALOG_CRON_KEYS.map(name => [name, 'false'])) }
+ const state = { name: 'control-api', original, applied, baseImageId, changed: true }
+ try {
+ patchDeployment(options.context, state.name, deploymentMutationPatch(resource, original, applied))
+ command('kubectl', ['--context', options.context, '-n', NAMESPACE, 'rollout', 'status', 'deployment/control-api', '--timeout=120s'],
+ { timeout: 150_000 })
+ const ready = await until(() => readyPod(options.context, 'control-api', original.uid), Boolean, 120_000,
+ 'COORDINATOR_CATALOG_API_NOT_READY')
+ const flags = json(options.context, ['-n', NAMESPACE, 'exec', ready.metadata.name, '-c', 'control-api', '--',
+ 'env', '-u', 'NODE_OPTIONS', 'node', '-e',
+ "const c=require('./dist/config.js').config;process.stdout.write(JSON.stringify({llm:c.llmCatalogSyncCronEnabled,subscription:c.subscriptionCatalogSyncCronEnabled}))"])
+ assert(flags.llm === false && flags.subscription === false, 'COORDINATOR_CATALOG_CRONS_STILL_ENABLED')
+ return state
+ } catch (error) {
+ // No QA mutation has started while opening this fence, so restore its exact
+ // original fields even when rollout/observation fails.
+ await restoreDeployments(options.context, [state])
+ throw error
+ }
+}
+
+function releaseOptionsForQa(options, receiptFile) {
+ const fields = { ...options.publicArguments, 'fixtures-receipt': receiptFile,
+ report: path.join(path.dirname(options.report), `${options.runId}-qa-finalization.json`), prepare: 'restart' }
+ for (const key of ['prepare-fixtures', 'budget-ids', 'gfs-parent-rid', 'operator-user']) delete fields[key]
+ return parseOptions(Object.entries(fields).flatMap(([key, value]) => value === true ? [`--${key}`] : [`--${key}`, String(value)]))
+}
+
+/** Physical callers keep this one real CAS fence across all cgroup windows and
+ * image suites. Its methods use the existing private driver/runner; no raw
+ * credential enters options, receipts or the recovery record.
+ */
+export async function withControlApiFixtureRuntime(options, privateMaterial, work) {
+ const fence = await openControlApiCatalogIsolation(options)
+ const images = []
+ let memoryState = options.fixtureBinding ? 'created' : 'not-created'
+ let receiptFile = options.publicArguments?.['fixtures-receipt'], referenceOptions = options
+ let result, failure
+ const check = value => assert(value.profile === options.profile && value.context === options.context && value.runId === options.runId && value.prepare === 'restart',
+ 'COORDINATOR_CATALOG_SCOPE_CHANGED')
+ const runtime = { initialSnapshot: fence.original,
+ async run(value) {
+ check(value); referenceOptions = value
+ if (value.prepareFixtures) memoryState = 'created-or-unknown'
+ else if (value.fixtureBinding) { memoryState = 'created'; receiptFile = value.publicArguments['fixtures-receipt'] }
+ const report = await seed(value, privateMaterial)
+ if (value.prepareFixtures) { memoryState = 'created'; receiptFile = value.report }
+ return report
+ },
+ async journey(argumentsForRunner, value) {
+ check(value)
+ if (value.prepareFixtures) memoryState = 'created-or-unknown'
+ if (value.fixtureBinding) { memoryState = 'created'; receiptFile = value.publicArguments['fixtures-receipt']; referenceOptions = value }
+ const grantStatus = { state: 'not-created', bindings: [] }; images.push(grantStatus)
+ const report = await coordinate(argumentsForRunner, value, privateMaterial, { catalogFence: fence, grantStatus })
+ if (value.prepareFixtures) { memoryState = 'created'; receiptFile = value.report; referenceOptions = value }
+ return report
+ },
+ }
+ try { result = await work(runtime) } catch (error) { failure = error }
+ finally {
+ if (memoryState === 'created' && receiptFile) {
+ try {
+ const released = await seed(releaseOptionsForQa(referenceOptions, receiptFile), privateMaterial, async session => session.revokeMemory())
+ assert(released.kind === 'control-api-authorize-qa-finalization.v1' && released.fixtureFinalization?.verified === true &&
+ released.fixtureFinalization.state === 'revoked' && released.fixtureFinalization.grantsRevoked === 2 &&
+ released.fixtureFinalization.hostsDetached === 2, 'COORDINATOR_MEMORY_QA_FINALIZATION_UNPROVED')
+ memoryState = 'revoked'
+ } catch (error) { failure ??= error }
+ }
+ const restoreSafe = catalogRestorationSafe(memoryState, images)
+ if (restoreSafe) {
+ try { await restoreDeployments(options.context, [fence]) } catch (error) { failure ??= error }
+ } else {
+ // Keep both actual cron overrides false: connected opaque QA can be
+ // touched by Control API's direct first cron tick, outside vendor hooks.
+ try {
+ writeCatalogRecovery(path.join(path.dirname(options.report), `${options.runId}-catalog-recovery.json`),
+ fence, options, memoryState, images)
+ } catch (error) { failure ??= error }
+ failure ??= new Error('COORDINATOR_CATALOG_RESTORE_BLOCKED_BY_QA')
+ }
+ }
+ if (failure) throw new Error(/^[A-Z0-9_]+$/.test(failure.message) ? failure.message : 'COORDINATOR_CATALOG_SCOPE_FAILED')
+ return result
+}
+
+export function catalogRestorationSafe(memoryState, images) {
+ return ['not-created', 'revoked'].includes(memoryState) &&
+ images.every(value => ['not-created', 'revoked'].includes(value.state))
+}
+
+function writeCatalogRecovery(filename, fence, options, memoryState, images) {
+ const directory = path.dirname(filename), stat = fs.lstatSync(directory)
+ assert(stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === process.getuid?.() && !(stat.mode & 0o077) &&
+ fs.realpathSync(directory) === directory, 'COORDINATOR_RECOVERY_DIRECTORY_UNSAFE')
+ fs.writeFileSync(filename, JSON.stringify({ kind: 'evenfire-control-api-qa-catalog-isolation-v1',
+ profile: options.profile, context: options.context, runId: options.runId,
+ deploymentUid: fence.original.uid, imageId: fence.baseImageId, initialControlledEnv: fence.original.env,
+ retainedControlledEnv: fence.applied.env, memoryQaState: memoryState,
+ imageQa: images.map(value => ({ state: value.state, bindings: value.bindings })),
+ reason: 'CATALOG_CRONS_REMAIN_FALSE_UNTIL_ALL_OWNED_QA_REVOCATIONS_VERIFIED', recordedAt: new Date().toISOString(),
+ }), { flag: 'wx', mode: 0o600 })
+}
+
+export function verifyQaModelProjection(configMap, binding) {
+ const bound = binding.provider === 'grok-subscription' ? projection.toEligibleGrokPolicyBinding : projection.toEligiblePolicyBinding
+ const models = JSON.parse(configMap.data?.[binding.provider] ?? '[]')
+ for (const [model, expected] of [[binding.modelId, 'supported'], [binding.unsupportedModelId, 'unsupported'],
+ [binding.fallback.modelId, 'supported']]) {
+ const eligible = bound(configMap, binding.connectionKey, model)
+ const matches = models.filter(value => value.model === model)
+ assert(eligible.binding?.credentialRevision === binding.credentialRevision &&
+ eligible.binding.catalogRevision === binding.catalogRevision && matches.length === 1 &&
+ imageCapability.resolveImageInputCapability(matches[0].imageInput, { transportSupported: true }).state === expected,
+ 'COORDINATOR_ACTUAL_MODEL_ELIGIBILITY_UNPROVED')
+ }
+ assert(binding.fallback.provider === binding.provider && binding.fallback.modelId !== binding.modelId &&
+ binding.fallback.modelId !== binding.unsupportedModelId, 'COORDINATOR_FALLBACK_GRANT_INVALID')
+ return true
+}
+
+// The actual import publishes its own allowlisted witness. This reader returns
+// only that witness and the append-only ledger; no process environment/argv,
+// credentials, request body or diagnostic output is read.
+export const VENDOR_OBSERVATION_PROGRAM = String.raw`
+const fs=require('node:fs');let raw='';
+process.stdin.on('data',chunk=>{raw+=chunk;if(raw.length>65536)process.exit(1)});
+process.stdin.on('end',()=>{try{
+ const input=JSON.parse(raw);
+ const read=file=>{const stat=fs.lstatSync(file);if(!stat.isFile()||stat.isSymbolicLink()||stat.nlink!==1||
+ (stat.mode&511)!==384||stat.uid!==process.getuid()||stat.size>4*1024*1024)throw Error('VENDOR_RECORD_INVALID');
+ return JSON.parse(fs.readFileSync(file,'utf8'))};
+ const witness=read(input.evidencePath+'.runtime.json');
+ if(witness.pid!==Number(witness.pid)||!Number.isSafeInteger(witness.pid)||witness.pid<1||
+ fs.readFileSync('/proc/'+witness.pid+'/stat','utf8').split(') ')[1].split(' ')[19]!==witness.startTime)throw Error('VENDOR_PROCESS_CHANGED');
+ process.stdout.write(JSON.stringify({witness,ledger:read(input.evidencePath)}));
+}catch{process.exitCode=1}});`
+
+async function activateVendors({ contextDir, source, memoryOptions, qa, runId, states }) {
+ const manifest = JSON.parse(fs.readFileSync(path.join(ROOT, 'deploy/minikube/.image-manifest.json'), 'utf8'))
+ assert(manifest.profile === memoryOptions.profile, 'COORDINATOR_IMAGE_MANIFEST_PROFILE')
+ const hostRef = 'clerum/mcp-host:test', hostId = manifest.images[hostRef]
+ assert(validImageId(hostId) && manifest.sourceRevisions?.[hostRef] === source.gitHead,
+ 'COORDINATOR_EXACT_MUSL_HOST_IMAGE_REQUIRED')
+ const sources = {}
+ for (const provider of PROVIDERS) {
+ const name = `${provider.startsWith('grok') ? 'grok' : 'codex'}-llm-proxy`
+ const resource = getDeployment(memoryOptions.context, name)
+ const names = ['NODE_ENV', 'NODE_OPTIONS', 'EVENFIRE_SUBSCRIPTION_IMAGE_VENDOR_FIXTURE',
+ 'SUBSCRIPTION_IMAGE_FIXTURE_PROVIDER', 'SUBSCRIPTION_IMAGE_EVIDENCE_PATH', 'E2E_SUBSCRIPTION_IMAGE_RUN_ID',
+ 'E2E_GROK_IMAGE_MODEL', 'E2E_GROK_IMAGE_UNSUPPORTED_MODEL', 'E2E_CODEX_IMAGE_MODEL', 'E2E_CODEX_IMAGE_UNSUPPORTED_MODEL']
+ const original = snapshotDeployment(resource, name, names)
+ const baseId = manifest.images[original.image] ?? manifest.images[`docker.io/${original.image}`]
+ const basePod = await until(() => readyPod(memoryOptions.context, name, original.uid), Boolean, 120_000,
+ 'COORDINATOR_BASE_PROXY_NOT_READY')
+ assert(validImageId(baseId) && imageId(basePod.status.containerStatuses.find(value => value.name === name).imageID) === baseId &&
+ (manifest.sourceRevisions?.[original.image] ?? manifest.sourceRevisions?.[`docker.io/${original.image}`]) === source.gitHead,
+ 'COORDINATOR_BASE_PROXY_SOURCE_MISMATCH')
+ const baseDist = command('kubectl', ['--context', memoryOptions.context, '-n', NAMESPACE, 'exec', basePod.metadata.name,
+ '-c', name, '--', 'env', '-u', 'NODE_OPTIONS', 'node', '-e',
+ "process.stdout.write(require('node:crypto').createHash('sha256').update(require('node:fs').readFileSync('dist/main.js')).digest('hex'))"],
+ { timeout: 30_000 }).trim()
+ const fixtureTag = `clerum/${name}-image-qa:${runId.slice(-12)}`
+ const built = JSON.parse(command('bash', [path.join(ROOT, 'scripts/e2e/build-subscription-image-proxy.sh'),
+ memoryOptions.profile, contextDir, original.image, baseId, hostId, fixtureTag, source.gitHead], { timeout: 450_000 }))
+ assert(built.baseImageId === baseId && built.hostImageId === hostId && built.gitHead === source.gitHead &&
+ validImageId(built.imageId), 'COORDINATOR_DERIVED_BUILD_IDENTITY')
+ const env = { ...original.env, NODE_ENV: 'test', NODE_OPTIONS: [original.env.NODE_OPTIONS,
+ `--import=${IMPORT_PATH}`].filter(Boolean).join(' '), EVENFIRE_SUBSCRIPTION_IMAGE_VENDOR_FIXTURE: '1',
+ SUBSCRIPTION_IMAGE_FIXTURE_PROVIDER: provider,
+ SUBSCRIPTION_IMAGE_EVIDENCE_PATH: `/tmp/evenfire-vendor/${runId}-${provider}.json`, E2E_SUBSCRIPTION_IMAGE_RUN_ID: runId }
+ for (const binding of qa.bindings) {
+ const prefix = binding.provider.startsWith('grok') ? 'GROK' : 'CODEX'
+ env[`E2E_${prefix}_IMAGE_MODEL`] = binding.modelId
+ env[`E2E_${prefix}_IMAGE_UNSUPPORTED_MODEL`] = binding.unsupportedModelId
+ }
+ const applied = { image: fixtureTag, env }
+ const state = { name, original, applied, baseImageId: baseId, fixtureImageId: built.imageId,
+ gitHead: source.gitHead, changed: true, provider }; states.push(state)
+ patchDeployment(memoryOptions.context, name, deploymentMutationPatch(getDeployment(memoryOptions.context, name), original, applied))
+ command('kubectl', ['--context', memoryOptions.context, '-n', NAMESPACE, 'rollout', 'status', `deployment/${name}`, '--timeout=120s'],
+ { timeout: 150_000 })
+ const pod = await until(() => readyPod(memoryOptions.context, name, original.uid), Boolean, 120_000,
+ 'COORDINATOR_DERIVED_PROXY_NOT_READY')
+ const observation = json(memoryOptions.context, ['-n', NAMESPACE, 'exec', '-i', pod.metadata.name, '-c', name,
+ '--', 'env', '-u', 'NODE_OPTIONS', '-u', 'EVENFIRE_SUBSCRIPTION_IMAGE_VENDOR_FIXTURE',
+ 'node', '-e', VENDOR_OBSERVATION_PROGRAM], JSON.stringify({ evidencePath: env.SUBSCRIPTION_IMAGE_EVIDENCE_PATH }))
+ const actual = observation.witness
+ assert(actual.kind === 'evenfire-subscription-image-vendor-runtime-v1' && actual.runId === runId && actual.provider === provider &&
+ /^24\./.test(actual.nodeVersion) && actual.musl === true && actual.nodeOptionsSha256 === digest(env.NODE_OPTIONS) &&
+ actual.importSha256 === digest(fs.readFileSync(path.join(contextDir, 'scripts/e2e/fixtures/subscription-image-provider.mjs'))) &&
+ actual.productionDistSha256 === baseDist, 'COORDINATOR_ACTUAL_IMPORT_OR_DIST_MISMATCH')
+ const physicalImageId = imageId(pod.status.containerStatuses.find(value => value.name === name).imageID)
+ assert(physicalImageId === built.imageId && physicalImageId !== baseId, 'COORDINATOR_DERIVED_IMAGE_UNPROVED')
+ const witness = { profile: memoryOptions.profile, gitHead: source.gitHead, podUid: pod.metadata.uid,
+ imageId: physicalImageId, fixtureImportSha256: actual.importSha256,
+ deploymentUid: original.uid, pid: actual.pid, processStartTime: actual.startTime,
+ productionDistSha256: baseDist, baseImageId: baseId, hostImageId: hostId }
+ Object.assign(state, { podName: pod.metadata.name, witness, env, runtimeWitness: actual })
+ sources[provider] = witness
+ }
+ return sources
+}
+
+function selectedFields(resource, original) {
+ assert(resource.metadata.uid === original.uid, 'COORDINATOR_DEPLOYMENT_OWNER_CHANGED')
+ const container = resource.spec.template.spec.containers.find(value => value.name === original.containerName)
+ const env = {}
+ for (const name of original.names) {
+ const values = (container.env ?? []).filter(value => value.name === name)
+ assert(values.length <= 1 && values.every(value => !value.valueFrom), 'COORDINATOR_RESTORE_ENV_AMBIGUOUS')
+ if (values.length) env[name] = values[0].value
+ }
+ return { image: container.image, env }
+}
+
+async function restoreDeployments(context, states) {
+ const failures = []
+ for (const state of [...states].reverse()) {
+ if (!state.changed) continue
+ try {
+ const current = getDeployment(context, state.name)
+ const wanted = { image: state.original.image, env: state.original.env }
+ if (!isDeepStrictEqual(selectedFields(current, state.original), wanted))
+ patchDeployment(context, state.name, deploymentMutationPatch(current, state.original, state.applied, true))
+ command('kubectl', ['--context', context, '-n', NAMESPACE, 'rollout', 'status', `deployment/${state.name}`, '--timeout=120s'],
+ { timeout: 150_000 })
+ assert(isDeepStrictEqual(selectedFields(getDeployment(context, state.name), state.original), wanted),
+ 'COORDINATOR_RESTORATION_UNPROVED')
+ const pod = await until(() => readyPod(context, state.name, state.original.uid), Boolean, 120_000, 'COORDINATOR_RESTORED_POD_UNREADY')
+ assert(imageId(pod.status.containerStatuses.find(value => value.name === state.name).imageID) === state.baseImageId,
+ 'COORDINATOR_RESTORED_IMAGE_ID_CHANGED')
+ state.changed = false
+ } catch { failures.push(state.name) }
+ }
+ assert(failures.length === 0, 'COORDINATOR_RESTORATION_FAILED')
+}
+
+export async function * vendorFrames({ admission, states, loginFrame, remainingFrame, observe, signal }) {
+ yield Buffer.from(JSON.stringify(loginFrame) + '\n')
+ if (remainingFrame) yield Buffer.from(JSON.stringify(remainingFrame) + '\n')
+ const previous = new Map(PROVIDERS.map(provider => [provider, []]))
+ const hashes = new Map()
+ while (!signal.aborted) {
+ for (const state of states.filter(value => value.provider)) {
+ const observed = await observe(state)
+ const frame = { kind: 'evenfire-subscription-image-vendor-frame-v1', runId: admission.runId,
+ provider: state.provider, source: state.witness, ledger: observed.ledger }
+ const binding = admission.bindings.find(value => value.provider === state.provider)
+ admitVendorFrame(frame, admission.runId, binding, admission.vendorSources[state.provider], previous.get(state.provider))
+ const hash = digest(JSON.stringify(frame))
+ if (hash !== hashes.get(state.provider)) {
+ previous.set(state.provider, structuredClone(frame.ledger.attempts)); hashes.set(state.provider, hash)
+ yield Buffer.from(JSON.stringify(frame) + '\n')
+ }
+ }
+ if (!signal.aborted) await pause(50) // Receipt polling never advances UI.
+ }
+}
+
+export const PRESSURE_RELAY_PROGRAM = String.raw`
+import fs from 'node:fs';import net from 'node:net';
+const socketPath=process.argv[1];
+if(!/^\/run\/evenfire-e2e\/pressure-[a-f0-9]{12}\.sock$/.test(socketPath)||fs.existsSync(socketPath))throw Error('RELAY_PATH_INVALID');
+const pending=new Map(),sockets=new Set();let next=0,buffer='';
+const server=net.createServer(socket=>{let raw='';sockets.add(socket);socket.requestBusy=false;
+ socket.on('data',chunk=>{raw+=chunk;if(raw.length>65536||socket.requestBusy){socket.destroy();return}
+ const end=raw.indexOf('\n');if(end<0)return;
+ const line=raw.slice(0,end);raw=raw.slice(end+1);if(raw){socket.destroy();return}
+ const callId=String(++next);pending.set(callId,socket);socket.requestBusy=true;socket.pause();
+ process.stdout.write(JSON.stringify({callId,input:JSON.parse(line)})+'\n');
+ });socket.on('close',()=>{sockets.delete(socket);for(const [id,value]of pending)if(value===socket)pending.delete(id)});
+});
+process.stdin.on('data',chunk=>{buffer+=chunk;if(buffer.length>65536)throw Error('RELAY_BOUND');
+ for(let end;(end=buffer.indexOf('\n'))>=0;){const frame=JSON.parse(buffer.slice(0,end));buffer=buffer.slice(end+1);
+ const socket=pending.get(frame.callId);if(!socket)throw Error('RELAY_UNKNOWN_REPLY');pending.delete(frame.callId);
+ socket.write(JSON.stringify(frame.data)+'\n');socket.requestBusy=false;socket.resume();
+ }});
+process.stdin.on('end',()=>{for(const socket of sockets)socket.destroy();server.close(()=>{
+ if(fs.existsSync(socketPath))fs.unlinkSync(socketPath);process.exitCode=0})});
+server.listen(socketPath,()=>{fs.chmodSync(socketPath,384);process.stdout.write(JSON.stringify({ready:true,socketPath})+'\n')});`
+
+/** A broken relay must not bypass native request release or Channel shutdown. */
+export async function closePressureResources(relay, pressure) {
+ const results = await Promise.allSettled([relay, pressure].map(resource =>
+ Promise.resolve().then(() => resource?.close())))
+ const failures = results.filter(value => value.status === 'rejected').map(value => value.reason)
+ if (failures.length) throw new AggregateError(failures, 'COORDINATOR_PRESSURE_RESTORATION_FAILED')
+}
+
+/** A successful runner does not authorize canonical vendor restoration. The
+ * actual seeder must first acknowledge both owned grant revocations/detaches.
+ */
+export async function withImageQaSession(session, runId, grantStatus, work) {
+ grantStatus.state = 'created-or-unknown'
+ try {
+ const qa = await session.prepareImages(runId)
+ grantStatus.state = 'created'
+ grantStatus.bindings = qa.bindings.map(({ provider, hostRef, hostUid, connectionKey, connectionId }) =>
+ ({ provider, hostRef, hostUid, connectionKey, connectionId }))
+ return await work(qa)
+ } finally {
+ const restored = await session.revokeImages()
+ assert(restored.verified === true && (grantStatus.state !== 'created' ||
+ (restored.grantsRevoked === grantStatus.bindings.length && restored.hostsDetached === grantStatus.bindings.length)),
+ 'COORDINATOR_QA_GRANT_RESTORATION_FAILED')
+ grantStatus.state = 'revoked'
+ }
+}
+
+export function qaRestorationTargets(grantState, states) {
+ return { restore: states.filter(state => !state.provider || grantState === 'revoked'),
+ retain: states.filter(state => state.provider && state.changed && grantState !== 'revoked') }
+}
+
+function writeQaRecovery(filename, grantStatus, states, options, runId) {
+ const directory = path.dirname(filename), stat = fs.lstatSync(directory)
+ assert(stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === process.getuid?.() &&
+ !(stat.mode & 0o077) && fs.realpathSync(directory) === directory, 'COORDINATOR_RECOVERY_DIRECTORY_UNSAFE')
+ fs.writeFileSync(filename, JSON.stringify({ kind: 'evenfire-subscription-image-qa-recovery-v1',
+ runId, profile: options.profile, context: options.context, recordedAt: new Date().toISOString(),
+ qaGrantState: grantStatus.state, qaBindings: grantStatus.bindings,
+ vendorsRetained: states.map(state => ({ provider: state.provider, name: state.name,
+ deploymentUid: state.original.uid, originalImageId: state.baseImageId, originalImage: state.original.image,
+ fixtureImageId: state.fixtureImageId, gitHead: state.gitHead,
+ originalControlledEnv: state.original.env, fixtureImage: state.applied.image,
+ fixtureControlledEnv: state.applied.env, witness: state.witness })),
+ canonicalRestore: 'BLOCKED_UNTIL_OWNED_GRANT_REVOCATION_VERIFIED',
+ }), { flag: 'wx', mode: 0o600 })
+}
+
+async function pressureRelay({ containerName, pressure, metadata, processIdentity }) {
+ const child = spawn('docker', ['exec', '-i', containerName, 'node', '--input-type=module', '-e', PRESSURE_RELAY_PROGRAM,
+ metadata.socketPath], { stdio: ['pipe', 'pipe', 'pipe'] })
+ child.stderr.resume(); child.stdout.setEncoding('utf8')
+ let buffer = '', chain = Promise.resolve(), failed, readyResolve, readyReject, accepting = true
+ const ready = new Promise((resolve, reject) => { readyResolve = resolve; readyReject = reject })
+ const observation = value => ({ ok: true, pressureRunId: metadata.pressureRunId, maxInFlight: metadata.maxInFlight,
+ owners: { baseline: 0, held: value.inFlight, drained: value.inFlight },
+ counts: { sameRunAttempts: value.counts.total, sameRunTickets: value.counts.tickets, reservations: value.counts.reservationRows },
+ pids: [value.pid], inspector: { pid: value.pid, startTime: processIdentity.startTime } })
+ child.stdout.on('data', chunk => {
+ if (!accepting) return
+ buffer += chunk
+ if (Buffer.byteLength(buffer) > 65536) { failed = true; readyReject(new Error('PRESSURE_RELAY_BOUND')); child.kill('SIGTERM'); return }
+ for (let end; (end = buffer.indexOf('\n')) >= 0;) {
+ const line = buffer.slice(0, end); buffer = buffer.slice(end + 1)
+ chain = chain.then(async () => {
+ const frame = JSON.parse(line)
+ if (frame.ready === true) { assert(frame.socketPath === metadata.socketPath, 'PRESSURE_RELAY_PATH_CHANGED'); readyResolve(); return }
+ const input = frame.input
+ assert(input.pressureRunId === metadata.pressureRunId && ['hello', 'hold', 'owners', 'release', 'close'].includes(input.command),
+ 'PRESSURE_RELAY_COMMAND_INVALID')
+ let value
+ if (input.command === 'hold') {
+ assert(input.maxInFlight === metadata.maxInFlight, 'PRESSURE_RELAY_POLICY_CHANGED')
+ value = await pressure.hold()
+ } else if (['release', 'close'].includes(input.command)) value = await pressure.release()
+ else value = await pressure.owners()
+ const data = observation(value)
+ if (input.command === 'hold') data.owners.drained = 0
+ child.stdin.write(JSON.stringify({ callId: frame.callId, data }) + '\n')
+ }).catch(() => { failed = true; readyReject(new Error('PRESSURE_RELAY_FAILED')); child.kill('SIGTERM') })
+ }
+ })
+ child.once('error', () => { failed = true; readyReject(new Error('PRESSURE_RELAY_START_FAILED')) })
+ const exited = new Promise(resolve => child.once('close', (code, signal) => {
+ if (code !== 0 || signal) { failed = true; readyReject(new Error('PRESSURE_RELAY_PRODUCER_FAILED')) }
+ resolve({ code, signal })
+ }))
+ const timer = setTimeout(() => { readyReject(new Error('PRESSURE_RELAY_READY_TIMEOUT')); child.kill('SIGTERM') }, 30_000)
+ try { await ready } finally { clearTimeout(timer) }
+ let closing
+ return { assertHealthy: () => assert(!failed, 'PRESSURE_RELAY_PRODUCER_FAILED'),
+ close: () => closing ??= (async () => {
+ accepting = false
+ await chain; child.stdin.end()
+ const timer = setTimeout(() => child.kill('SIGKILL'), 10_000)
+ const result = await exited; clearTimeout(timer)
+ assert(!failed && result.code === 0 && !result.signal, 'PRESSURE_RELAY_PRODUCER_FAILED')
+ })() }
+}
+
+export async function coordinate(prepareArguments, memoryOptions, privateMaterial, catalogRuntime) {
+ assert(process.versions.node.startsWith('24.') && fs.realpathSync(process.cwd()) === ROOT, 'COORDINATOR_NODE24_WORKTREE_REQUIRED')
+ assert((memoryOptions.prepareFixtures === true || memoryOptions.fixtureBinding) && privateMaterial && typeof privateMaterial.operatorPassword === 'string' &&
+ privateMaterial.operatorPassword.length >= 8 && privateMaterial.operatorPassword.length <= 1024 &&
+ (privateMaterial.cookie === undefined || (typeof privateMaterial.cookie === 'string' &&
+ privateMaterial.cookie.length <= 8192 && !/[\r\n\0]/.test(privateMaterial.cookie))) &&
+ Object.keys(privateMaterial).every(key => ['operatorPassword', 'cookie'].includes(key)),
+ 'COORDINATOR_PRIVATE_OPERATOR_REQUIRED')
+ assert(memoryOptions.operatorUser === `${memoryOptions.runId}-operator`, 'COORDINATOR_ISOLATED_OPERATOR_REQUIRED')
+ assert(memoryOptions.prepare === 'restart', 'COORDINATOR_RECONCILED_BASELINE_REQUIRED')
+ const values = Object.fromEntries(Array.from({ length: prepareArguments.length / 2 }, (_, index) =>
+ [prepareArguments[index * 2].slice(2), prepareArguments[index * 2 + 1]]))
+ assert(values.mode === 'fixture' && values.profile === memoryOptions.profile && /^subscription-image-[a-f0-9]{12}$/.test(values['run-id'] ?? '') &&
+ values.frames === '-' && !values.bindings && path.isAbsolute(values.scratch), 'COORDINATOR_PUBLIC_ARGUMENTS_INVALID')
+ const suite = resolveSuite(values.suite)
+ command('bash', [path.join(ROOT, 'scripts/minikube/require-t2-mutation-lock.sh')], { timeout: 10_000 })
+ const states = [], abort = new AbortController()
+ const grantStatus = catalogRuntime?.grantStatus ?? { state: 'not-created', bindings: [] }
+ let catalogFence = catalogRuntime?.catalogFence, memoryState = memoryOptions.fixtureBinding ? 'created' : 'not-created'
+ let result, failure, relay, pressure
+ let pressureClosing
+ const closePressure = () => pressureClosing ??= closePressureResources(relay, pressure)
+ const onSignal = () => abort.abort(new Error('COORDINATOR_INTERRUPTED'))
+ process.once('SIGINT', onSignal); process.once('SIGTERM', onSignal)
+ try {
+ if (!catalogFence) catalogFence = await openControlApiCatalogIsolation(memoryOptions)
+ if (values.suite === 'admission-recovery') {
+ const resource = getDeployment(memoryOptions.context, 'control-api')
+ const original = snapshotDeployment(resource, 'control-api', ['NODE_OPTIONS'])
+ const basePod = await until(() => readyPod(memoryOptions.context, 'control-api', original.uid), Boolean, 120_000,
+ 'COORDINATOR_BASE_API_NOT_READY')
+ const baseImageId = imageId(basePod.status.containerStatuses.find(value => value.name === 'control-api').imageID)
+ assert(validImageId(baseImageId), 'COORDINATOR_BASE_API_IMAGE_UNKNOWN')
+ const applied = { image: original.image, env: { NODE_OPTIONS:
+ `--max-old-space-size=${memoryOptions.candidate.heapSizeMiB} --inspect=127.0.0.1:${memoryOptions.inspectorPort}` } }
+ const state = { name: 'control-api', original, applied, baseImageId, changed: true }; states.push(state)
+ patchDeployment(memoryOptions.context, state.name, deploymentMutationPatch(resource, original, applied))
+ command('kubectl', ['--context', memoryOptions.context, '-n', NAMESPACE, 'rollout', 'status', 'deployment/control-api', '--timeout=120s'],
+ { timeout: 150_000 })
+ }
+ if (memoryOptions.prepareFixtures) memoryState = 'created-or-unknown'
+ await seed(memoryOptions, privateMaterial, async session => {
+ memoryState = 'created'
+ await withImageQaSession(session, values['run-id'], grantStatus, async qa => {
+ assert(qa.credentialState === 'opaque-qa-not-real-G8' && qa.vendorCronsDisabled === true &&
+ qa.operatorDesktopUserId === session.fixtures.operatorDesktopUserId && qa.operatorDesktopUserId !== session.fixtures.operatorId,
+ 'COORDINATOR_DESKTOP_OPERATOR_UNPROVED')
+ const configMap = json(memoryOptions.context, ['-n', memoryOptions.hostNamespace, 'get', 'configmap',
+ 'clerum-llm-allowed-models', '-o', 'json'])
+ for (const binding of qa.bindings) verifyQaModelProjection(configMap, binding)
+ let closed = false, remainingFrame
+ const flow = { bindings: qa.bindings.map(({ provider, hostRef, hostLabel, modelId, modelLabel,
+ unsupportedModelId, unsupportedModelLabel }) => ({ provider, hostRef, hostLabel, modelId, modelLabel,
+ unsupportedModelId, unsupportedModelLabel })),
+ async beforeBuild(input) {
+ flow.vendorSources = await activateVendors({ ...input, memoryOptions, qa, runId: values['run-id'], states })
+ },
+ async configure(admission, { containerName }) {
+ let metadata
+ if (admission.suiteId === 'admission-recovery') {
+ const child = spawn('kubectl', ['--context', memoryOptions.context, '-n', NAMESPACE, 'exec', '-i', session.apiPod.name,
+ '-c', 'control-api', '--', 'env', '-u', 'NODE_OPTIONS', 'node', '--max-old-space-size=64',
+ '--input-type=module', '-e', buildAuthorizeMemoryCompanionBundle()], { cwd: ROOT, stdio: ['pipe', 'pipe', 'pipe'] })
+ pressure = await openAuthorizeAdmissionPressure(new Channel(child, pressureCommandDeadlineMs(memoryOptions.candidate)), {
+ candidate: memoryOptions.candidate, inspectorPort: memoryOptions.inspectorPort,
+ runId: `pr806-memory-${admission.runId.slice(-12)}`, hostNamespace: memoryOptions.hostNamespace,
+ bindings: qa.bindings.map(value => ({ hostRef: value.hostRef, hostUid: value.hostUid })) })
+ const owner = await pressure.owners()
+ const processIdentity = json(memoryOptions.context, ['-n', NAMESPACE, 'exec', '-i', session.apiPod.name,
+ '-c', 'control-api', '--', 'env', '-u', 'NODE_OPTIONS', 'node', '-e',
+ "let s='';process.stdin.on('data',d=>s+=d);process.stdin.on('end',()=>{const pid=JSON.parse(s).pid;const startTime=require('node:fs').readFileSync('/proc/'+pid+'/stat','utf8').split(') ')[1].split(' ')[19];process.stdout.write(JSON.stringify({pid,startTime}))})"],
+ JSON.stringify({ pid: owner.pid }))
+ metadata = { kind: 'evenfire-subscription-image-pressure-metadata-v1', profile: admission.profile,
+ context: admission.context, worktreeId: admission.stack.worktreeId,
+ sourceManifestSha256: admission.sourceManifestSha256, podUid: session.apiPod.uid,
+ imageId: session.apiPod.imageId, pressureRunId: `subscription-image-pressure-${admission.runId.slice(-12)}`,
+ hostRefs: qa.bindings.map(value => value.hostRef), maxInFlight: memoryOptions.candidate.concurrency,
+ readDeadlineMs: pressure.policy.readDeadlineMs, workDeadlineMs: pressure.policy.workDeadlineMs,
+ closeGraceMs: pressure.policy.closeGraceMs, commandDeadlineMs: pressure.commandDeadlineMs,
+ socketPath: `${RUNNER_RUN_BASE}/pressure-${admission.runId.slice(-12)}.sock` }
+ relay = await pressureRelay({ containerName, pressure, metadata, processIdentity })
+ }
+ if (suite.fixtureReceiptEnv) {
+ const prepared = await prepareRemainingFixtures({ admission, runRoot: `${RUNNER_RUN_BASE}/${admission.runId}`,
+ hostBindings: qa.bindings,
+ prepareGfs: input => session.prepareGfs({ ...input, userId: qa.operatorDesktopUserId,
+ parentResourceId: session.fixtures.gfs.parentResourceId }),
+ pressure: metadata && { ...metadata,
+ receiptFile: '/runner-admission/main-admission.json' } })
+ remainingFrame = prepared.frame
+ }
+ return metadata ? { pressureMetadata: metadata } : undefined
+ },
+ frames(admission) {
+ const loginFrame = Object.fromEntries([['kind', 'evenfire-subscription-image-login-v1'],
+ ['runId', admission.runId], ['email', qa.loginEmail], ['password', privateMaterial.operatorPassword]])
+ return Readable.from(vendorFrames({ admission, states, remainingFrame, loginFrame, signal: abort.signal,
+ observe: state => {
+ relay?.assertHealthy()
+ const current = readyPod(memoryOptions.context, state.name, state.original.uid)
+ assert(current?.metadata?.uid === state.witness.podUid &&
+ imageId(current.status.containerStatuses.find(value => value.name === state.name).imageID) === state.witness.imageId,
+ 'COORDINATOR_VENDOR_POD_CHANGED')
+ const observed = json(memoryOptions.context, ['-n', NAMESPACE, 'exec', '-i', state.podName, '-c', state.name,
+ '--', 'env', '-u', 'NODE_OPTIONS', '-u', 'EVENFIRE_SUBSCRIPTION_IMAGE_VENDOR_FIXTURE', 'node', '-e', VENDOR_OBSERVATION_PROGRAM],
+ JSON.stringify({ evidencePath: state.env.SUBSCRIPTION_IMAGE_EVIDENCE_PATH }))
+ assert(isDeepStrictEqual(observed.witness, state.runtimeWitness), 'COORDINATOR_VENDOR_SOURCE_CHANGED')
+ return observed
+ } }), { objectMode: false, highWaterMark: 64 * 1024 })
+ },
+ async close() {
+ if (closed) return
+ closed = true; abort.abort()
+ await closePressure()
+ },
+ }
+ result = await prepare(prepareArguments, flow)
+ assert(result === 0 && !abort.signal.reason?.message?.includes('INTERRUPTED'), 'COORDINATOR_RUNNER_FAILED')
+ })
+ })
+ } catch (error) { failure = error }
+ finally {
+ abort.abort()
+ try { await closePressure() } catch (error) {
+ failure = failure ? new AggregateError([failure, error], 'COORDINATOR_OPERATION_AND_CLEANUP_FAILED') : error
+ }
+ const targets = qaRestorationTargets(grantStatus.state, states)
+ try { await restoreDeployments(memoryOptions.context, targets.restore) } catch (error) {
+ failure = failure ? new AggregateError([failure, error], 'COORDINATOR_OPERATION_AND_CLEANUP_FAILED') : error
+ }
+ if (targets.retain.length) {
+ try {
+ writeQaRecovery(path.join(path.dirname(memoryOptions.report), `${values['run-id']}-qa-recovery.json`),
+ grantStatus, targets.retain, memoryOptions, values['run-id'])
+ } catch (error) {
+ failure = failure ? new AggregateError([failure, error], 'COORDINATOR_OPERATION_AND_CLEANUP_FAILED') : error
+ }
+ failure ??= new Error('COORDINATOR_CANONICAL_VENDOR_RESTORE_BLOCKED')
+ }
+ if (catalogFence && !catalogRuntime) {
+ if (memoryState === 'not-created' && ['not-created', 'revoked'].includes(grantStatus.state)) {
+ try { await restoreDeployments(memoryOptions.context, [catalogFence]) } catch (error) { failure ??= error }
+ } else {
+ try { writeCatalogRecovery(path.join(path.dirname(memoryOptions.report), `${values['run-id']}-catalog-isolation.json`),
+ catalogFence, memoryOptions, memoryState, [grantStatus]) } catch (error) { failure ??= error }
+ }
+ }
+ process.off('SIGINT', onSignal); process.off('SIGTERM', onSignal)
+ }
+ if (failure) throw new Error(/^[A-Z0-9_]+$/.test(failure.message) ? failure.message : 'COORDINATOR_FAILED')
+ return { kind: 'evenfire-subscription-image-coordinator-v1', producerExit: result,
+ vendorBoundary: 'derived-external-fetch-only', restoration: { verified: true, scope: 'vendor-images-and-inspector' },
+ catalogIsolation: { retainedForMemoryFixtures: memoryState === 'created',
+ ...(catalogFence ? { initialSnapshot: catalogFence.original } : {}) }, realOAuthOrG8: 'NOT_RUN' }
+}
+
+async function main() {
+ const argv = process.argv.slice(2)
+ const index = argv.indexOf('--memory-config')
+ assert(index >= 0 && argv[index + 1], 'COORDINATOR_MEMORY_CONFIG_REQUIRED')
+ const [, filename] = argv.splice(index, 2)
+ const memoryOptions = parseOptions(['--config', filename])
+ let bytes = Buffer.alloc(0), material
+ for await (const chunk of process.stdin) {
+ bytes = Buffer.concat([bytes, chunk]); assert(bytes.length <= 64 * 1024, 'COORDINATOR_PRIVATE_INPUT_BOUND')
+ const end = bytes.indexOf(10)
+ if (end < 0) continue
+ assert(end === bytes.length - 1, 'COORDINATOR_ONE_PRIVATE_INPUT_REQUIRED')
+ material = JSON.parse(bytes.toString('utf8')); bytes.fill(0); bytes = Buffer.alloc(0); break
+ }
+ assert(material?.kind === 'evenfire-subscription-image-private-control-v1', 'COORDINATOR_PRIVATE_INPUT_REQUIRED')
+ const privateMaterial = Object.fromEntries(['operatorPassword', 'cookie'].filter(field => material[field] !== undefined)
+ .map(field => [field, material[field]]))
+ material = undefined
+ const result = await coordinate(argv, memoryOptions, privateMaterial)
+ for (const field of Object.keys(privateMaterial)) delete privateMaterial[field]
+ process.stdout.write(JSON.stringify(result) + '\n')
+}
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
+ main().catch(() => { process.stderr.write('SUBSCRIPTION_IMAGE_COORDINATOR_FAILED\n'); process.exitCode = 1 })
diff --git a/scripts/e2e/fixtures/subscription-image-admission-pressure.mjs b/scripts/e2e/fixtures/subscription-image-admission-pressure.mjs
index 9a9a79863..6d8c94ee8 100644
--- a/scripts/e2e/fixtures/subscription-image-admission-pressure.mjs
+++ b/scripts/e2e/fixtures/subscription-image-admission-pressure.mjs
@@ -1,3 +1,4 @@
+// E2E_GUARDIAN_IPC_FLOW: private fixture protocol, never a renderer transition.
// Frozen client surface for the private admission-pressure companion.
//
// openAdmissionPressure({ receiptFile, socketPath, deadlineMs }) -> {
@@ -39,6 +40,18 @@ const refuse = code => {
}
const MAX_FRAME_BYTES = 64 * 1024
+/** Existing lifecycle observation grace plus one private IPC delivery window.
+ * Values come from the actual compiled policy returned by the companion.
+ * @param {{readDeadlineMs:number, workDeadlineMs:number, closeGraceMs:number}} policy
+ */
+export function pressureCommandDeadlineMs(policy) {
+ if (!Number.isSafeInteger(policy?.readDeadlineMs) || policy.readDeadlineMs < 100 || policy.readDeadlineMs > 60_000 ||
+ !Number.isSafeInteger(policy.workDeadlineMs) || policy.workDeadlineMs < 100 || policy.workDeadlineMs > 120_000 ||
+ !Number.isSafeInteger(policy.closeGraceMs) || policy.closeGraceMs < 1 || policy.closeGraceMs > 10_000 ||
+ policy.closeGraceMs >= policy.readDeadlineMs) refuse('ADMISSION_PRESSURE_POLICY_INVALID')
+ return Math.max(policy.readDeadlineMs, policy.workDeadlineMs + policy.closeGraceMs + 5_000) + 5_000
+}
+
export function verifyPressureMetadata(metadata) {
if (
metadata?.kind !== 'evenfire-subscription-image-pressure-metadata-v1' ||
@@ -56,6 +69,8 @@ export function verifyPressureMetadata(metadata) {
metadata.hostRefs.some(value => typeof value !== 'string' || !value) ||
!Number.isSafeInteger(metadata.maxInFlight) ||
metadata.maxInFlight < 1 ||
+ metadata.maxInFlight > 4 ||
+ metadata.commandDeadlineMs !== pressureCommandDeadlineMs(metadata) ||
typeof metadata.socketPath !== 'string' ||
!path.isAbsolute(metadata.socketPath)
)
@@ -103,12 +118,22 @@ export function verifyPressureObservation(value, expectation = {}, metadata) {
}
/**
- * @param {{ receiptFile?: string, deadlineMs?: number }} options
+ * @param {{ receiptFile?: string }} options
*/
-export async function openAdmissionPressure({ receiptFile, deadlineMs = 30_000 } = {}) {
+export async function openAdmissionPressure({ receiptFile } = {}) {
if (typeof receiptFile !== 'string' || !path.isAbsolute(receiptFile))
refuse('ADMISSION_PRESSURE_RECEIPT_REQUIRED')
- const metadata = verifyPressureMetadata(readMainRecord(receiptFile).value)
+ const sealed = readMainRecord(receiptFile).value
+ // The existing four-file admission volume seals this metadata inside the
+ // main admission. The private socket is created by an owned docker-exec
+ // relay in the runner's tmpfs; no extra mount or plaintext account is needed.
+ const metadata = verifyPressureMetadata(sealed.pressureMetadata ?? sealed)
+ const deadlineMs = metadata.commandDeadlineMs
+ if (sealed.pressureMetadata && (sealed.profile !== metadata.profile ||
+ sealed.context !== metadata.context || sealed.stack?.worktreeId !== metadata.worktreeId ||
+ sealed.sourceManifestSha256 !== metadata.sourceManifestSha256 ||
+ metadata.hostRefs.some(host => !sealed.bindings?.some(binding => binding.hostRef === host))))
+ refuse('ADMISSION_PRESSURE_SEALED_BINDING')
if (!fs.existsSync(metadata.socketPath)) refuse('ADMISSION_PRESSURE_SOCKET_MISSING')
const socket = await new Promise((resolve, reject) => {
const candidate = net.connect({ path: metadata.socketPath })
diff --git a/scripts/e2e/fixtures/subscription-image-provider.mjs b/scripts/e2e/fixtures/subscription-image-provider.mjs
index 9ea0d0300..862fc52d4 100644
--- a/scripts/e2e/fixtures/subscription-image-provider.mjs
+++ b/scripts/e2e/fixtures/subscription-image-provider.mjs
@@ -9,6 +9,7 @@ import { deepStrictEqual } from 'node:assert/strict'
import { createRequire } from 'node:module'
import fs from 'node:fs'
import path from 'node:path'
+import { fileURLToPath } from 'node:url'
import { requirePixelRenderer } from './subscription-image-challenge.cjs'
import { decodeInChild, hasCompleteImageContainer } from './subscription-image-decoder.mjs'
@@ -330,6 +331,26 @@ export function createSubscriptionImageVendor({ runId, bindings, acceptedProvide
return { respond, snapshot }
}
+/** The pod's emptyDir masks image-time /tmp contents. Create the immediate
+ * private evidence directory at runtime, then verify it without following a
+ * symlink or changing an existing directory's ownership/permissions.
+ */
+export function prepareVendorEvidenceDirectory(evidencePath) {
+ if (typeof evidencePath !== 'string' || !path.isAbsolute(evidencePath) ||
+ path.resolve(evidencePath) !== evidencePath || fs.existsSync(evidencePath)) {
+ throw new Error('External vendor evidence requires a fresh absolute path')
+ }
+ const directory = path.dirname(evidencePath)
+ try { fs.mkdirSync(directory, { mode: 0o700 }) }
+ catch (error) { if (error.code !== 'EEXIST') throw error }
+ const parent = fs.lstatSync(directory)
+ if (!parent.isDirectory() || parent.isSymbolicLink() || parent.uid !== process.getuid?.() ||
+ (parent.mode & 0o777) !== 0o700 || fs.realpathSync(directory) !== directory) {
+ throw new Error('External vendor evidence requires a private owned directory')
+ }
+ return directory
+}
+
/** Explicit derived-runtime hook. The caller owns restoration/image/lease proof. */
export function installSubscriptionImageVendor(env = process.env) {
if (env.NODE_ENV !== 'test' || env.EVENFIRE_SUBSCRIPTION_IMAGE_VENDOR_FIXTURE !== '1') {
@@ -338,13 +359,7 @@ export function installSubscriptionImageVendor(env = process.env) {
const acceptedProvider = env.SUBSCRIPTION_IMAGE_FIXTURE_PROVIDER
if (!Object.hasOwn(SUBSCRIPTION_VENDOR_URLS, acceptedProvider ?? '')) throw new Error('External vendor fixture requires its physical proxy provider binding')
const evidencePath = env.SUBSCRIPTION_IMAGE_EVIDENCE_PATH
- if (!evidencePath || !path.isAbsolute(evidencePath) || fs.existsSync(evidencePath)) {
- throw new Error('External vendor evidence requires a fresh absolute path')
- }
- const parent = fs.statSync(path.dirname(evidencePath))
- if (!parent.isDirectory() || parent.uid !== process.getuid?.() || (parent.mode & 0o077) !== 0) {
- throw new Error('External vendor evidence requires a private owned directory')
- }
+ prepareVendorEvidenceDirectory(evidencePath)
const bindings = ['GROK', 'CODEX'].map(kind => ({ provider: `${kind.toLowerCase()}-subscription`,
modelId: env[`E2E_${kind}_IMAGE_MODEL`], unsupportedModelId: env[`E2E_${kind}_IMAGE_UNSUPPORTED_MODEL`] }))
const vendor = createSubscriptionImageVendor({ runId: env.E2E_SUBSCRIPTION_IMAGE_RUN_ID, bindings, acceptedProvider,
@@ -353,6 +368,21 @@ export function installSubscriptionImageVendor(env = process.env) {
fs.writeFileSync(temporary, JSON.stringify(snapshot), { flag: 'wx', mode: 0o600 })
fs.renameSync(temporary, evidencePath)
} })
+ // Physical activation witness from the actual imported application process.
+ // Whitelist QA metadata; never read proc environ, headers or credential slots.
+ if (process.platform !== 'linux' || !/^24\./.test(process.versions.node)) {
+ throw new Error('External vendor fixture requires its inspected Linux Node24 runtime')
+ }
+ const witness = { kind: 'evenfire-subscription-image-vendor-runtime-v1',
+ runId: env.E2E_SUBSCRIPTION_IMAGE_RUN_ID, provider: acceptedProvider,
+ pid: process.pid, uid: process.getuid(),
+ startTime: fs.readFileSync('/proc/self/stat', 'utf8').split(') ')[1].split(' ')[19],
+ nodeVersion: process.versions.node, musl: !process.report.getReport().header.glibcVersionRuntime,
+ importSha256: sha256(fs.readFileSync(fileURLToPath(import.meta.url))),
+ nodeOptionsSha256: sha256(env.NODE_OPTIONS ?? ''),
+ productionDistSha256: sha256(fs.readFileSync(path.join(process.cwd(), 'dist/main.js'))),
+ bindings }
+ fs.writeFileSync(`${evidencePath}.runtime.json`, JSON.stringify(witness), { flag: 'wx', mode: 0o600 })
fs.writeFileSync(evidencePath, JSON.stringify(vendor.snapshot()), { flag: 'wx', mode: 0o600 })
const original = globalThis.fetch
globalThis.fetch = async (input, init) => {
diff --git a/scripts/e2e/fixtures/subscription-image-proxy.Dockerfile b/scripts/e2e/fixtures/subscription-image-proxy.Dockerfile
new file mode 100644
index 000000000..bf9edfdfc
--- /dev/null
+++ b/scripts/e2e/fixtures/subscription-image-proxy.Dockerfile
@@ -0,0 +1,23 @@
+# Test-only derivative of the actual owned proxy and normal Alpine/musl Host.
+# The coordinator verifies both base image IDs against the exact-head manifest
+# before/after building. Production dist and its dependencies stay byte-identical.
+ARG PROXY_BASE_IMAGE
+ARG HOST_BASE_IMAGE
+FROM ${HOST_BASE_IMAGE} AS pixels
+FROM ${PROXY_BASE_IMAGE}
+ARG SOURCE_HEAD
+LABEL org.evenfire.qa.subscription-image-source=${SOURCE_HEAD}
+USER 0:0
+COPY scripts/e2e/fixtures/subscription-image-provider.mjs \
+ scripts/e2e/fixtures/subscription-image-decoder.mjs \
+ scripts/e2e/fixtures/subscription-image-challenge.cjs /app/scripts/e2e/fixtures/
+COPY packages/grok-provider-attempt-contract/ /app/packages/grok-provider-attempt-contract/
+COPY --from=pixels /app/mcp-host/package.json /app/mcp-host/package.json
+# @napi-rs/canvas selects the Host's installed platform addon. No glibc Desktop
+# binary, package installation, lifecycle script or new project dependency.
+COPY --from=pixels /app/mcp-host/node_modules/@napi-rs/ /app/mcp-host/node_modules/@napi-rs/
+RUN mkdir -p /tmp/evenfire-vendor \
+ && chown 1001:1001 /tmp/evenfire-vendor && chmod 0700 /tmp/evenfire-vendor \
+ && chmod -R a+rX /app/scripts/e2e/fixtures /app/packages/grok-provider-attempt-contract /app/mcp-host \
+ && node -e "if(process.versions.node.split('.')[0]!=='24'||process.report.getReport().header.glibcVersionRuntime)process.exit(1);require('/app/scripts/e2e/fixtures/subscription-image-challenge.cjs').requirePixelRenderer()"
+USER 1001:1001
diff --git a/scripts/e2e/prepare-subscription-image-admission.mjs b/scripts/e2e/prepare-subscription-image-admission.mjs
index cdbfb8f26..e8acd914f 100644
--- a/scripts/e2e/prepare-subscription-image-admission.mjs
+++ b/scripts/e2e/prepare-subscription-image-admission.mjs
@@ -1,4 +1,6 @@
#!/usr/bin/env node
+// E2E_GUARDIAN_IPC_FLOW: host admission and private channel preparation only;
+// actual Desktop network transitions and business signals stay in the specs.
// Host-side orchestration for the subscription-image runner chain:
// export allowlisted source -> build pinned image -> create private container ->
// copy the in-container session observation -> seal docker inspect, stack marker,
@@ -85,11 +87,23 @@ function privateDirectory(directory) {
fs.chmodSync(directory, 0o700)
}
+export function validatePrepareScratch(directory) {
+ if (!path.isAbsolute(directory) || path.resolve(directory) !== directory ||
+ fs.realpathSync(path.dirname(directory)) !== path.dirname(directory)) refuse('PREPARE_SCRATCH_PATH')
+ if (!fs.existsSync(directory)) return
+ const stat = fs.lstatSync(directory)
+ if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid?.() ||
+ (stat.mode & 0o777) !== 0o700 || fs.readdirSync(directory).length !== 0) refuse('PREPARE_SCRATCH_NOT_FRESH')
+}
+
function writePrivateJson(filename, value) {
fs.writeFileSync(filename, JSON.stringify(value), { flag: 'wx', mode: 0o600 })
}
-export async function prepare(argv = process.argv.slice(2)) {
+export async function prepare(argv = process.argv.slice(2), privateFlow) {
+ if (privateFlow !== undefined && (!privateFlow || typeof privateFlow !== 'object' ||
+ !Array.isArray(privateFlow.bindings) || typeof privateFlow.frames !== 'function' ||
+ typeof privateFlow.close !== 'function')) refuse('PREPARE_PRIVATE_FLOW_INVALID')
const values = parseArgs(argv)
const suiteId = required(values, 'suite')
const mode = required(values, 'mode')
@@ -98,7 +112,7 @@ export async function prepare(argv = process.argv.slice(2)) {
const scratch = required(values, 'scratch')
const framesFile = required(values, 'frames')
const detector = required(values, 'red-detector')
- const bindingsFile = required(values, 'bindings')
+ const bindingsFile = privateFlow ? undefined : required(values, 'bindings')
const cacheRoot = values['cache-root'] ?? DEFAULT_CACHE_ROOT
if (!/^[a-z0-9][a-z0-9-]{0,62}$/.test(profile ?? '') || profile === 'clerum-test')
refuse('PREPARE_PROFILE_REQUIRED')
@@ -107,11 +121,12 @@ export async function prepare(argv = process.argv.slice(2)) {
if (!suite.modes.includes(mode)) refuse('SUITE_MODE_NOT_ADMITTED')
// Login frames travel through the live private input channel, never a file.
if (framesFile !== '-') refuse('PREPARE_PRIVATE_INPUT_REQUIRES_STDIN')
- if (!path.isAbsolute(scratch) || !path.isAbsolute(bindingsFile) || !path.isAbsolute(detector))
+ if (!path.isAbsolute(scratch) || (!privateFlow && !path.isAbsolute(bindingsFile)) || !path.isAbsolute(detector))
refuse('PREPARE_ABSOLUTE_PATHS')
if (scratch.startsWith(`${SCRIPT_ROOT}/`) || scratch === SCRIPT_ROOT)
refuse('PREPARE_SCRATCH_INSIDE_REPO')
- for (const filename of [bindingsFile]) {
+ validatePrepareScratch(scratch)
+ for (const filename of bindingsFile ? [bindingsFile] : []) {
const stat = fs.lstatSync(filename)
const parent = path.dirname(filename)
if (
@@ -136,6 +151,7 @@ export async function prepare(argv = process.argv.slice(2)) {
const source = exportSourceContext(SCRIPT_ROOT, contextDir, detector)
if (source.gitHead !== run('git', ['-C', SCRIPT_ROOT, 'rev-parse', 'HEAD']).trim())
refuse('PREPARE_SOURCE_RACE')
+ await privateFlow?.beforeBuild?.({ contextDir, source })
// Third-party artifact (separate allowlist and receipt; never part of the Git
// source manifest). The host fetch stays behind the local sfw wrapper.
const sfwAsset = sfwFreeAsset(values.arch ?? process.arch)
@@ -307,7 +323,7 @@ export async function prepare(argv = process.argv.slice(2)) {
}
const sealedFilename = path.join(scratch, 'subscription-image-source.json')
run('docker', ['cp', `${containerName}:/opt/evenfire/subscription-image-source.json`, sealedFilename])
- const bindings = JSON.parse(fs.readFileSync(bindingsFile, 'utf8'))
+ const bindings = privateFlow?.bindings ?? JSON.parse(fs.readFileSync(bindingsFile, 'utf8'))
const transports = Object.fromEntries(
Object.entries(PORT_FORWARD_SERVICES).map(([kind, expected]) => [
kind,
@@ -349,6 +365,7 @@ export async function prepare(argv = process.argv.slice(2)) {
},
bindings,
transports,
+ vendorSources: privateFlow?.vendorSources,
})
admission.volume = {
name: volumeName,
@@ -356,6 +373,14 @@ export async function prepare(argv = process.argv.slice(2)) {
createdAt: new Date(Date.parse(volumeInspect.CreatedAt)).toISOString(),
inspectSha256: digest(JSON.stringify(volumeInspect)),
}
+ if (privateFlow?.configure) {
+ const additions = await privateFlow.configure(admission, { containerName })
+ if (additions !== undefined) {
+ if (!additions || typeof additions !== 'object' ||
+ Object.keys(additions).some(key => key !== 'pressureMetadata')) refuse('PREPARE_ADMISSION_EXTENSION_INVALID')
+ Object.assign(admission, additions)
+ }
+ }
writePrivateJson(path.join(admissionDir, 'main-admission.json'), admission)
const framesPayload = [
'main-admission.json',
@@ -398,15 +423,27 @@ export async function prepare(argv = process.argv.slice(2)) {
attached.kill('SIGTERM')
}
attached.stdin.on('error', onInputError)
- process.stdin.once('error', onInputError)
- process.stdin.pipe(attached.stdin)
+ const input = privateFlow ? await privateFlow.frames(admission, { containerName }) : process.stdin
+ input.once('error', onInputError)
+ input.pipe(attached.stdin)
+ const interrupted = () => { inputFailed = true; attached.kill('SIGTERM') }
+ process.once('SIGTERM', interrupted)
+ process.once('SIGINT', interrupted)
+ const deadline = setTimeout(interrupted, 3_600_000)
let result
try {
- result = await new Promise(resolve => attached.once('exit', (code, signal) => resolve({ code, signal })))
+ result = await new Promise((resolve, reject) => {
+ attached.once('error', () => reject(new RunnerAdmissionError('PREPARE_ATTACH_FAILED')))
+ attached.once('close', (code, signal) => resolve({ code, signal }))
+ })
} finally {
- process.stdin.unpipe(attached.stdin)
- process.stdin.off('error', onInputError)
- process.stdin.pause()
+ clearTimeout(deadline)
+ process.off('SIGTERM', interrupted)
+ process.off('SIGINT', interrupted)
+ input.unpipe(attached.stdin)
+ input.off('error', onInputError)
+ input.pause()
+ await privateFlow?.close()
}
const status = run('docker', ['wait', containerName], { timeout: 30_000 }).trim()
for (const name of ['runner.json', 'journey-result.json', path.basename(observationFile)]) {
@@ -449,6 +486,16 @@ export async function prepare(argv = process.argv.slice(2)) {
}
}
throw err
+ } finally {
+ await privateFlow?.close()
+ if (values.keep !== 'true') {
+ for (const args of [['rm', '-f', containerName], ['volume', 'rm', '-f', volumeName]]) {
+ try { run('docker', args, { timeout: 60_000 }) } catch {
+ // Restoration of the leased stack belongs to the coordinator. An
+ // absent fresh container/volume is harmless; never touch other names.
+ }
+ }
+ }
}
}
diff --git a/scripts/e2e/prepare-subscription-remaining-fixtures.mjs b/scripts/e2e/prepare-subscription-remaining-fixtures.mjs
index 10a7384fe..dc7496e22 100644
--- a/scripts/e2e/prepare-subscription-remaining-fixtures.mjs
+++ b/scripts/e2e/prepare-subscription-remaining-fixtures.mjs
@@ -67,7 +67,9 @@ function canonicalReceipt(receipt) {
} else {
fixtures[provider] = { ...common, controlApiPodUid: raw.controlApiPodUid,
controlApiImageId: raw.controlApiImageId, maxInFlight: raw.maxInFlight,
- readDeadlineMs: raw.readDeadlineMs, pressure: { receiptFile: raw.pressure?.receiptFile },
+ readDeadlineMs: raw.readDeadlineMs, pressure: { receiptFile: raw.pressure?.receiptFile,
+ workDeadlineMs: raw.pressure?.workDeadlineMs, closeGraceMs: raw.pressure?.closeGraceMs,
+ commandDeadlineMs: raw.pressure?.commandDeadlineMs },
fallback: { provider: raw.fallback?.provider, modelId: raw.fallback?.modelId } }
}
}
diff --git a/scripts/e2e/prepare-subscription-remaining-fixtures.prepare.mjs b/scripts/e2e/prepare-subscription-remaining-fixtures.prepare.mjs
index ebd31cf3b..163b18e1b 100644
--- a/scripts/e2e/prepare-subscription-remaining-fixtures.prepare.mjs
+++ b/scripts/e2e/prepare-subscription-remaining-fixtures.prepare.mjs
@@ -44,18 +44,20 @@ export async function prepareRemainingFixtures({ admission, runRoot, hostBinding
fixtures[binding.provider] = { ...common, folderNames: result.folderNames, files }
evidence[binding.provider] = result.evidence
} else {
- const other = admission.bindings.find(item => item.provider !== binding.provider)
const configured = actual[0].fallback
if (!pressure || pressure.profile !== admission.profile || pressure.context !== admission.context ||
pressure.sourceManifestSha256 !== admission.sourceManifestSha256 ||
- !pressure.hostRefs?.includes(binding.hostRef) || configured?.provider !== other.provider ||
- configured?.modelId !== other.modelId ||
+ !pressure.hostRefs?.includes(binding.hostRef) || configured?.provider !== binding.provider ||
+ typeof configured?.modelId !== 'string' || configured.modelId === binding.modelId ||
+ !actual[0].catalogueModels?.includes(configured.modelId) ||
!runtime.fallbacks.some(item => item.provider === configured.provider && item.model === configured.modelId)) {
throw new Error('REMAINING_PREPARE_ACTUAL_PRESSURE_AND_FALLBACK_REQUIRED')
}
fixtures[binding.provider] = { ...common, controlApiPodUid: pressure.podUid,
controlApiImageId: pressure.imageId, maxInFlight: pressure.maxInFlight,
- readDeadlineMs: pressure.readDeadlineMs, pressure: { receiptFile: pressure.receiptFile },
+ readDeadlineMs: pressure.readDeadlineMs, pressure: { receiptFile: pressure.receiptFile,
+ workDeadlineMs: pressure.workDeadlineMs, closeGraceMs: pressure.closeGraceMs,
+ commandDeadlineMs: pressure.commandDeadlineMs },
fallback: { provider: configured.provider, modelId: configured.modelId } }
}
const fresh = observeQaHostRuntime(target)
diff --git a/scripts/tests/control-api-qa-runtime.test.mjs b/scripts/tests/control-api-qa-runtime.test.mjs
new file mode 100644
index 000000000..25a6104a2
--- /dev/null
+++ b/scripts/tests/control-api-qa-runtime.test.mjs
@@ -0,0 +1,116 @@
+// Hermetic controls for the actual CAS/wrapper functions. No cluster, vendor,
+// credentials or physical measurement is used or claimed by these tests.
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import fs from 'node:fs'
+import vm from 'node:vm'
+import { stripTypeScriptTypes } from 'node:module'
+import { isDeepStrictEqual } from 'node:util'
+import { CATALOG_CRON_KEYS, snapshotDeployment, deploymentMutationPatch, openControlApiCatalogIsolation,
+ withControlApiFixtureRuntime, catalogRestorationSafe } from '../e2e/coordinate-subscription-image-journeys.mjs'
+import { revokeMemoryFixtures } from './lib/control-api-authorize-memory-seeder.ts'
+
+const id = number => `00000000-0000-4000-8000-${String(number).padStart(12, '0')}`
+const image = `sha256:${'a'.repeat(64)}`
+function patchObject(object, operations) {
+ for (const operation of operations) {
+ const names = operation.path.slice(1).split('/'), key = names.pop(), target = names.reduce((current, name) => current[name], object)
+ if (operation.op === 'test') assert.deepEqual(target[key], operation.value)
+ else if (operation.op === 'remove') Array.isArray(target) ? target.splice(Number(key), 1) : delete target[key]
+ else if (key === '-') target.push(operation.value)
+ else target[key] = structuredClone(operation.value)
+ }
+}
+function fixture({ seedFailure = false } = {}) {
+ const resource = { metadata: { uid: id(1), resourceVersion: '1' }, spec: { replicas: 1,
+ template: { spec: { containers: [{ name: 'control-api', image, env: [...CATALOG_CRON_KEYS.map(name => ({ name, value: 'true' })),
+ { name: 'UNRELATED', value: 'keep' }] }] } } } }
+ const events = [], recovery = []
+ const effective = () => Object.fromEntries(resource.spec.template.spec.containers[0].env.map(entry => [entry.name, entry.value]))
+ const context = { assert: (value, code) => assert(value, code), validImageId: value => value === image,
+ imageId: value => value, ROOT: '/unit-source', NAMESPACE: 'control-plane', CATALOG_CRON_KEYS, snapshotDeployment, deploymentMutationPatch,
+ getDeployment: () => structuredClone(resource), readyPod: () => ({ metadata: { name: 'unit-api' }, status: { containerStatuses: [{ name: 'control-api', imageID: image }] } }),
+ until: async check => check(), json: () => ({ llm: effective()[CATALOG_CRON_KEYS[0]] === 'true', subscription: effective()[CATALOG_CRON_KEYS[1]] === 'true' }),
+ patchDeployment: (_context, _name, patch) => { patchObject(resource, patch); events.push('cron-patch'); resource.metadata.resourceVersion = String(Number(resource.metadata.resourceVersion) + 1) },
+ command: () => '', catalogRestorationSafe,
+ restoreDeployments: async (_context, states) => {
+ for (const state of states) patchObject(resource, deploymentMutationPatch(resource, state.original, state.applied, true))
+ events.push('restore-original-crons')
+ },
+ seed: async (_options, _private, finalization) => {
+ assert(CATALOG_CRON_KEYS.every(name => effective()[name] === 'false')); events.push('seed-sees-false')
+ if (seedFailure) throw new Error('UNIT_SEED_FAILED')
+ if (finalization) { await finalization({ revokeMemory: async () => { events.push('memory-revoked-verified'); return { verified: true } } });
+ return { kind: 'control-api-authorize-qa-finalization.v1', fixtureFinalization: { verified: true, state: 'revoked', grantsRevoked: 2, hostsDetached: 2 } } }
+ return { fixtures: {} }
+ },
+ releaseOptionsForQa: options => options,
+ writeCatalogRecovery: (...args) => { recovery.push(args); events.push('retain-crons-false') },
+ Error, Promise, Object, path: { dirname: () => '/unit-report', join: (...parts) => parts.join('/') },
+ }
+ const sandbox = vm.createContext(context)
+ context.openControlApiCatalogIsolation = vm.runInContext(`(${openControlApiCatalogIsolation.toString()})`, sandbox)
+ const wrapper = vm.runInContext(`(${withControlApiFixtureRuntime.toString()})`, sandbox)
+ const options = { profile: 'qa-unit', context: 'qa-unit', runId: 'pr806-memory-123456789abc', prepare: 'restart', prepareFixtures: true,
+ report: '/unit-report/seed.json', publicArguments: {} }
+ return { wrapper, options, events, recovery, resource, effective }
+}
+
+test('actual wrapper applies real cron keys false before seed and restores true only after final memory revocation', async () => {
+ const world = fixture()
+ await world.wrapper(world.options, {}, async runtime => {
+ assert.deepEqual(runtime.initialSnapshot.env, Object.fromEntries(CATALOG_CRON_KEYS.map(name => [name, 'true'])))
+ await runtime.run(world.options)
+ })
+ assert.deepEqual(world.events, ['cron-patch', 'seed-sees-false', 'seed-sees-false', 'memory-revoked-verified', 'restore-original-crons'])
+ assert(CATALOG_CRON_KEYS.every(name => world.effective()[name] === 'true'))
+ assert.equal(world.effective().UNRELATED, 'keep')
+})
+test('partial seed failure keeps actual crons false and records recovery; a zero-QA work failure restores originals', async () => {
+ const partial = fixture({ seedFailure: true })
+ await assert.rejects(partial.wrapper(partial.options, {}, runtime => runtime.run(partial.options)), /UNIT_SEED_FAILED/)
+ assert(CATALOG_CRON_KEYS.every(name => partial.effective()[name] === 'false'))
+ assert(!partial.events.includes('restore-original-crons')); assert.equal(partial.recovery.length, 1)
+ const zero = fixture()
+ await assert.rejects(zero.wrapper(zero.options, {}, async () => { throw new Error('UNIT_BEFORE_QA') }), /UNIT_BEFORE_QA/)
+ assert(CATALOG_CRON_KEYS.every(name => zero.effective()[name] === 'true'))
+ for (const state of ['created', 'created-or-unknown']) {
+ assert.equal(catalogRestorationSafe(state, []), false)
+ assert.equal(catalogRestorationSafe('revoked', [{ state }]), false)
+ }
+})
+
+test('real first-tick scheduler would call direct sync after restart; disabled actual config gate prevents scheduling', async () => {
+ const full = fs.readFileSync(new URL('../../control-api/src/services/subscriptionCatalogSyncCron.ts', import.meta.url), 'utf8')
+ const start = full.indexOf('let intervalHandle:'), source = stripTypeScriptTypes(full.slice(start)).replace(/^export /gm, '')
+ let callback, calls = 0
+ const context = vm.createContext({ Math: { random: () => 0, floor: Math.floor }, log: { info() {}, error() {} },
+ runSubscriptionCatalogSyncTick: async deps => { await deps.sync() },
+ setTimeout: fn => { callback = fn; return { unref() {} } }, setInterval: () => ({ unref() {} }), clearTimeout() {}, clearInterval() {} })
+ const startCron = vm.runInContext(source + '\nstartSubscriptionCatalogSyncCron', context)
+ const gate = value => { if (value === 'true') startCron({ sync: async () => { calls++ } }, 21600000) }
+ gate('false'); assert.equal(callback, undefined); assert.equal(calls, 0)
+ gate('true'); callback(); await Promise.resolve(); assert.equal(calls, 1)
+})
+
+test('final memory action detaches/revokes only two exact owned grants and preserves audit/GFS data', async () => {
+ const runId = 'pr806-memory-123456789abc', events = [], rows = new Map()
+ const prepared = { bindings: [0, 1].map(index => ({ hostRef: ['pr806-memory-grok-host', 'pr806-memory-grok-host-2'][index],
+ hostUid: id(10 + index), connectionKey: `${runId}-grok-${index + 1}`, connectionId: id(20 + index) })) }
+ const session = { request: async request => {
+ const binding = prepared.bindings.find(value => request.path.endsWith(value.hostRef))
+ if (request.method === 'GET') return { status: 200, json: { metadata: { uid: binding.hostUid, resourceVersion: '1',
+ labels: { 'evenfire.io/qa-memory-run': runId } }, spec: { model: { provider: 'grok-subscription', name: 'grok-4.6', connectionRef: binding.connectionKey } } } }
+ events.push('host-detached'); return { status: 200, json: { metadata: { uid: binding.hostUid }, spec: request.body.spec } }
+ } }
+ const prod = { db: { withTransaction: fn => fn({ query: async (sql, values) => {
+ assert(!/DELETE|UPDATE .*audit/i.test(sql)); const index = prepared.bindings.findIndex(value => value.connectionKey === values[0])
+ return { rows: [{ id: id(20 + index), credential_revision: 1, account_fingerprint: `qa-memory-${runId}-${index + 1}` }] }
+ } }) }, connection: { revokeGrokSubscriptionConnection: async (_tx, key) => {
+ events.push('grant-revoked'); const binding = prepared.bindings.find(value => value.connectionKey === key); rows.set(key, 'revoked'); return { id: binding.connectionId, status: 'revoked' }
+ } }, gateway: { llmAllowedModelsConfigMap: () => ({ materialize: async () => events.push('projection-published') }) } }
+ const result = await revokeMemoryFixtures({ prod, session, options: { runId }, prepared })
+ assert.equal(result.state, 'revoked'); assert.equal(result.grantsRevoked, 2); assert.equal(result.retainedAuditAndQaData, true)
+ assert.deepEqual(events, ['host-detached', 'grant-revoked', 'host-detached', 'grant-revoked', 'projection-published'])
+ assert.equal(rows.size, 2)
+})
diff --git a/scripts/tests/lib/control-api-authorize-memory-seeder.ts b/scripts/tests/lib/control-api-authorize-memory-seeder.ts
index 5c32f0a44..aacc1b4d5 100644
--- a/scripts/tests/lib/control-api-authorize-memory-seeder.ts
+++ b/scripts/tests/lib/control-api-authorize-memory-seeder.ts
@@ -7,6 +7,7 @@ import path from 'node:path'
import http from 'node:http'
import { createRequire } from 'node:module'
import { randomBytes, createHash } from 'node:crypto'
+import { isDeepStrictEqual } from 'node:util'
type SeedOptions = { runId: string; hostNamespace: string; operatorUser: string; operatorPassword?: string; cookie?: string }
const seedAssert = (condition: unknown, code: string) => { if (!condition) throw new Error(code) }
@@ -67,7 +68,11 @@ export class PrivateCookieSession {
seedAssert(me.status === 200 && me.json?.me?.username === this.#operator && me.json.me.role === 'admin', 'SEED_OPERATOR_IDENTITY_MISMATCH')
const link = await this.#raw('GET', `/api/v1/admin/control-admins/${me.json.me.id}/gfs-operator-link`)
seedAssert(link.status === 200, 'SEED_DESKTOP_OPERATOR_LINK_UNAVAILABLE')
- return { id: me.json.me.id, username: me.json.me.username, setupPerformed, ...validateDesktopOperatorLink(me.json.me.id, link.json) }
+ const identity = await this.#db.pool.query('SELECT email FROM control_admin_users WHERE id = $1', [me.json.me.id])
+ seedAssert(identity.rows.length === 1 && typeof identity.rows[0].email === 'string' &&
+ /^[^\s@]+@[^\s@]+$/.test(identity.rows[0].email), 'SEED_OPERATOR_EMAIL_UNPROVED')
+ return { id: me.json.me.id, username: me.json.me.username, email: identity.rows[0].email,
+ setupPerformed, ...validateDesktopOperatorLink(me.json.me.id, link.json) }
}
async request(input: { method: string; path: string; body?: unknown; binary?: boolean }) {
const value = await this.#raw(input.method, input.path, input.body, input.binary === true)
@@ -80,6 +85,205 @@ export function assertQaBudget(budget: any, hostRef: string, name: string): void
Number(budget.limit_amount) === 100 && Number(budget.max_task_amount) === 200 && Number(budget.min_start_amount) === 1 && budget.period === 'daily' && budget.timezone === 'UTC' &&
scope && Object.keys(scope).length === 3 && scope.host_ref?.length === 1 && scope.host_ref[0] === hostRef && scope.provider?.length === 1 && scope.provider[0] === 'grok-subscription' && scope.model?.length === 1 && scope.model[0] === 'grok-4.6', 'QA_DANGER_ZONE_BUDGET_MISMATCH')
}
+/** Planned QA identities only. Eligibility is proved from persisted catalogue
+ * rows, the materialized production projection and the actual Host later.
+ * One Host has one oauth-broker provider; its alternate uses the same grant.
+ */
+export function subscriptionImageSeedInputs(runId: string) {
+ seedAssert(/^subscription-image-[a-f0-9]{12}$/.test(runId), 'IMAGE_SEED_RUN_INVALID')
+ const suffix = runId.slice(-12)
+ return ['grok', 'codex'].map(kind => ({
+ provider: `${kind}-subscription`, connectionKey: `qa-image-${kind}-${suffix}`,
+ hostRef: `qa-image-${kind}-${suffix}`, hostLabel: `qa-image-${kind}-${suffix}`,
+ modelId: `qa-${kind}-image-${suffix}`, modelLabel: `QA ${kind} image ${suffix}`,
+ unsupportedModelId: `qa-${kind}-text-${suffix}`, unsupportedModelLabel: `QA ${kind} text ${suffix}`,
+ fallback: { provider: `${kind}-subscription`, modelId: `qa-${kind}-alternate-${suffix}` },
+ }))
+}
+
+export async function prepareSubscriptionImageBindings({ prod, session, options, prepared, runId, state }: any) {
+ seedAssert(prepared?.operatorDesktopUserId && prepared.operatorLink?.status === 'active' && state.length === 0,
+ 'IMAGE_SEED_PRIVATE_SESSION_REQUIRED')
+ const context = await session.request({ method: 'GET', path: '/api/v1/admin/contexts/context1' })
+ seedAssert(context.status === 200 && context.json?.metadata?.uid === prepared.context.uid &&
+ !context.json.metadata.deletionTimestamp && Array.isArray(context.json.spec?.mcpServers) &&
+ context.json.spec.mcpServers.length === 0, 'IMAGE_SEED_CONTEXT_CHANGED')
+ const planned = subscriptionImageSeedInputs(runId)
+ const result = []
+ for (const input of planned) {
+ const kind = input.provider.startsWith('grok') ? 'grok' : 'codex'
+ const service = kind === 'grok' ? prod.connection : prod.codexConnection
+ const fingerprint = `qa-image-${runId}-${kind}`
+ const connection = await prod.db.withTransaction(async (tx: any) => {
+ const found = await tx.query(`SELECT id FROM ${kind}_subscription_connections WHERE connection_key=$1`, [input.connectionKey])
+ seedAssert(found.rows.length === 0, 'IMAGE_SEED_CONNECTION_COLLISION')
+ // Opaque test-only material belongs to this isolated QA grant. A cached
+ // access token is necessary: a refresh would escape the external fixture
+ // boundary through Control API. This is never OAuth or G8 evidence.
+ const credential = { refreshToken: randomBytes(32).toString('base64url'),
+ accessToken: randomBytes(32).toString('base64url'),
+ accessTokenExpiresAt: new Date(Date.now() + 2 * 60 * 60 * 1000),
+ accountFingerprint: fingerprint, status: 'connected',
+ ...(kind === 'codex' ? { chatgptAccountId: `qa-image-${runId}` } : {}) }
+ const create = kind === 'grok' ? service.insertInitialGrokSubscriptionConnection : service.insertInitialCodexSubscriptionConnection
+ const row = await create(tx, prod.encryption.deriveOAuthEncryptionKey(prod.config.oauthEncryptionKey), credential, input.connectionKey)
+ const publish = kind === 'grok' ? service.recordGrokCatalogOutcome : service.recordCodexCatalogOutcome
+ const ready = await publish(tx, { connectionKey: input.connectionKey, catalogStatus: 'ready',
+ connectionStatus: 'connected', expectedCredentialRevision: 1, expectedCatalogRevision: 0 })
+ seedAssert(ready?.id === row.id && ready.catalogRevision === 1, 'IMAGE_SEED_CATALOG_REVISION_FAILED')
+ for (const model of [input.modelId, input.unsupportedModelId, input.fallback.modelId]) {
+ await tx.query(`INSERT INTO ${kind}_catalog_models (connection_id,model,enabled,source,discovered_at,last_seen_at,stale)
+ VALUES ($1,$2,true,'manual',NOW(),NOW(),false)`, [row.id, model])
+ }
+ return ready
+ })
+ const owned = { ...input, runId, connectionId: connection.id, hostNamespace: options.hostNamespace,
+ credentialRevision: connection.credentialRevision, catalogRevision: connection.catalogRevision }
+ state.push(owned)
+ const models = [input.modelId, input.unsupportedModelId, input.fallback.modelId]
+ for (const [index, model] of models.entries()) {
+ const capability = { state: index === 1 ? 'unsupported' : 'supported',
+ evidence: { source: 'curated', reference: `evidence:${runId}`, checkedAt: new Date().toISOString() } }
+ const body = { provider: input.provider, model, display_name: index === 0 ? input.modelLabel :
+ index === 1 ? input.unsupportedModelLabel : `QA ${kind} alternate ${runId.slice(-12)}`,
+ enabled: true, image_input: capability }
+ const created = await session.request({ method: 'POST', path: '/api/v1/admin/llm-models', body })
+ seedAssert(created.status === 201 && created.json?.provider === input.provider && created.json.model === model &&
+ created.json.enabled === true && isDeepStrictEqual(created.json.image_input, capability), 'IMAGE_SEED_MODEL_API_FAILED')
+ const durable = await session.request({ method: 'GET', path: `/api/v1/admin/llm-models/${created.json.id}` })
+ seedAssert(durable.status === 200 && isDeepStrictEqual(durable.json.image_input, capability), 'IMAGE_SEED_MODEL_NOT_DURABLE')
+ }
+ await prod.gateway.llmAllowedModelsConfigMap().materialize()
+ const spec = { host: input.hostRef, contextRef: 'context1', desktop: { x11: true, browser: false },
+ model: { provider: input.provider, name: input.modelId, connectionRef: input.connectionKey },
+ allowedModels: models.map(model => ({ provider: input.provider, model })),
+ llmPolicy: { fallbacks: [{ provider: input.provider, model: input.fallback.modelId }] } }
+ const created = await session.request({ method: 'POST', path: '/api/v1/admin/hosts',
+ body: { metadata: { name: input.hostRef, labels: { 'evenfire.io/qa-image-run': runId } }, spec } })
+ seedAssert(created.status === 201 && /^[a-f0-9-]{36}$/.test(created.json?.metadata?.uid) &&
+ created.json.metadata.namespace === options.hostNamespace &&
+ created.json.metadata.labels?.['evenfire.io/qa-image-run'] === runId &&
+ isDeepStrictEqual(created.json.spec.model, spec.model) && isDeepStrictEqual(created.json.spec.allowedModels, spec.allowedModels) &&
+ isDeepStrictEqual(created.json.spec.llmPolicy?.fallbacks, spec.llmPolicy.fallbacks), 'IMAGE_SEED_ACTUAL_HOST_FAILED')
+ Object.assign(owned, { hostUid: created.json.metadata.uid, runId, spec })
+ const durable = await session.request({ method: 'GET', path: `/api/v1/admin/hosts/${input.hostRef}` })
+ seedAssert(durable.status === 200 && durable.json?.metadata?.uid === owned.hostUid &&
+ isDeepStrictEqual(durable.json.spec.model, spec.model) && isDeepStrictEqual(durable.json.spec.llmPolicy?.fallbacks, spec.llmPolicy.fallbacks),
+ 'IMAGE_SEED_HOST_NOT_DURABLE')
+ result.push({ ...input, hostNamespace: options.hostNamespace, hostUid: owned.hostUid,
+ connectionId: connection.id, credentialRevision: connection.credentialRevision, catalogRevision: connection.catalogRevision,
+ catalogueModels: models, catalogProjectionPublished: true })
+ }
+ return { kind: 'evenfire-subscription-image-qa-bindings-v1', runId, bindings: result,
+ operatorDesktopUserId: prepared.operatorDesktopUserId, loginEmail: prepared.operatorEmail,
+ credentialState: 'opaque-qa-not-real-G8', vendorCronsDisabled: true, upstreamDispatch: 'NOT_RUN' }
+}
+
+export async function revokeSubscriptionImageBindings({ prod, session, state }: any) {
+ for (const binding of state) {
+ if (binding.hostUid) {
+ const current = await session.request({ method: 'GET', path: `/api/v1/admin/hosts/${binding.hostRef}` })
+ seedAssert(current.status === 200 && current.json?.metadata?.uid === binding.hostUid &&
+ current.json.metadata.labels?.['evenfire.io/qa-image-run'] === binding.runId &&
+ isDeepStrictEqual(current.json.spec.model, binding.spec.model) &&
+ isDeepStrictEqual(current.json.spec.llmPolicy?.fallbacks, binding.spec.llmPolicy.fallbacks), 'IMAGE_SEED_HOST_RESTORE_CONFLICT')
+ const detached = await session.request({ method: 'PUT', path: `/api/v1/admin/hosts/${binding.hostRef}`,
+ body: { metadata: { uid: binding.hostUid, resourceVersion: current.json.metadata.resourceVersion,
+ labels: current.json.metadata.labels }, spec: { ...current.json.spec,
+ model: { ...current.json.spec.model, connectionRef: 'unassigned' }, llmPolicy: { fallbacks: [] } } } })
+ seedAssert(detached.status === 200 && detached.json?.metadata?.uid === binding.hostUid &&
+ isDeepStrictEqual(detached.json.spec?.model, { ...binding.spec.model, connectionRef: 'unassigned' }) &&
+ Array.isArray(detached.json.spec?.llmPolicy?.fallbacks) && detached.json.spec.llmPolicy.fallbacks.length === 0,
+ 'IMAGE_SEED_HOST_DETACH_FAILED')
+ }
+ const kind = binding.provider.startsWith('grok') ? 'grok' : 'codex'
+ await prod.db.withTransaction(async (tx: any) => {
+ const current = await tx.query(`SELECT id::text,credential_revision,account_fingerprint FROM ${kind}_subscription_connections
+ WHERE connection_key=$1 AND revoked_at IS NULL FOR UPDATE`, [binding.connectionKey])
+ seedAssert(current.rows.length === 1 && current.rows[0].id === binding.connectionId &&
+ Number(current.rows[0].credential_revision) === 1 && current.rows[0].account_fingerprint === `qa-image-${binding.runId}-${kind}`,
+ 'IMAGE_SEED_GRANT_RESTORE_CONFLICT')
+ const service = kind === 'grok' ? prod.connection : prod.codexConnection
+ const revoke = kind === 'grok' ? service.revokeGrokSubscriptionConnection : service.revokeCodexSubscriptionConnection
+ const value = await revoke(tx, binding.connectionKey)
+ seedAssert(value?.id === binding.connectionId && value.status === 'revoked', 'IMAGE_SEED_GRANT_REVOKE_FAILED')
+ })
+ }
+ await prod.gateway.llmAllowedModelsConfigMap().materialize()
+ return { verified: true, grantsRevoked: state.length, hostsDetached: state.filter((value: any) => value.hostUid).length,
+ retainedQaData: true, realOAuthOrG8: 'NOT_RUN' }
+}
+/** Resume the retained memory fixtures, not a prior image run's detached and
+ * revoked QA bindings. A later image suite creates a fresh qa-image run below.
+ */
+export async function resumeMemoryFixtures({ prod, session, options, expected, operator }: any) {
+ seedAssert(/^pr806-memory-[a-f0-9]{12}$/.test(options.runId) && expected?.fixtureCredentialState === 'opaque-qa-not-real-G8' &&
+ expected.bindings?.length === 2, 'SEED_RESUME_RECEIPT_INVALID')
+ seedAssert(operator.id === expected.operatorId && operator.desktopUserId === expected.operatorDesktopUserId &&
+ operator.username === expected.operatorUsername, 'SEED_RESUME_OPERATOR_CHANGED')
+ const request = async (route: string) => session.request({ method: 'GET', path: route })
+ const context = await request('/api/v1/admin/contexts/context1')
+ seedAssert(context.status === 200 && context.json?.metadata?.uid === expected.context.uid &&
+ !context.json.metadata.deletionTimestamp && Array.isArray(context.json.spec?.mcpServers) && context.json.spec.mcpServers.length === 0,
+ 'SEED_RESUME_CONTEXT_CHANGED')
+ for (const [index, binding] of expected.bindings.entries()) {
+ seedAssert(binding.hostRef === ['pr806-memory-grok-host', 'pr806-memory-grok-host-2'][index] &&
+ binding.connectionKey === `${options.runId}-grok-${index + 1}`, 'SEED_RESUME_BINDING_FOREIGN')
+ const host = await request(`/api/v1/admin/hosts/${binding.hostRef}`)
+ seedAssert(host.status === 200 && host.json?.metadata?.uid === binding.hostUid &&
+ host.json.metadata.namespace === options.hostNamespace && host.json.metadata.labels?.['evenfire.io/qa-memory-run'] === options.runId &&
+ host.json.spec.contextRef === 'context1' && host.json.spec.model?.provider === 'grok-subscription' &&
+ host.json.spec.model.name === 'grok-4.6' && host.json.spec.model.connectionRef === binding.connectionKey, 'SEED_RESUME_HOST_CHANGED')
+ const grant = await prod.connection.getSafeGrokSubscriptionConnection(prod.db.pool, binding.connectionKey)
+ seedAssert(grant?.id === binding.connectionId && grant.status === 'connected' && grant.catalogStatus === 'ready' &&
+ grant.credentialRevision === 1 && grant.catalogRevision === 1 &&
+ grant.accountFingerprint === `qa-memory-${options.runId}-${index + 1}`, 'SEED_RESUME_GRANT_CHANGED')
+ const budget = await request(`/api/v1/admin/budgets/${binding.budgetId}`)
+ seedAssert(budget.status === 200, 'SEED_RESUME_BUDGET_MISSING'); assertQaBudget(budget.json, binding.hostRef, binding.budgetName)
+ }
+ const directory = await request(`/api/v1/gfs/proxy/v1/resources/${expected.gfs.parentRid}`)
+ seedAssert(directory.status === 200 && directory.json?.data?.resourceId === expected.gfs.parentResourceId &&
+ directory.json.data.kind === 'directory' && directory.json.data.name === options.runId, 'SEED_RESUME_GFS_CHANGED')
+ return { ...expected, operatorEmail: operator.email }
+}
+
+/** Final scope release, after all suites/cgroup windows. Keep audit/budget/GFS
+ * rows, but detach and revoke the two exact memory QA bindings before crons can
+ * resume. This makes later reuse fail closed rather than spending opaque QA.
+ */
+export async function revokeMemoryFixtures({ prod, session, options, prepared }: any) {
+ seedAssert(prepared?.bindings?.length === 2, 'MEMORY_QA_RELEASE_RECEIPT_REQUIRED')
+ for (const [index, binding] of prepared.bindings.entries()) {
+ seedAssert(binding.hostRef === ['pr806-memory-grok-host', 'pr806-memory-grok-host-2'][index] &&
+ binding.connectionKey === `${options.runId}-grok-${index + 1}`, 'MEMORY_QA_RELEASE_FOREIGN_BINDING')
+ const current = await session.request({ method: 'GET', path: `/api/v1/admin/hosts/${binding.hostRef}` })
+ const expected = { provider: 'grok-subscription', name: 'grok-4.6', connectionRef: binding.connectionKey }
+ seedAssert(current.status === 200 && current.json?.metadata?.uid === binding.hostUid &&
+ current.json.metadata.labels?.['evenfire.io/qa-memory-run'] === options.runId &&
+ isDeepStrictEqual(current.json.spec.model, expected), 'MEMORY_QA_RELEASE_HOST_CHANGED')
+ const detached = await session.request({ method: 'PUT', path: `/api/v1/admin/hosts/${binding.hostRef}`,
+ body: { metadata: { uid: binding.hostUid, resourceVersion: current.json.metadata.resourceVersion, labels: current.json.metadata.labels },
+ spec: { ...current.json.spec, model: { ...expected, connectionRef: 'unassigned' }, llmPolicy: { fallbacks: [] } } } })
+ seedAssert(detached.status === 200 && detached.json?.metadata?.uid === binding.hostUid &&
+ isDeepStrictEqual(detached.json.spec?.model, { ...expected, connectionRef: 'unassigned' }) &&
+ Array.isArray(detached.json.spec?.llmPolicy?.fallbacks) && detached.json.spec.llmPolicy.fallbacks.length === 0,
+ 'MEMORY_QA_RELEASE_DETACH_UNPROVED')
+ await prod.db.withTransaction(async (tx: any) => {
+ const observed = await tx.query(`SELECT id::text,credential_revision,account_fingerprint FROM grok_subscription_connections
+ WHERE connection_key=$1 AND revoked_at IS NULL FOR UPDATE`, [binding.connectionKey])
+ seedAssert(observed.rows.length === 1 && observed.rows[0].id === binding.connectionId &&
+ Number(observed.rows[0].credential_revision) === 1 &&
+ observed.rows[0].account_fingerprint === `qa-memory-${options.runId}-${index + 1}`, 'MEMORY_QA_RELEASE_GRANT_CHANGED')
+ const revoked = await prod.connection.revokeGrokSubscriptionConnection(tx, binding.connectionKey)
+ seedAssert(revoked?.id === binding.connectionId && revoked.status === 'revoked', 'MEMORY_QA_RELEASE_GRANT_UNPROVED')
+ })
+ }
+ await prod.gateway.llmAllowedModelsConfigMap().materialize()
+ return { verified: true, state: 'revoked', grantsRevoked: 2, hostsDetached: 2,
+ connectionIds: prepared.bindings.map((binding: any) => binding.connectionId),
+ hostUids: prepared.bindings.map((binding: any) => binding.hostUid), retainedAuditAndQaData: true }
+}
+
export async function runSeedCompanion(actions: { prepareGfsImages?: (input: any) => Promise<unknown> } = {}): Promise<void> {
const output = process.stdout.write.bind(process.stdout)
process.stdout.write = () => true; process.stderr.write = () => true
@@ -88,11 +292,27 @@ export async function runSeedCompanion(actions: { prepareGfsImages?: (input: any
const require = createRequire(path.join(process.cwd(), 'package.json'))
let prod: any, options: SeedOptions, phase = 'private-input', input = Buffer.alloc(0)
let session: PrivateCookieSession
+ let prepared: any
+ const imageState: any[] = []
async function request(method: string, route: string, body?: unknown): Promise<any> { const value = await session.request({ method, path: route, body }); return { status: value.status, body: value.json } }
+ async function resume(expected: any) {
+ seedAssert(/^pr806-memory-[a-f0-9]{12}$/.test(options.runId) && expected?.fixtureCredentialState === 'opaque-qa-not-real-G8' &&
+ expected.bindings?.length === 2, 'SEED_RESUME_RECEIPT_INVALID')
+ prod = { config: require('./dist/config.js').config, db: require('./dist/db.js'),
+ connection: require('./dist/services/grokSubscriptionConnection.js'), codexConnection: require('./dist/services/codexSubscriptionConnection.js'),
+ encryption: require('./dist/oauth/encryption.js'), gateway: new (require('./dist/k8s.js').K8sGateway)() }
+ seedAssert(prod.config.hostsNamespace === options.hostNamespace && prod.config.grokSubscriptionEnabled && prod.config.codexSubscriptionEnabled &&
+ prod.config.subscriptionCatalogSyncCronEnabled === false && prod.config.llmCatalogSyncCronEnabled === false, 'SEED_RESUME_RUNTIME_INVALID')
+ session = new PrivateCookieSession(prod.config, prod.db, options)
+ const operator = await session.authenticate()
+ options.operatorPassword = undefined; options.cookie = undefined
+ return resumeMemoryFixtures({ prod, session, options, expected, operator })
+ }
async function prepare() {
seedAssert(/^pr806-memory-[a-f0-9]{12}$/.test(options.runId), 'SEED_RUN_ID_INVALID')
prod = { config: require('./dist/config.js').config, db: require('./dist/db.js'),
- connection: require('./dist/services/grokSubscriptionConnection.js'), encryption: require('./dist/oauth/encryption.js'),
+ connection: require('./dist/services/grokSubscriptionConnection.js'),
+ codexConnection: require('./dist/services/codexSubscriptionConnection.js'), encryption: require('./dist/oauth/encryption.js'),
gateway: new (require('./dist/k8s.js').K8sGateway)() }
seedAssert(prod.config.hostsNamespace === options.hostNamespace && prod.config.grokSubscriptionEnabled && prod.config.codexSubscriptionEnabled, 'SEED_PROVIDER_OR_NAMESPACE_MISMATCH')
seedAssert(prod.config.subscriptionCatalogSyncCronEnabled === false && prod.config.llmCatalogSyncCronEnabled === false, 'SEED_VENDOR_CRON_MUST_BE_DISABLED')
@@ -173,7 +393,8 @@ export async function runSeedCompanion(actions: { prepareGfsImages?: (input: any
const parent = await request('GET', `/api/v1/gfs/proxy/v1/resources/${directory.body.data.rid}`)
seedAssert(parent.status === 200 && parent.body?.data?.resourceId === directory.body.data.resourceId && parent.body.data.name === options.runId && parent.body.data.parentResourceId === root.body.resourceId, 'SEED_GFS_DIRECTORY_NOT_DURABLE')
phase = 'complete'
- return { fixtureCredentialState: 'opaque-qa-not-real-G8', upstreamDispatch: 'NOT_RUN', operatorUsername: operator.username, operatorId: operator.id, operatorDesktopUserId: operator.desktopUserId,
+ return { fixtureCredentialState: 'opaque-qa-not-real-G8', upstreamDispatch: 'NOT_RUN', operatorUsername: operator.username, operatorEmail: operator.email,
+ operatorId: operator.id, operatorDesktopUserId: operator.desktopUserId,
operatorLink: { status: operator.gfsOperatorLinkStatus, generation: operator.generation, rowVersion: operator.rowVersion }, setupPerformed: operator.setupPerformed,
context: { name: 'context1', namespace: context.body.metadata.namespace, uid: context.body.metadata.uid, resourceVersion: context.body.metadata.resourceVersion, mcpServers: [] }, bindings,
gfs: { drive: 'main', rootRid: root.body.rid, rootResourceId: root.body.resourceId, parentRid: directory.body.data.rid, parentResourceId: directory.body.data.resourceId, name: options.runId, createStatus: 201, durableReadStatus: 200 },
@@ -185,25 +406,41 @@ export async function runSeedCompanion(actions: { prepareGfsImages?: (input: any
for await (const chunk of process.stdin) {
input = Buffer.concat([input, chunk]); seedAssert(input.length <= 4 * 1024 * 1024, 'SEED_PRIVATE_INPUT_BOUND')
const end = input.indexOf(10); if (end < 0) continue
- seedAssert(end === input.length - 1, 'SEED_ONE_PRIVATE_COMMAND_REQUIRED')
- const frame = JSON.parse(input.toString()); input = Buffer.alloc(0)
- seedAssert(['prepare', 'prepare-gfs-images'].includes(frame.kind) && typeof frame.callId === 'string', 'SEED_PRIVATE_COMMAND_INVALID')
- options = frame.options
+ for (let newline; (newline = input.indexOf(10)) >= 0;) {
+ const line = input.subarray(0, newline)
+ const frame = JSON.parse(line.toString()); line.fill(0); input = Buffer.from(input.subarray(newline + 1))
+ seedAssert(['prepare', 'resume', 'prepare-gfs-images', 'prepare-subscription-images', 'revoke-subscription-images', 'revoke-memory-fixtures'].includes(frame.kind) && typeof frame.callId === 'string', 'SEED_PRIVATE_COMMAND_INVALID')
handled = true
try {
- if (frame.kind === 'prepare') emit({ callId: frame.callId, data: await prepare() })
- else {
+ if (frame.kind === 'prepare' || frame.kind === 'resume') {
+ seedAssert(!session, 'SEED_PRIVATE_SESSION_ALREADY_PREPARED')
+ options = frame.options; prepared = frame.kind === 'prepare' ? await prepare() : await resume(frame.input?.fixtures)
+ emit({ callId: frame.callId, data: prepared })
+ } else if (frame.kind === 'prepare-subscription-images') {
+ seedAssert(session && prepared && !frame.options, 'IMAGE_SEED_PRIVATE_SESSION_REQUIRED')
+ emit({ callId: frame.callId, data: await prepareSubscriptionImageBindings({ prod, session, options, prepared,
+ runId: frame.input?.runId, state: imageState }) })
+ } else if (frame.kind === 'revoke-subscription-images') {
+ seedAssert(session && prepared && !frame.options, 'IMAGE_SEED_PRIVATE_SESSION_REQUIRED')
+ emit({ callId: frame.callId, data: await revokeSubscriptionImageBindings({ prod, session, state: imageState }) })
+ } else if (frame.kind === 'revoke-memory-fixtures') {
+ seedAssert(session && prepared && !frame.options, 'MEMORY_QA_RELEASE_PRIVATE_SESSION_REQUIRED')
+ emit({ callId: frame.callId, data: await revokeMemoryFixtures({ prod, session, options, prepared }) })
+ } else {
seedAssert(actions.prepareGfsImages, 'GFS_FIXTURE_ACTION_NOT_INSTALLED')
- prod = { config: require('./dist/config.js').config, db: require('./dist/db.js') }
- seedAssert(/^pr806-memory-[a-f0-9]{12}$/.test(options.runId) && options.hostNamespace === prod.config.hostsNamespace, 'GFS_FIXTURE_BINDING_INVALID')
- session = new PrivateCookieSession(prod.config, prod.db, options); await session.authenticate()
+ if (!session) {
+ options = frame.options; prod = { config: require('./dist/config.js').config, db: require('./dist/db.js') }
+ seedAssert(/^pr806-memory-[a-f0-9]{12}$/.test(options.runId) && options.hostNamespace === prod.config.hostsNamespace, 'GFS_FIXTURE_BINDING_INVALID')
+ session = new PrivateCookieSession(prod.config, prod.db, options); await session.authenticate()
+ } else seedAssert(!frame.options, 'SEED_PRIVATE_SESSION_CANNOT_BE_REBOUND')
emit({ callId: frame.callId, data: await actions.prepareGfsImages!({ ...frame.input, session }) })
}
}
catch (error: any) { emit({ callId: frame.callId, failed: true, code: /^[A-Z0-9_]+$/.test(error.message) ? error.message : 'SEED_REAL_RUNTIME_FAILED', phase }); process.exitCode = 1 }
- break
+ }
}
seedAssert(handled, 'SEED_PRIVATE_INPUT_MISSING')
+ seedAssert(input.length === 0, 'SEED_PRIVATE_PARTIAL_COMMAND')
} catch { emit({ fatal: true, code: 'SEED_PRIVATE_PROTOCOL_FAILED', phase }); process.exitCode = 1 }
finally { if (prod) await prod.db.pool.end().catch(() => {}); http.globalAgent.destroy() }
}
diff --git a/scripts/tests/lib/subscription-image-source-context.mjs b/scripts/tests/lib/subscription-image-source-context.mjs
index beeda91dd..7f63a9616 100644
--- a/scripts/tests/lib/subscription-image-source-context.mjs
+++ b/scripts/tests/lib/subscription-image-source-context.mjs
@@ -42,6 +42,7 @@ const fixedFiles = [
'scripts/e2e/prepare-subscription-remaining-fixtures.runtime.mjs',
'scripts/e2e/prepare-subscription-remaining-fixtures.prepare.mjs',
'scripts/e2e/fixtures/subscription-image-runner.Dockerfile',
+ 'scripts/e2e/fixtures/subscription-image-proxy.Dockerfile',
'scripts/e2e/fixtures/subscription-image-provider.mjs',
'scripts/e2e/fixtures/subscription-image-decoder.mjs',
'scripts/e2e/fixtures/subscription-image-challenge.cjs',
diff --git a/scripts/tests/measure-control-api-authorize-memory.mjs b/scripts/tests/measure-control-api-authorize-memory.mjs
index 1aaef827b..0e91b5edf 100644
--- a/scripts/tests/measure-control-api-authorize-memory.mjs
+++ b/scripts/tests/measure-control-api-authorize-memory.mjs
@@ -7,6 +7,7 @@ import { fileURLToPath } from 'node:url'
import { stripTypeScriptTypes } from 'node:module'
import { createHash } from 'node:crypto'
import { buildAuthorizeFixture, buildRejectedFixture, buildGfsFixture, SHAPES, MIB, sha256 } from './lib/control-api-authorize-memory-fixtures.mjs'
+import { pressureCommandDeadlineMs } from '../e2e/fixtures/subscription-image-admission-pressure.mjs'
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')
const NAMESPACE = 'control-plane', DEPLOYMENT = 'control-api', CONTAINER = 'control-api'
const sleepPoll = ms => new Promise(resolve => setTimeout(resolve, ms))
@@ -146,7 +147,10 @@ async function waitFor(check, predicate, timeoutMs, code) { const deadline = Dat
export async function openAuthorizeAdmissionPressure(channel, privateOptions) {
const baseline = await channel.call({ kind: 'hello', options: { ...privateOptions, pressureOnly: true } })
assert(baseline.pressureOnly === true && baseline.owner.inFlight === 0 && baseline.policy.maxInFlight === privateOptions.candidate.concurrency, 'PRESSURE_BASELINE_UNPROVED')
+ const commandDeadlineMs = pressureCommandDeadlineMs(baseline.policy)
+ channel.timeoutMs = commandDeadlineMs
const handles = []
+ let closing
const zeroBusiness = async () => {
const counts = await channel.call({ kind: 'counts' })
assert(counts.total === 0 && counts.tickets === 0 && counts.reservationRows === 0, 'PRESSURE_REACHED_BUSINESS_WORK')
@@ -165,15 +169,26 @@ export async function openAuthorizeAdmissionPressure(channel, privateOptions) {
return value
}
return {
+ policy: Object.freeze({ ...baseline.policy }), commandDeadlineMs,
async hold() {
- assert(handles.length === 0, 'PRESSURE_ALREADY_HELD')
+ assert(!closing && handles.length === 0, 'PRESSURE_ALREADY_HELD_OR_CLOSING')
for (let index = 0; index < baseline.policy.maxInFlight; index++) {
+ assert(!closing, 'PRESSURE_CLOSING')
const handle = await channel.open({ requestId: `${privateOptions.runId}-pressure-${index + 1}`, route: 'authorize', hostRef: privateOptions.bindings[index % privateOptions.bindings.length].hostRef, length: 35 * MIB - 4096 })
handles.push(handle); await channel.call({ kind: 'write', requestId: handle.requestId, length: 1 }, Buffer.from('{'))
}
return waitFor(owners, value => value.inFlight === baseline.policy.maxInFlight && value.serverReads.reads.length === handles.length && value.serverReads.reads.every(read => handles.some(handle => handle.requestId === read.requestId) && read.receivedBodyBytes > 0 && read.complete === false), baseline.policy.readDeadlineMs, 'AUTHENTICATED_SERVER_BODY_HOLD_UNOBSERVED')
}, owners, release,
- async close() { if (handles.length) await release(); return channel.stop() },
+ close() {
+ return closing ??= (async () => {
+ const failures = []
+ try { if (handles.length) await release() } catch (error) { failures.push(error) }
+ let producer
+ try { producer = await channel.stop() } catch (error) { failures.push(error) }
+ if (failures.length) throw new AggregateError(failures, 'PRESSURE_CLOSE_FAILED')
+ return producer
+ })()
+ },
}
}
export function validateReceipt(report) {
@@ -196,7 +211,9 @@ export function validateReceipt(report) {
}
return true
}
-export async function run(options, privateSeedOptions) {
+export async function run(options, privateSeedOptions, privateFixtureWork) {
+ assert(privateFixtureWork === undefined || ((options.prepareFixtures === true || options.fixtureBinding) &&
+ typeof privateFixtureWork === 'function'), 'PRIVATE_FIXTURE_WORK_INVALID')
assert(privateSeedOptions === undefined || (privateSeedOptions && typeof privateSeedOptions === 'object' && !Array.isArray(privateSeedOptions) && Object.keys(privateSeedOptions).every(key => ['cookie', 'operatorPassword'].includes(key))), 'PRIVATE_SEED_ARGUMENT_INVALID')
for (const field of ['cookie', 'operatorPassword', 'password', 'privateSeedOptions', 'dsn', 'privateKey']) assert(!(field in options) && !(field in (options.publicArguments ?? {})), 'PRIVATE_MATERIAL_CANNOT_ENTER_PUBLIC_OPTIONS')
// The long-lived caller retains its original object in RAM across preparation,
@@ -225,13 +242,13 @@ export async function run(options, privateSeedOptions) {
assert(options.report.startsWith(allowedReports + path.sep) && !fs.existsSync(options.report), 'FRESH_CANONICAL_REPORT_REQUIRED')
fs.mkdirSync(path.dirname(options.report), { recursive: true, mode: 0o700 }); assert(fs.realpathSync(path.dirname(options.report)) === path.dirname(options.report), 'REPORT_PARENT_SYMLINK')
if (options.fixtureBinding) assert(options.fixtureBinding.worktreeId === worktreeId && options.fixtureBinding.clusterFingerprint === marker.clusterFingerprint, 'FIXTURE_PROFILE_IDENTITY_CHANGED')
- const report = { kind: options.prepareFixtures ? 'control-api-authorize-memory-fixtures.v1' : 'control-api-authorize-memory.v1', status: 'running', startedAt: new Date().toISOString(), source: { head, worktreeId, policySha256: sha256(fs.readFileSync(path.join(ROOT, 'control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts'))), driverSha256: sha256(fs.readFileSync(fileURLToPath(import.meta.url))), companionBundleSha256: sha256(buildAuthorizeMemoryCompanionBundle()), clusterFingerprint: marker.clusterFingerprint, imagesGeneratedAt: manifest.generated }, options, runs: [], restoration: { verified: false } }
+ const report = { kind: options.prepareFixtures || privateFixtureWork ? 'control-api-authorize-memory-fixtures.v1' : 'control-api-authorize-memory.v1', status: 'running', startedAt: new Date().toISOString(), source: { head, worktreeId, policySha256: sha256(fs.readFileSync(path.join(ROOT, 'control-api/src/middleware/llmProviderAttemptAdmissionLimits.ts'))), driverSha256: sha256(fs.readFileSync(fileURLToPath(import.meta.url))), companionBundleSha256: sha256(buildAuthorizeMemoryCompanionBundle()), clusterFingerprint: marker.clusterFingerprint, imagesGeneratedAt: manifest.generated }, options, runs: [], restoration: { verified: false } }
const save = () => { const temporary = options.report + '.next'; fs.writeFileSync(temporary, JSON.stringify(report, null, 2), { flag: 'wx', mode: 0o600 }); fs.renameSync(temporary, options.report) }
const deployment = await json(['-n', NAMESPACE, 'get', 'deployment', DEPLOYMENT, '-o', 'json'])
assert(deployment.spec.replicas === 1, 'SINGLE_APP_REPLICA_REQUIRED')
const ci = deployment.spec.template.spec.containers.findIndex(container => container.name === CONTAINER)
assert(ci >= 0 && !deployment.spec.template.spec.hostNetwork, 'OWNED_CONTAINER_REQUIRED')
- if (options.prepareFixtures) {
+ if (options.prepareFixtures || privateFixtureWork) {
const ready = (await json(['-n', NAMESPACE, 'get', 'pods', '-l', 'app=control-api', '-o', 'json'])).items.filter(pod => !pod.metadata.deletionTimestamp && pod.status.phase === 'Running' && pod.status.containerStatuses?.some(item => item.name === CONTAINER && item.ready))
assert(ready.length === 1, 'SEED_READY_API_POD_AMBIGUOUS')
const pod = ready[0], status = pod.status.containerStatuses.find(item => item.name === CONTAINER), spec = pod.spec.containers.find(item => item.name === CONTAINER)
@@ -242,13 +259,42 @@ export async function run(options, privateSeedOptions) {
const imageId = status.imageID.match(/sha256:[a-f0-9]{64}$/)?.[0]
assert(imageId && imageId === (manifest.images[spec.image] ?? manifest.images[`docker.io/${spec.image}`]) && manifest.sourceRevisions?.[spec.image] === head, 'SEED_IMAGE_SOURCE_MISMATCH')
report.podUid = pod.metadata.uid; report.imageId = imageId; report.containerId = status.containerID
- report.source.seederBundleSha256 = sha256(buildAuthorizeMemorySeederBundle()); report.phase = 'real-fixture-preparation'; save()
- const child = spawn('kubectl', ['--context', options.context, '-n', NAMESPACE, 'exec', '-i', pod.metadata.name, '-c', CONTAINER, '--', 'env', '-u', 'NODE_OPTIONS', 'node', '--max-old-space-size=256', '--input-type=module', '-e', buildAuthorizeMemorySeederBundle()], { cwd: ROOT, env, stdio: ['pipe', 'pipe', 'pipe'] })
+ const { prepareGfsImages } = await import('../e2e/prepare-subscription-remaining-fixtures.gfs.mjs')
+ const seederBundle = buildAuthorizeMemorySeederBundle(privateFixtureWork ? prepareGfsImages : undefined)
+ report.source.seederBundleSha256 = sha256(seederBundle); report.phase = 'real-fixture-preparation'; save()
+ const child = spawn('kubectl', ['--context', options.context, '-n', NAMESPACE, 'exec', '-i', pod.metadata.name, '-c', CONTAINER, '--', 'env', '-u', 'NODE_OPTIONS', 'node', '--max-old-space-size=256', '--input-type=module', '-e', seederBundle], { cwd: ROOT, env, stdio: ['pipe', 'pipe', 'pipe'] })
const channel = new Channel(child, 120000)
try {
- report.fixtures = await channel.call({ kind: 'prepare', options: { runId: options.runId, hostNamespace: options.hostNamespace, operatorUser: options.operatorUser, ...privateMaterial } })
+ const resumed = !options.prepareFixtures && readOwnedJson(options.publicArguments['fixtures-receipt']).fixtures
+ report.fixtures = await channel.call({ kind: options.prepareFixtures ? 'prepare' : 'resume',
+ options: { runId: options.runId, hostNamespace: options.hostNamespace, operatorUser: options.operatorUser, ...privateMaterial },
+ ...(resumed ? { input: { fixtures: resumed } } : {}) })
+ if (privateFixtureWork) {
+ // Only the actual session's private command interface is passed out.
+ // No cookie/password, server module or mutable report is exposed.
+ await privateFixtureWork({ fixtures: report.fixtures,
+ apiPod: { name: pod.metadata.name, uid: pod.metadata.uid, imageId },
+ source: { head, worktreeId, clusterFingerprint: marker.clusterFingerprint },
+ prepareImages: runId => channel.call({ kind: 'prepare-subscription-images', input: { runId } }),
+ prepareGfs: input => channel.call({ kind: 'prepare-gfs-images', input }),
+ revokeImages: () => channel.call({ kind: 'revoke-subscription-images' }),
+ revokeMemory: async () => {
+ const result = await channel.call({ kind: 'revoke-memory-fixtures' })
+ assert(result.verified === true && result.state === 'revoked' && result.grantsRevoked === 2 && result.hostsDetached === 2,
+ 'MEMORY_QA_FINALIZATION_UNPROVED')
+ report.fixtureFinalization = result
+ return result
+ },
+ })
+ }
report.producer = await channel.stop(); report.finishedAt = new Date().toISOString(); report.status = 'complete'; report.phase = 'complete'
- validateFixtureReceipt(report, options); save(); return report
+ if (report.fixtureFinalization) {
+ report.kind = 'control-api-authorize-qa-finalization.v1'
+ report.releasedFixtureIdentities = report.fixtures.bindings.map(({ hostRef, hostUid, connectionKey, connectionId }) =>
+ ({ hostRef, hostUid, connectionKey, connectionId }))
+ delete report.fixtures
+ } else validateFixtureReceipt(report, options)
+ save(); return report
} catch {
report.status = 'failed'; report.failure = 'REAL_FIXTURE_PREPARATION_FAILED'; report.finishedAt = new Date().toISOString()
await channel.stop().catch(() => {}); report.producer = channel.exit ?? { code: null, signal: 'unknown' }; save(); throw new Error('SEED_FAILED_REPORT_SAVED')
diff --git a/scripts/tests/subscription-image-coordinator-regressions.test.mjs b/scripts/tests/subscription-image-coordinator-regressions.test.mjs
new file mode 100644
index 000000000..64d32134f
--- /dev/null
+++ b/scripts/tests/subscription-image-coordinator-regressions.test.mjs
@@ -0,0 +1,234 @@
+// Hermetic causal tests for the private coordinator. Synthetic API/IPC state
+// here is not physical profile, authentication, deployment or E2E evidence.
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import net from 'node:net'
+import { once } from 'node:events'
+import { prepareVendorEvidenceDirectory } from '../e2e/fixtures/subscription-image-provider.mjs'
+import { closePressureResources, withImageQaSession, qaRestorationTargets } from '../e2e/coordinate-subscription-image-journeys.mjs'
+import { openAdmissionPressure, pressureCommandDeadlineMs, verifyPressureMetadata } from '../e2e/fixtures/subscription-image-admission-pressure.mjs'
+import { openAuthorizeAdmissionPressure } from './measure-control-api-authorize-memory.mjs'
+import { prepareSubscriptionImageBindings, revokeSubscriptionImageBindings, resumeMemoryFixtures } from './lib/control-api-authorize-memory-seeder.ts'
+
+const uid = number => `00000000-0000-4000-8000-${String(number).padStart(12, '0')}`
+const policy = { maxInFlight: 1, readDeadlineMs: 10000, workDeadlineMs: 30000, closeGraceMs: 250 }
+const temporary = () => fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'pr806-coordinator-regression-')))
+
+test('runtime evidence directory survives an empty tmp mount and refuses a symlink or broad existing mode', () => {
+ const root = temporary()
+ try {
+ const directory = path.join(root, 'evenfire-vendor'), file = path.join(directory, 'run.json')
+ assert(!fs.existsSync(directory)); assert.equal(prepareVendorEvidenceDirectory(file), directory)
+ assert.equal(fs.lstatSync(directory).mode & 0o777, 0o700)
+ assert.equal(fs.lstatSync(directory).uid, process.getuid())
+ fs.chmodSync(directory, 0o755)
+ assert.throws(() => prepareVendorEvidenceDirectory(file), /private owned directory/)
+ assert.equal(fs.lstatSync(directory).mode & 0o777, 0o755)
+ const link = path.join(root, 'linked'); fs.symlinkSync(directory, link)
+ assert.throws(() => prepareVendorEvidenceDirectory(path.join(link, 'run.json')), /private owned directory/)
+ } finally { fs.rmSync(root, { recursive: true, force: true }) }
+})
+
+function pressureChannel({ releaseFails = false, stopFails = false } = {}) {
+ let held = false, resolveResult, stops = 0
+ return { get held() { return held }, get stops() { return stops },
+ async call(input) {
+ if (input.kind === 'hello') return { pressureOnly: true, owner: { inFlight: 0 }, serverPid: 4, policy }
+ if (input.kind === 'owners') return { pid: 4, instanceCount: 1, inFlight: held ? 1 : 0,
+ serverReads: { pid: 4, reads: held ? [{ requestId: 'pr806-memory-123456789abc-pressure-1', receivedBodyBytes: 1, complete: false }] : [] } }
+ if (input.kind === 'counts') return { total: 0, tickets: 0, reservationRows: 0 }
+ if (input.kind === 'close') {
+ if (releaseFails) throw new Error('UNIT_RELEASE_FAILED')
+ held = false; resolveResult({ transportClosed: true })
+ }
+ return {}
+ },
+ async open(input) { held = true; return { requestId: input.requestId, result: new Promise(resolve => { resolveResult = resolve }) } },
+ async stop() { stops++; held = false; resolveResult?.({ transportClosed: true });
+ if (stopFails) throw new Error('UNIT_STOP_FAILED'); return { code: 0 } },
+ }
+}
+const openPressure = channel => openAuthorizeAdmissionPressure(channel, { runId: 'pr806-memory-123456789abc',
+ candidate: { concurrency: 1 }, bindings: [{ hostRef: 'unit-host' }] })
+
+test('relay rejection still releases held native requests and stops Channel before aggregate failure', async () => {
+ const channel = pressureChannel(), pressure = await openPressure(channel)
+ await pressure.hold(); assert.equal(channel.held, true)
+ const relayError = new Error('UNIT_RELAY_FAILED')
+ await assert.rejects(closePressureResources({ close: async () => { throw relayError } }, pressure), error =>
+ error instanceof AggregateError && error.errors.includes(relayError))
+ assert.equal(channel.held, false); assert.equal(channel.stops, 1)
+ assert.equal(channel.timeoutMs, 40250); assert.deepEqual(pressure.policy, policy)
+})
+test('native release rejection cannot bypass Channel.stop and preserves both cleanup errors', async () => {
+ const channel = pressureChannel({ releaseFails: true, stopFails: true }), pressure = await openPressure(channel)
+ await pressure.hold()
+ await assert.rejects(pressure.close(), error => error instanceof AggregateError &&
+ error.errors.map(value => value.message).join(',') === 'UNIT_RELEASE_FAILED,UNIT_STOP_FAILED')
+ assert.equal(channel.held, false); assert.equal(channel.stops, 1)
+})
+
+test('policy-derived command window covers actual release bounds without an arbitrary caller override', () => {
+ assert.equal(pressureCommandDeadlineMs(policy), 40250)
+ const maximum = { readDeadlineMs: 60000, workDeadlineMs: 120000, closeGraceMs: 10000 }
+ assert.equal(pressureCommandDeadlineMs(maximum), 140000)
+ for (const patch of [{ readDeadlineMs: undefined }, { workDeadlineMs: 120001 }, { closeGraceMs: 10000 }])
+ assert.throws(() => pressureCommandDeadlineMs({ ...policy, ...patch }), /POLICY_INVALID/)
+})
+
+test('real private IPC client remains pending beyond old30s and accepts a policy-bounded release reply', async t => {
+ // macOS TMPDIR is longer than AF_UNIX's pathname bound; use the canonical
+ // short OS tmp root for this unit-only socket, as the Linux relay also does.
+ const root = fs.realpathSync(fs.mkdtempSync(path.join(fs.realpathSync('/tmp'), 'p806-')))
+ const socketPath = path.join(root, 'pressure.sock'), receiptFile = path.join(root, 'pressure.json')
+ const metadata = { kind: 'evenfire-subscription-image-pressure-metadata-v1', profile: 'qa-unit', context: 'qa-unit',
+ worktreeId: 'a'.repeat(40), sourceManifestSha256: 'b'.repeat(64), podUid: uid(1), imageId: `sha256:${'c'.repeat(64)}`,
+ pressureRunId: 'subscription-image-pressure-123456789abc', hostRefs: ['qa-a', 'qa-b'], ...policy,
+ commandDeadlineMs: pressureCommandDeadlineMs(policy), socketPath }
+ fs.writeFileSync(receiptFile, JSON.stringify(metadata), { mode: 0o600 })
+ const sockets = new Set(), observation = { ok: true, pressureRunId: metadata.pressureRunId, maxInFlight: 1,
+ owners: { baseline: 0, held: 0, drained: 0 }, counts: { sameRunAttempts: 0, sameRunTickets: 0, reservations: 0 },
+ pids: [4], inspector: { pid: 4, startTime: 'unit-process' } }
+ let releaseSocket, releaseSeen
+ const seen = new Promise(resolve => { releaseSeen = resolve })
+ const server = net.createServer(socket => {
+ sockets.add(socket); let buffer = ''
+ socket.on('data', chunk => {
+ buffer += chunk
+ for (let end; (end = buffer.indexOf('\n')) >= 0;) {
+ const frame = JSON.parse(buffer.slice(0, end)); buffer = buffer.slice(end + 1)
+ if (frame.command === 'release') { releaseSocket = socket; releaseSeen() }
+ else socket.write(JSON.stringify(observation) + '\n')
+ }
+ })
+ socket.on('close', () => sockets.delete(socket))
+ })
+ server.listen(socketPath); await once(server, 'listening')
+ t.mock.timers.enable({ apis: ['setTimeout'] })
+ let client
+ try {
+ client = await openAdmissionPressure({ receiptFile })
+ let settled = false
+ const response = client.release().then(value => { settled = true; return value })
+ await seen; t.mock.timers.tick(30001); await Promise.resolve()
+ assert.equal(settled, false)
+ releaseSocket.write(JSON.stringify(observation) + '\n')
+ assert.equal((await response).owners.drained, 0)
+ assert.throws(() => verifyPressureMetadata({ ...metadata, commandDeadlineMs: 30000 }), /METADATA_INVALID/)
+ } finally {
+ if (client) await client.close()
+ t.mock.timers.reset(); for (const socket of sockets) socket.destroy()
+ await new Promise(resolve => server.close(resolve)); fs.rmSync(root, { recursive: true, force: true })
+ }
+})
+
+function qaWorld({ residualFallback = false } = {}) {
+ const options = { runId: 'pr806-memory-123456789abc', hostNamespace: 'qa-hosts' }
+ const operator = { id: uid(1), desktopUserId: uid(2), username: `${options.runId}-operator`, email: '[email protected]' }
+ const memory = { fixtureCredentialState: 'opaque-qa-not-real-G8', operatorId: operator.id,
+ operatorDesktopUserId: operator.desktopUserId, operatorUsername: operator.username, operatorEmail: operator.email,
+ operatorLink: { status: 'active' }, context: { uid: uid(3) }, gfs: { parentRid: 'a'.repeat(32), parentResourceId: uid(4) }, bindings: [] }
+ const hosts = new Map(), grants = new Map(), budgets = new Map(), models = new Map()
+ for (const [index, hostRef] of ['pr806-memory-grok-host', 'pr806-memory-grok-host-2'].entries()) {
+ const connectionKey = `${options.runId}-grok-${index + 1}`, budgetName = `${options.runId}-${hostRef}-budget`
+ memory.bindings.push({ hostRef, hostUid: uid(10 + index), connectionKey, connectionId: uid(20 + index), budgetId: uid(30 + index), budgetName })
+ hosts.set(hostRef, { metadata: { uid: uid(10 + index), namespace: options.hostNamespace,
+ labels: { 'evenfire.io/qa-memory-run': options.runId } }, spec: { contextRef: 'context1',
+ model: { provider: 'grok-subscription', name: 'grok-4.6', connectionRef: connectionKey } } })
+ grants.set(connectionKey, { id: uid(20 + index), status: 'connected', catalogStatus: 'ready', credentialRevision: 1,
+ catalogRevision: 1, accountFingerprint: `qa-memory-${options.runId}-${index + 1}` })
+ budgets.set(uid(30 + index), { name: budgetName, enabled: true, scope: { host_ref: [hostRef], provider: ['grok-subscription'], model: ['grok-4.6'] },
+ unit: 'tokens', currency: null, enforcement: 'block', limit_amount: 100, max_task_amount: 200, min_start_amount: 1, period: 'daily', timezone: 'UTC' })
+ }
+ const session = { request: async request => {
+ if (request.path.endsWith('contexts/context1')) return { status: 200, json: { metadata: { uid: uid(3) }, spec: { mcpServers: [] } } }
+ if (request.path.includes('/budgets/')) return { status: 200, json: budgets.get(request.path.split('/').at(-1)) }
+ if (request.path.startsWith('/api/v1/gfs/')) return { status: 200, json: { data: { resourceId: uid(4), kind: 'directory', name: options.runId } } }
+ if (request.path.endsWith('llm-models') && request.method === 'POST') {
+ const row = { ...request.body, id: String(models.size + 1) }; models.set(row.id, row); return { status: 201, json: row }
+ }
+ if (request.path.includes('/llm-models/')) return { status: 200, json: models.get(request.path.split('/').at(-1)) }
+ if (request.method === 'POST' && request.path.endsWith('/hosts')) {
+ const row = { metadata: { ...request.body.metadata, uid: uid(100 + hosts.size), namespace: options.hostNamespace, resourceVersion: '1' }, spec: request.body.spec }
+ hosts.set(row.metadata.name, row); return { status: 201, json: row }
+ }
+ const name = request.path.split('/').at(-1), row = hosts.get(name)
+ if (request.method === 'PUT') {
+ const updated = { ...row, spec: structuredClone(request.body.spec) }
+ if (residualFallback) updated.spec.llmPolicy.fallbacks = structuredClone(row.spec.llmPolicy.fallbacks)
+ hosts.set(name, updated); return { status: 200, json: updated }
+ }
+ return { status: 200, json: row }
+ } }
+ const insert = async (_tx, _key, credential, key) => {
+ const row = { id: uid(200 + grants.size), status: 'connected', catalogStatus: 'ready', credentialRevision: 1,
+ catalogRevision: 0, accountFingerprint: credential.accountFingerprint }; grants.set(key, row); return row
+ }
+ const publish = async (_tx, input) => ({ ...grants.get(input.connectionKey), catalogRevision: 1 })
+ const revoke = async (_tx, key) => { const row = grants.get(key); row.status = 'revoked'; return row }
+ const prod = { config: {}, encryption: { deriveOAuthEncryptionKey: () => Buffer.alloc(32) },
+ db: { pool: {}, withTransaction: work => work({ query: async (sql, values) => ({ rows:
+ sql.includes('FOR UPDATE') ? [{ id: grants.get(values[0]).id, credential_revision: 1, account_fingerprint: grants.get(values[0]).accountFingerprint }] : [] }) }) },
+ gateway: { llmAllowedModelsConfigMap: () => ({ materialize: async () => {} }) },
+ connection: { insertInitialGrokSubscriptionConnection: insert, recordGrokCatalogOutcome: publish,
+ revokeGrokSubscriptionConnection: revoke, getSafeGrokSubscriptionConnection: async (_pool, key) => grants.get(key) },
+ codexConnection: { insertInitialCodexSubscriptionConnection: insert, recordCodexCatalogOutcome: publish, revokeCodexSubscriptionConnection: revoke } }
+ return { options, operator, memory, session, prod, hosts, grants }
+}
+
+test('detach refuses an API success retaining a QA fallback before it can claim verified revocation', async () => {
+ const world = qaWorld({ residualFallback: true }), state = []
+ await prepareSubscriptionImageBindings({ ...world, prepared: world.memory, runId: 'subscription-image-111111111111', state })
+ await assert.rejects(revokeSubscriptionImageBindings({ ...world, state }), /HOST_DETACH_FAILED/)
+ assert.equal(world.grants.get(state[0].connectionKey).status, 'connected')
+})
+test('prepareA -> revokeA -> resume retained memory -> prepareB uses fresh image identities; resumeA fails', async () => {
+ const world = qaWorld(), stateA = []
+ const qaA = await prepareSubscriptionImageBindings({ ...world, prepared: world.memory, runId: 'subscription-image-111111111111', state: stateA })
+ assert.equal((await revokeSubscriptionImageBindings({ ...world, state: stateA })).verified, true)
+ for (const binding of qaA.bindings) {
+ assert.equal(world.grants.get(binding.connectionKey).status, 'revoked')
+ assert.deepEqual(world.hosts.get(binding.hostRef).spec.llmPolicy.fallbacks, [])
+ assert.equal(world.hosts.get(binding.hostRef).spec.model.connectionRef, 'unassigned')
+ }
+ const resumed = await resumeMemoryFixtures({ ...world, expected: world.memory })
+ const qaB = await prepareSubscriptionImageBindings({ ...world, prepared: resumed, runId: 'subscription-image-222222222222', state: [] })
+ assert(qaB.bindings.every(binding => world.grants.get(binding.connectionKey).status === 'connected'))
+ assert(qaB.bindings.every(binding => !qaA.bindings.some(before => before.connectionKey === binding.connectionKey || before.hostRef === binding.hostRef)))
+ await assert.rejects(resumeMemoryFixtures({ ...world, expected: { ...world.memory, bindings: qaA.bindings } }), /BINDING_FOREIGN/)
+})
+
+test('revocation barrier retains vendor fixtures on residual fallback while allowing independent CAPI restoration', async () => {
+ const world = qaWorld({ residualFallback: true }), state = [], timeline = []
+ const status = { state: 'not-created', bindings: [] }
+ const session = { prepareImages: runId => prepareSubscriptionImageBindings({ ...world, prepared: world.memory, runId, state }),
+ revokeImages: async () => { timeline.push('revoke-attempt'); return revokeSubscriptionImageBindings({ ...world, state }) } }
+ await assert.rejects(withImageQaSession(session, 'subscription-image-333333333333', status,
+ async () => { timeline.push('fixture-run'); return 0 }), /HOST_DETACH_FAILED/)
+ const deployments = [{ name: 'grok-llm-proxy', provider: 'grok-subscription', changed: true },
+ { name: 'codex-llm-proxy', provider: 'codex-subscription', changed: true }, { name: 'control-api', changed: true }]
+ const plan = qaRestorationTargets(status.state, deployments)
+ for (const target of plan.restore) timeline.push(`restore:${target.name}`)
+ assert.deepEqual(timeline, ['fixture-run', 'revoke-attempt', 'restore:control-api'])
+ assert.equal(status.state, 'created'); assert.equal(status.bindings.length, 2)
+ assert.equal(plan.retain.length, 2)
+ assert(plan.retain.every(target => target.provider))
+ assert.equal(world.grants.get(state[0].connectionKey).status, 'connected')
+ assert.equal(qaRestorationTargets('created-or-unknown', deployments).retain.length, 2)
+})
+test('revocation barrier proves owned detach/revoke before any canonical vendor restore', async () => {
+ const world = qaWorld(), state = [], timeline = [], status = { state: 'not-created', bindings: [] }
+ const session = { prepareImages: runId => prepareSubscriptionImageBindings({ ...world, prepared: world.memory, runId, state }),
+ revokeImages: async () => { const result = await revokeSubscriptionImageBindings({ ...world, state });
+ timeline.push('revoke-verified'); return result } }
+ assert.equal(await withImageQaSession(session, 'subscription-image-444444444444', status,
+ async () => { timeline.push('fixture-run'); return 0 }), 0)
+ const plan = qaRestorationTargets(status.state, [{ name: 'grok-llm-proxy', provider: 'grok-subscription', changed: true },
+ { name: 'control-api', changed: true }])
+ for (const target of plan.restore) timeline.push(`restore:${target.name}`)
+ assert.deepEqual(timeline, ['fixture-run', 'revoke-verified', 'restore:grok-llm-proxy', 'restore:control-api'])
+ assert.equal(status.state, 'revoked'); assert.deepEqual(plan.retain, [])
+})
diff --git a/scripts/tests/subscription-image-coordinator.test.mjs b/scripts/tests/subscription-image-coordinator.test.mjs
new file mode 100644
index 000000000..4ca6469a2
--- /dev/null
+++ b/scripts/tests/subscription-image-coordinator.test.mjs
@@ -0,0 +1,250 @@
+// Hermetic source/protocol tests. Synthetic public metadata is never evidence
+// of a running profile, OAuth, physical deployment or visible Desktop flow.
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { spawnSync } from 'node:child_process'
+import { snapshotDeployment, deploymentMutationPatch, verifyQaModelProjection, vendorFrames } from '../e2e/coordinate-subscription-image-journeys.mjs'
+import { subscriptionImageSeedInputs, prepareSubscriptionImageBindings } from './lib/control-api-authorize-memory-seeder.ts'
+import { createSubscriptionImageVendor, SUBSCRIPTION_VENDOR_URLS } from '../e2e/fixtures/subscription-image-provider.mjs'
+import { validateRemainingFixture } from '../../desktop-app/test/e2e-playwright/subscriptionRemainingJourneysContract.ts'
+import { validatePrepareScratch } from '../e2e/prepare-subscription-image-admission.mjs'
+
+const runId = 'subscription-image-123456789abc'
+const planned = subscriptionImageSeedInputs(runId)
+const digest = 'a'.repeat(64), uid = '12345678-1234-4234-8234-123456789abc'
+const resource = () => ({ metadata: { uid, resourceVersion: '4' }, spec: { replicas: 1,
+ template: { spec: { containers: [{ name: 'unit-proxy', image: `sha256:${digest}`,
+ env: [{ name: 'NODE_ENV', value: 'production' }, { name: 'UNRELATED', value: 'unit-scope' }] }] } } } })
+function applyPatch(value, operations) {
+ const result = structuredClone(value)
+ for (const op of operations) {
+ const segments = op.path.slice(1).split('/'), key = segments.pop()
+ const parent = segments.reduce((current, name) => current[name], result)
+ if (op.op === 'test') assert.deepEqual(parent[key], op.value)
+ else if (op.op === 'remove') Array.isArray(parent) ? parent.splice(Number(key), 1) : delete parent[key]
+ else if (key === '-') parent.push(op.value)
+ else parent[key] = structuredClone(op.value)
+ }
+ return result
+}
+
+test('activation and restoration preserve unrelated fields and compare actual UID/RV/image/controlled values', () => {
+ const before = resource(), original = snapshotDeployment(before, 'unit-proxy', ['NODE_ENV', 'NODE_OPTIONS', 'QA_RUN'])
+ assert.deepEqual(original.env, { NODE_ENV: 'production' })
+ const applied = { image: `sha256:${'b'.repeat(64)}`, env: { NODE_ENV: 'test', NODE_OPTIONS: '--import=/app/unit.mjs', QA_RUN: runId } }
+ const active = applyPatch(before, deploymentMutationPatch(before, original, applied))
+ active.metadata.resourceVersion = '8'; active.spec.template.spec.containers[0].env[1].value = 'concurrent-unrelated'
+ const restored = applyPatch(active, deploymentMutationPatch(active, original, applied, true))
+ assert.equal(restored.spec.template.spec.containers[0].image, before.spec.template.spec.containers[0].image)
+ assert.deepEqual(restored.spec.template.spec.containers[0].env,
+ [{ name: 'NODE_ENV', value: 'production' }, { name: 'UNRELATED', value: 'concurrent-unrelated' }])
+ for (const change of [value => { value.metadata.uid = 'recreated' },
+ value => { value.spec.template.spec.containers[0].image = 'different' },
+ value => { value.spec.template.spec.containers[0].env.find(item => item.name === 'QA_RUN').value = 'other-run' }]) {
+ const mutated = structuredClone(active); change(mutated)
+ assert.throws(() => deploymentMutationPatch(mutated, original, applied, true), /OWNER_CHANGED|RESTORE_CONFLICT/)
+ }
+})
+test('unknown, duplicated, valueFrom or previously activated controlled fields refuse before patch', () => {
+ for (const env of [[{ name: 'NODE_OPTIONS', value: '--inspect=0.0.0.0:9229' }],
+ [{ name: 'NODE_ENV', value: 'test' }, { name: 'NODE_ENV', value: 'test' }],
+ [{ name: 'NODE_ENV', valueFrom: { configMapKeyRef: { name: 'unit', key: 'value' } } }],
+ [{ name: 'QA_RUN', value: 'other-run' }]]) {
+ const value = resource(); value.spec.template.spec.containers[0].env = env
+ assert.throws(() => snapshotDeployment(value, 'unit-proxy', ['NODE_ENV', 'NODE_OPTIONS', 'QA_RUN']))
+ }
+})
+
+test('preparer admits only a fresh private scratch and never chmods a shared/nonempty directory', () => {
+ const directory = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'qa-scratch-unit-')))
+ try {
+ fs.chmodSync(directory, 0o700); validatePrepareScratch(directory)
+ fs.chmodSync(directory, 0o755)
+ assert.throws(() => validatePrepareScratch(directory), /SCRATCH_NOT_FRESH/)
+ assert.equal(fs.statSync(directory).mode & 0o777, 0o755)
+ fs.chmodSync(directory, 0o700); fs.writeFileSync(path.join(directory, 'owned-unit-data'), 'unit')
+ assert.throws(() => validatePrepareScratch(directory), /SCRATCH_NOT_FRESH/)
+ assert.equal(fs.readFileSync(path.join(directory, 'owned-unit-data'), 'utf8'), 'unit')
+ } finally { fs.rmSync(directory, { recursive: true, force: true }) }
+})
+
+test('QA writer uses the private authenticated API for models/Hosts and keeps random grant material out of results', async () => {
+ const calls = [], models = new Map(), hosts = new Map(), state = []
+ const options = { runId: 'pr806-memory-123456789abc', hostNamespace: 'qa-hosts' }
+ const prepared = { operatorDesktopUserId: uid, operatorEmail: '[email protected]', operatorLink: { status: 'active' }, context: { uid } }
+ const session = { request: async request => {
+ calls.push({ method: request.method, path: request.path })
+ if (request.path.endsWith('contexts/context1')) return { status: 200, json: { metadata: { uid }, spec: { mcpServers: [] } } }
+ if (request.path.endsWith('llm-models') && request.method === 'POST') {
+ const row = { ...request.body, id: String(models.size + 1) }; models.set(row.id, row); return { status: 201, json: row }
+ }
+ if (request.path.includes('llm-models/')) return { status: 200, json: models.get(request.path.split('/').at(-1)) }
+ if (request.method === 'POST' && request.path.endsWith('hosts')) {
+ const row = { metadata: { ...request.body.metadata, uid, namespace: options.hostNamespace }, spec: request.body.spec }
+ hosts.set(row.metadata.name, row); return { status: 201, json: row }
+ }
+ return { status: 200, json: hosts.get(request.path.split('/').at(-1)) }
+ } }
+ let credentialWrites = 0, materialized = 0
+ const rows = new Map()
+ const insert = async (_tx, _key, credential, key) => {
+ assert(typeof credential.refreshToken === 'string' && typeof credential.accessToken === 'string')
+ assert(credential.accessTokenExpiresAt.getTime() - Date.now() >= 60 * 60 * 1000)
+ credentialWrites++
+ const value = { id: uid, credentialRevision: 1, catalogRevision: 0 }; rows.set(key, value); return value
+ }
+ const publish = async (_tx, input) => ({ ...rows.get(input.connectionKey), catalogRevision: 1 })
+ const prod = { config: {}, encryption: { deriveOAuthEncryptionKey: () => Buffer.alloc(32) },
+ db: { withTransaction: work => work({ query: async () => ({ rows: [] }) }) },
+ gateway: { llmAllowedModelsConfigMap: () => ({ materialize: async () => { materialized++ } }) },
+ connection: { insertInitialGrokSubscriptionConnection: insert, recordGrokCatalogOutcome: publish },
+ codexConnection: { insertInitialCodexSubscriptionConnection: insert, recordCodexCatalogOutcome: publish } }
+ const result = await prepareSubscriptionImageBindings({ prod, session, options, prepared, runId, state })
+ assert.equal(credentialWrites, 2); assert.equal(materialized, 2); assert.equal(result.bindings.length, 2)
+ assert.equal(calls.filter(value => value.method === 'POST' && value.path.endsWith('llm-models')).length, 6)
+ assert.equal(calls.filter(value => value.method === 'POST' && value.path.endsWith('hosts')).length, 2)
+ for (const binding of result.bindings) {
+ assert.equal(binding.fallback.provider, binding.provider)
+ assert.notEqual(binding.fallback.modelId, binding.modelId)
+ assert.equal(binding.hostNamespace, options.hostNamespace)
+ assert.equal(binding.hostUid, uid)
+ }
+ assert(!/refreshToken|accessToken|operatorPassword|cookie/.test(JSON.stringify(result)))
+ assert(!/refreshToken|accessToken|operatorPassword|cookie/.test(JSON.stringify(state)))
+ let wrote = false
+ await assert.rejects(prepareSubscriptionImageBindings({ prod, session: { request: async () => {
+ wrote = true; return { status: 200, json: { metadata: { uid: 'foreign' }, spec: { mcpServers: [] } } }
+ } }, options, prepared, runId, state: [] }), /CONTEXT_CHANGED/)
+ assert.equal(wrote, true); assert.equal(credentialWrites, 2)
+})
+
+function modelProjection(binding) {
+ const models = [binding.modelId, binding.unsupportedModelId, binding.fallback.modelId]
+ return { metadata: { annotations: {
+ 'clerum.io/codex-enabled': 'true', 'clerum.io/grok-enabled': 'true', 'clerum.io/content-hash': digest,
+ 'clerum.io/codex-connections': JSON.stringify({ [binding.connectionKey]: { status: 'connected',
+ catalogRevision: 1, connectionRevision: 1, models } }),
+ 'clerum.io/grok-connections': JSON.stringify({ [binding.connectionKey]: { status: 'connected',
+ catalogRevision: 1, connectionRevision: 1, models } }),
+ } }, data: { [binding.provider]: JSON.stringify(models.map((model, index) => ({ model,
+ imageInput: { state: index === 1 ? 'unsupported' : 'supported', evidence: {
+ source: 'curated', reference: `evidence:${runId}`, checkedAt: new Date(Date.now() - 1000).toISOString() } } }))) } }
+}
+test('both actual catalogue projections must independently prove primary, unsupported and same-grant alternate', () => {
+ for (const input of planned) {
+ const binding = { ...input, credentialRevision: 1, catalogRevision: 1 }
+ const good = modelProjection(binding)
+ assert.equal(verifyQaModelProjection(good, binding), true)
+ for (const mutate of [value => { value.data[binding.provider] = '[]' },
+ value => { value.metadata.annotations[`clerum.io/${binding.provider.startsWith('grok') ? 'grok' : 'codex'}-enabled`] = 'false' },
+ value => { const rows = JSON.parse(value.data[binding.provider]); rows[2].imageInput = { state: 'unknown' }; value.data[binding.provider] = JSON.stringify(rows) }]) {
+ const bad = structuredClone(good); mutate(bad)
+ assert.throws(() => verifyQaModelProjection(bad, binding), /ELIGIBILITY_UNPROVED/)
+ }
+ assert.throws(() => verifyQaModelProjection(good, { ...binding,
+ fallback: { ...binding.fallback, provider: binding.provider.startsWith('grok') ? 'codex-subscription' : 'grok-subscription' } }), /FALLBACK_GRANT_INVALID/)
+ }
+})
+
+const vendorSource = () => ({ podUid: uid, imageId: `sha256:${digest}`, fixtureImportSha256: digest,
+ profile: 'qa-unit', gitHead: 'a'.repeat(40) })
+test('live producer emits login once, empty ledgers and actual subsequent attempt, then stops; historical mutation rejects', async () => {
+ const bindings = planned.map(({ provider, modelId, unsupportedModelId }) => ({ provider, modelId, unsupportedModelId }))
+ const vendors = bindings.map(binding => createSubscriptionImageVendor({ runId, bindings, acceptedProvider: binding.provider }))
+ const sources = Object.fromEntries(bindings.map(binding => [binding.provider, vendorSource()]))
+ const states = bindings.map((binding, index) => ({ provider: binding.provider, witness: sources[binding.provider], index }))
+ const admission = { runId, bindings, vendorSources: sources }, controller = new AbortController()
+ const generator = vendorFrames({ admission, states, loginFrame: { kind: 'unit-login-metadata' }, signal: controller.signal,
+ observe: state => ({ ledger: vendors[state.index].snapshot() }) })
+ assert.equal(JSON.parse((await generator.next()).value).kind, 'unit-login-metadata')
+ for (let index = 0; index < 2; index++) assert.deepEqual(JSON.parse((await generator.next()).value).ledger.attempts, [])
+ await vendors[0].respond(SUBSCRIPTION_VENDOR_URLS[bindings[0].provider].responses, { method: 'POST',
+ body: JSON.stringify({ model: bindings[0].modelId, input: [{ role: 'user', content: [{ type: 'input_text',
+ text: `Receipt: ${uid}` }] }] }) })
+ const changed = JSON.parse((await generator.next()).value)
+ assert.equal(changed.ledger.attempts.length, 1)
+ // Advance one prefix before corrupting it: a vacuous initial-ledger check
+ // would miss this change and falsely accept mutable historical evidence.
+ let observation = structuredClone(changed.ledger)
+ const history = vendorFrames({ admission, states: [states[0]], loginFrame: {}, signal: controller.signal,
+ observe: () => ({ ledger: observation }) })
+ await history.next(); await history.next()
+ observation.attempts[0].requestSha256 = 'c'.repeat(64)
+ await assert.rejects(history.next(), /HISTORY_CHANGED/)
+ controller.abort(); await generator.return()
+})
+
+test('admission fixture accepts eligible same-provider alternate and rejects impossible cross-broker fallback', () => {
+ const bindings = planned.map(value => ({ ...value })), runRoot = '/run/evenfire-e2e/unit'
+ const run = { runId, profile: 'qa-unit', context: 'qa-unit', bindings, runRoot }
+ const receipt = { kind: 'evenfire-subscription-remaining-journey-fixture-v1', runId, suite: 'admission-recovery',
+ profile: run.profile, context: run.context, sourceManifestSha256: digest, preparedAt: new Date().toISOString(),
+ fixtures: Object.fromEntries(bindings.map(binding => [binding.provider, { hostRef: binding.hostRef, podUid: uid,
+ imageId: `sha256:${digest}`, controlApiPodUid: uid, controlApiImageId: `sha256:${digest}`, maxInFlight: 1,
+ readDeadlineMs: 10000, pressure: { receiptFile: '/runner-admission/main-admission.json' }, fallback: binding.fallback }])) }
+ const input = { suite: 'admission-recovery', fixtureReceiptPath: `${runRoot}/remaining.json` }
+ assert.equal(validateRemainingFixture(receipt, input, run, digest), receipt)
+ const bad = structuredClone(receipt)
+ bad.fixtures[bindings[0].provider].fallback = { provider: bindings[1].provider, modelId: bindings[1].modelId }
+ assert.throws(() => validateRemainingFixture(bad, input, run, digest), /eligible physical fallback/)
+})
+
+test('public source separates private input, actual frame stream, vendor-only derived closure and CAS restore', () => {
+ const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../..')
+ const source = fs.readFileSync(path.join(root, 'scripts/e2e/coordinate-subscription-image-journeys.mjs'), 'utf8')
+ const dockerfile = fs.readFileSync(path.join(root, 'scripts/e2e/fixtures/subscription-image-proxy.Dockerfile'), 'utf8')
+ assert(source.includes('seed(memoryOptions, privateMaterial, async session'))
+ assert(source.includes('session.revokeImages()') && source.includes('restoreDeployments(memoryOptions.context, states)'))
+ assert(!source.includes('/environ') && !source.includes('page.evaluate') && !source.includes('storageState'))
+ assert(dockerfile.includes('/app/mcp-host/node_modules/@napi-rs/') && !dockerfile.includes('npm ') && !dockerfile.includes('COPY dist'))
+})
+
+test('actual derived builder refuses ambiguous shell exports, nonlocal endpoints and changed base IDs before build', () => {
+ const root = path.resolve(path.dirname(new URL(import.meta.url).pathname), '../..')
+ const temporary = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'qa-derived-builder-unit-')))
+ try {
+ const bin = path.join(temporary, 'bin'), scripts = path.join(temporary, 'scripts')
+ fs.mkdirSync(bin); fs.mkdirSync(path.join(scripts, 'e2e'), { recursive: true })
+ fs.mkdirSync(path.join(scripts, 'minikube'))
+ for (const rel of ['scripts/e2e/build-subscription-image-proxy.sh', 'scripts/minikube/docker-cli-env.sh',
+ 'scripts/minikube/run-with-deadline.mjs']) fs.copyFileSync(path.join(root, rel), path.join(temporary, rel))
+ const trace = path.join(temporary, 'operations')
+ fs.writeFileSync(path.join(scripts, 'minikube/require-t2-mutation-lock.sh'), 'exit 0\n')
+ const interpreter = `#!${process.execPath}\n`
+ fs.writeFileSync(path.join(bin, 'docker'), interpreter + `
+import fs from 'node:fs';const args=process.argv.slice(2);fs.appendFileSync(process.env.QA_TRACE,JSON.stringify(args)+'\\n');
+if(args[0]==='context')process.stdout.write('unix:///private/tmp/qa-builder-unit.sock');
+else if(args[0]==='image'&&args[1]==='inspect'){
+ const ref=args.at(-1);process.stdout.write(ref.includes('host')?process.env.QA_HOST_ID:ref.includes('-base')||ref==='clerum/grok-llm-proxy:test'?process.env.QA_PROXY_ID:process.env.QA_DERIVED_ID);
+}
+`)
+ fs.writeFileSync(path.join(bin, 'minikube'), interpreter + `
+process.stdout.write(process.env.QA_DOCKER_EXPORTS);
+`)
+ fs.chmodSync(path.join(bin, 'docker'), 0o755); fs.chmodSync(path.join(bin, 'minikube'), 0o755)
+ const expected = `sha256:${'b'.repeat(64)}`, host = `sha256:${'c'.repeat(64)}`, derived = `sha256:${'d'.repeat(64)}`
+ const env = { ...process.env, PATH: `${bin}:${path.dirname(process.execPath)}:/usr/bin:/bin`,
+ DOCKER_HOST: 'unix:///private/tmp/qa-builder-unit.sock', MINIKUBE_PROFILE: 'qa-unit', CONTROL_API_REAL_PG_CONTEXT: 'qa-unit',
+ QA_TRACE: trace, QA_PROXY_ID: expected, QA_HOST_ID: host, QA_DERIVED_ID: derived,
+ QA_DOCKER_EXPORTS: 'export DOCKER_HOST="unix:///private/tmp/qa-builder-unit.sock"\nexport MINIKUBE_ACTIVE_DOCKERD="qa-unit"\n' }
+ const args = [path.join(scripts, 'e2e/build-subscription-image-proxy.sh'), 'qa-unit', temporary,
+ 'clerum/grok-llm-proxy:test', expected, host, 'clerum/grok-llm-proxy-image-qa:123456789abc', 'a'.repeat(40)]
+ const successful = spawnSync('bash', args, { env, encoding: 'utf8', timeout: 30000 })
+ assert.equal(successful.status, 0, successful.stderr)
+ assert.deepEqual(JSON.parse(successful.stdout), { imageId: derived, baseImageId: expected, hostImageId: host, gitHead: 'a'.repeat(40) })
+ assert(fs.readFileSync(trace, 'utf8').includes('"build"'))
+ for (const patch of [
+ { QA_DOCKER_EXPORTS: 'export DOCKER_HOST="tcp://192.0.2.1:2376"\nexport MINIKUBE_ACTIVE_DOCKERD="qa-unit"\n' },
+ { QA_DOCKER_EXPORTS: 'export DOCKER_HOST="unix:///private/tmp/qa-builder-unit.sock"\nexport PATH="/bad"\n' },
+ { QA_DOCKER_EXPORTS: '' }, { QA_PROXY_ID: `sha256:${'e'.repeat(64)}` },
+ ]) {
+ fs.writeFileSync(trace, '')
+ const refused = spawnSync('bash', args, { env: { ...env, ...patch }, encoding: 'utf8', timeout: 30000 })
+ assert.notEqual(refused.status, 0)
+ assert(!fs.readFileSync(trace, 'utf8').includes('"build"'))
+ }
+ } finally { fs.rmSync(temporary, { recursive: true, force: true }) }
+})
diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.sh
index 5f88a66da..3c4f173ae 100755
--- a/scripts/tests/test-codex-subscription-t0.sh
+++ b/scripts/tests/test-codex-subscription-t0.sh
@@ -240,6 +240,9 @@ run_group "codex-upstream-contract-freeze" "tests/e2e" "integration/codex-subscr
run_node_group "approved-tools-fixtures-and-runner" \
"scripts/tests/subscription-t0-discovery.test.mjs" \
"scripts/tests/subscription-image-runner.test.mjs" \
+ "scripts/tests/subscription-image-coordinator.test.mjs" \
+ "scripts/tests/subscription-image-coordinator-regressions.test.mjs" \
+ "scripts/tests/control-api-qa-runtime.test.mjs" \
"scripts/tests/subscription-image-input-frames.test.mjs" \
"scripts/e2e/prepare-subscription-remaining-fixtures.test.mjs" \
"scripts/e2e/fixtures/subscription-image-provider.test.mjs" \
diff --git a/scripts/tests/test-grok-subscription-t0.sh b/scripts/tests/test-grok-subscription-t0.sh
index d5d57c930..f35cbec54 100755
--- a/scripts/tests/test-grok-subscription-t0.sh
+++ b/scripts/tests/test-grok-subscription-t0.sh
@@ -238,6 +238,9 @@ echo "Grok subscription T0 aggregator"
run_node_group "subscription-discovery-and-offline-image-provider" \
"scripts/tests/subscription-t0-discovery.test.mjs" \
"scripts/tests/subscription-image-runner.test.mjs" \
+ "scripts/tests/subscription-image-coordinator.test.mjs" \
+ "scripts/tests/subscription-image-coordinator-regressions.test.mjs" \
+ "scripts/tests/control-api-qa-runtime.test.mjs" \
"scripts/tests/subscription-image-input-frames.test.mjs" \
"scripts/e2e/prepare-subscription-remaining-fixtures.test.mjs" \
"scripts/e2e/fixtures/subscription-image-provider.test.mjs"