Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(codex): enforce fixture provenance and owned recovery
Alfredo Lopez2679f6a671b7
diff --git a/.github/workflows/ci-public.yml b/.github/workflows/ci-public.ymlindex 738449fc2..12a1076cf 100644--- a/.github/workflows/ci-public.yml+++ b/.github/workflows/ci-public.yml@@ -137,6 +137,8 @@ jobs:scripts/e2e/prepare-codex-approved-tools.test.mjs \scripts/e2e/run-codex-approved-tools.test.mjs \scripts/e2e/desktop-login-seed.test.mjs \+ scripts/e2e/approved-tools-image-proof.test.mjs \+ scripts/e2e/approved-tools-restoration.test.mjs \tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs# Typecheck against the base tsconfig (covers src + test dirs).diff --git a/Makefile b/Makefileindex 0d0ef4aef..da9f35eaa 100644--- a/Makefile+++ b/Makefile@@ -257,11 +257,15 @@ minikube-build-custom-coordinator-fixture-body:.PHONY: minikube-install-627-real-toolsminikube-install-627-real-tools: ## Install real Worktracker and Wikipedia baseline under the owned profile lease- @bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/install-627-real-tools.mjs+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/install-627-real-tools.mjs.PHONY: minikube-build-627-worktrackerminikube-build-627-worktracker: ## Build the reviewed Worktracker sources locally without registry credentials or publication- @bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-627-worktracker.sh+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-627-worktracker.sh.PHONY: minikube-build-codex-approved-tools-fixtures minikube-build-codex-approved-tools-fixtures-bodyminikube-build-codex-approved-tools-fixtures: ## Acquire optional Codex tools fixture images before T2 reconcile@@ -1475,4 +1479,6 @@ help: ## Show this help.PHONY: minikube-build-627-githubminikube-build-627-github: ## Build the reviewed GitHub MCP locally for the owned profile architecture- @bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-627-github.sh+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-627-github.shdiff --git a/scripts/e2e/approved-tools-image-proof.mjs b/scripts/e2e/approved-tools-image-proof.mjsnew file mode 100644index 000000000..fe89c591e--- /dev/null+++ b/scripts/e2e/approved-tools-image-proof.mjs@@ -0,0 +1,248 @@+// Provenance proof for the optional Codex approved-tools fixture images.+//+// The fixture lane swaps the running codex-llm-proxy Deployment onto a locally+// built fixture image, and the workflow scenario runs a locally built+// coordinator fixture. Those refs are acquired by+// `make minikube-build-codex-approved-tools-fixtures`, which runs+// scripts/minikube/build-images.sh five times with --only. Each run rewrites+// deploy/minikube/.image-manifest.json, two of the fixtures are BUILT FROM the+// normal proxy image and the normal workflow coordinator image, and any of the+// five can still be the one an earlier commit built.+//+// A ref that merely EXISTS is not proof that the delegation test exercised the+// code under review, and a fixture built from a stale base embeds that stale+// base. So this module certifies nothing until:+//+// * the manifest names the exact profile that is about to be mutated;+// * the profile currently resolves every expected ref to the recorded digest;+// * every expected ref records the reviewed commit as its OWN source+// revision (the manifest's gitHead is informational and certifies nothing);+// * each fixture's recorded base and the base's current digest agree.+//+// It is a pure validator. It never spawns a command, reads a file, or writes+// one: the caller observes the profile with the same inventory read+// build-images.sh already uses, reads the manifest, and hands both in before+// the first cluster mutation. Failure throws an Error whose message starts with+// a stable code (IMAGE_PROOF_*) and names the acquisition command.+//+// The same repository is spelled two ways in practice: `minikube image ls`+// reports the canonical `docker.io/clerum/...` repository, while the manifest+// and deploy/images.json name the local refs as `clerum/...`. Both are one+// image, so comparisons run on the canonical form -- and two spellings of one+// image carrying different digests are a conflict, never two images.+import { IMAGES, localRef } from '../release/images-manifest.mjs'++export const ACQUIRE_COMMAND = 'make minikube-build-codex-approved-tools-fixtures'++const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/+const HEAD_PATTERN = /^[0-9a-f]{40}$/+const PROFILE_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/++function localImageRef(name) {+ const entry = IMAGES.find(image => image.name === name)+ if (!entry)+ throw new Error(+ `IMAGE_PROOF_MANIFEST_INVALID: deploy/images.json has no image named ${name}, so the fixture set cannot be resolved`+ )+ return localRef(entry)+}++// The three fixture refs the prepare step swaps in, and the two normal images+// they are built from. All five must belong to the reviewed commit.+export const FIXTURE_IMAGES = Object.freeze([+ localImageRef('codex-approved-tools-mcp-e2e'),+ localImageRef('codex-approved-tools-proxy-e2e'),+ localImageRef('codex-approved-tools-workflow-e2e'),+])++export const BASE_IMAGES = Object.freeze([+ localImageRef('codex-llm-proxy'),+ localImageRef('workflow-custom-sdk-e2e'),+])++export const EXPECTED_IMAGES = Object.freeze([...FIXTURE_IMAGES, ...BASE_IMAGES])++// Fixtures whose Dockerfile builds FROM another image this profile holds. The+// manifest records that base and the digest it resolved to; a fixture built+// from an older base still names the same base TAG, so the tag proves nothing+// on its own.+export const DERIVED_FIXTURES = Object.freeze([+ { ref: localImageRef('codex-approved-tools-proxy-e2e'), base: localImageRef('codex-llm-proxy') },+ {+ ref: localImageRef('codex-approved-tools-workflow-e2e'),+ base: localImageRef('workflow-custom-sdk-e2e'),+ },+])++function isRecord(value) {+ return Boolean(value) && typeof value === 'object' && !Array.isArray(value)+}++function normalizeDigest(value) {+ if (typeof value !== 'string') return ''+ const normalized = value.toLowerCase()+ return DIGEST_PATTERN.test(normalized) ? normalized : ''+}++// Docker's own normalization: a first path component that is not a registry+// host is a Docker Hub namespace, and a ref with no path component at all is a+// library image. Anything this cannot normalize stays its own value, so it+// simply fails a later comparison instead of silently matching something else.+function canonicalImageRef(ref) {+ if (typeof ref !== 'string' || !ref || /\s/.test(ref)) return ''+ const separator = ref.indexOf('/')+ if (separator === -1) return `docker.io/library/${ref}`+ const registry = ref.slice(0, separator)+ const looksLikeRegistry =+ registry.includes('.') || registry.includes(':') || registry === 'localhost'+ return looksLikeRegistry ? ref : `docker.io/${ref}`+}++function normalizeObservations(images) {+ if (!Array.isArray(images))+ throw new Error(+ 'IMAGE_PROOF_MANIFEST_INVALID: images must be the observed [{ ref, id }] inventory of the profile being mutated'+ )+ const observed = new Map()+ for (const entry of images) {+ if (!isRecord(entry) || typeof entry.ref !== 'string' || !entry.ref)+ throw new Error('IMAGE_PROOF_MANIFEST_INVALID: every observed image needs a ref and an id')+ const canonical = canonicalImageRef(entry.ref)+ if (!canonical)+ throw new Error(`IMAGE_PROOF_IMAGE_MISMATCH: '${entry.ref}' is not a usable image reference`)+ const digest = normalizeDigest(entry.id)+ if (!digest)+ throw new Error(+ `IMAGE_PROOF_IMAGE_MISMATCH: the observed id for ${entry.ref} is not a sha256 digest`+ )+ const previous = observed.get(canonical)+ if (previous && previous.id !== digest)+ throw new Error(+ `IMAGE_PROOF_IMAGE_MISMATCH: ${previous.ref} and ${entry.ref} are the same image with different ids (${previous.id}, ${digest})`+ )+ if (!previous) observed.set(canonical, { ref: entry.ref, id: digest })+ }+ return observed+}++// The manifest's own images map, keyed by canonical ref. Two spellings of one+// image may both be recorded; they must agree, or the manifest contradicts+// itself and no record from it can be trusted.+function indexRecordedImages(images) {+ const index = new Map()+ for (const [ref, value] of Object.entries(images)) {+ const canonical = canonicalImageRef(ref)+ const digest = normalizeDigest(value)+ if (!canonical || !digest) continue+ const existing = index.get(canonical)+ if (existing && existing.id !== digest)+ throw new Error(+ `IMAGE_PROOF_MANIFEST_INVALID: the manifest records ${existing.ref} and ${ref} with different digests, but they are the same image`+ )+ if (!existing) index.set(canonical, { ref, id: digest })+ }+ return index+}++function indexRecordedRevisions(revisions) {+ const index = new Map()+ for (const [ref, value] of Object.entries(revisions)) {+ const canonical = canonicalImageRef(ref)+ if (!canonical || typeof value !== 'string') continue+ const existing = index.get(canonical)+ if (existing && existing !== value)+ throw new Error(+ `IMAGE_PROOF_MANIFEST_INVALID: the manifest records ${ref} twice with different source revisions (${existing}, ${value})`+ )+ if (!existing) index.set(canonical, value)+ }+ return index+}++// Certify the fixture images this profile holds against the commit under+// review. Returns the record the caller stores with the fixture run state:+// { profile, sourceHead, images, sourceRevisions, derivedFrom }.+export function buildApprovedToolsImageProof({ profile, sourceHead, manifest, images } = {}) {+ if (typeof profile !== 'string' || !PROFILE_PATTERN.test(profile))+ throw new Error(+ `IMAGE_PROOF_PROFILE_MISMATCH: '${profile ?? ''}' is not a usable Minikube profile`+ )+ if (typeof sourceHead !== 'string' || !HEAD_PATTERN.test(sourceHead))+ throw new Error(+ `IMAGE_PROOF_HEAD_MISMATCH: '${sourceHead ?? ''}' is not the 40-character commit under review`+ )+ if (!isRecord(manifest))+ throw new Error(+ `IMAGE_PROOF_MANIFEST_INVALID: deploy/minikube/.image-manifest.json is absent or unreadable; acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ if (manifest.profile !== profile)+ throw new Error(+ `IMAGE_PROOF_PROFILE_MISMATCH: the manifest records profile '${manifest.profile ?? ''}', this run mutates '${profile}'; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ if (!isRecord(manifest.images))+ throw new Error(+ `IMAGE_PROOF_MANIFEST_INVALID: the manifest has no images map; acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ const revisions = isRecord(manifest.sourceRevisions) ? manifest.sourceRevisions : {}+ const derived = isRecord(manifest.derivedFrom) ? manifest.derivedFrom : {}+ const recorded = indexRecordedImages(manifest.images)+ const recordedRevisions = indexRecordedRevisions(revisions)+ const observed = normalizeObservations(images)++ const proofImages = {}+ const proofRevisions = {}+ for (const ref of EXPECTED_IMAGES) {+ const record = recorded.get(canonicalImageRef(ref))+ if (!record)+ throw new Error(+ `IMAGE_PROOF_IMAGE_MISSING: the manifest records no image digest for ${ref}; acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ const observedEntry = observed.get(canonicalImageRef(ref))+ if (!observedEntry)+ throw new Error(+ `IMAGE_PROOF_IMAGE_MISSING: profile '${profile}' holds no image tagged ${ref}; acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ if (observedEntry.id !== record.id)+ throw new Error(+ `IMAGE_PROOF_IMAGE_MISMATCH: profile '${profile}' resolves ${observedEntry.ref} to ${observedEntry.id}, the manifest recorded ${record.ref} as ${record.id}; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ const revision = recordedRevisions.get(canonicalImageRef(ref))+ if (typeof revision !== 'string' || !HEAD_PATTERN.test(revision))+ throw new Error(+ `IMAGE_PROOF_HEAD_MISSING: the manifest records no source revision for ${ref}; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ if (revision !== sourceHead)+ throw new Error(+ `IMAGE_PROOF_HEAD_MISMATCH: ${ref} was built at ${revision}, the commit under review is ${sourceHead}; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ proofImages[ref] = record.id+ proofRevisions[ref] = revision+ }++ const proofDerived = {}+ for (const { ref, base } of DERIVED_FIXTURES) {+ const binding = derived[ref]+ if (+ !isRecord(binding) ||+ canonicalImageRef(binding.ref) !== canonicalImageRef(base) ||+ !normalizeDigest(binding.id)+ )+ throw new Error(+ `IMAGE_PROOF_BASE_BINDING_MISSING: the manifest does not record ${ref} as built from ${base}; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ const bindingId = normalizeDigest(binding.id)+ if (bindingId !== proofImages[base])+ throw new Error(+ `IMAGE_PROOF_BASE_MISMATCH: ${ref} was built from ${base} at ${bindingId}, profile '${profile}' now holds ${proofImages[base]}; re-acquire the fixtures with: ${ACQUIRE_COMMAND}`+ )+ proofDerived[ref] = { ref: base, id: bindingId }+ }++ return {+ profile,+ sourceHead,+ images: proofImages,+ sourceRevisions: proofRevisions,+ derivedFrom: proofDerived,+ }+}diff --git a/scripts/e2e/approved-tools-image-proof.test.mjs b/scripts/e2e/approved-tools-image-proof.test.mjsnew file mode 100644index 000000000..630e969c5--- /dev/null+++ b/scripts/e2e/approved-tools-image-proof.test.mjs@@ -0,0 +1,276 @@+// Pure validation coverage for scripts/e2e/approved-tools-image-proof.mjs.+//+// Every input here is in-memory metadata: manifests, observed image records,+// and reviewed commit ids built from synthetic strings. Nothing in this file+// runs a command, touches the filesystem, or reaches a cluster, and every+// digest is generated, not copied.+//+// This is NOT the end-to-end coverage for the acquisition contract. The hermetic+// builder test -- which runs scripts/minikube/build-images.sh against fake+// runtime binaries and proves the carry-across-partial-runs behaviour -- is+// still owed, and its absence is the open gap for this change.+import assert from 'node:assert/strict'+import { createHash } from 'node:crypto'+import test from 'node:test'+import {+ BASE_IMAGES,+ DERIVED_FIXTURES,+ EXPECTED_IMAGES,+ FIXTURE_IMAGES,+ buildApprovedToolsImageProof,+} from './approved-tools-image-proof.mjs'++const PROFILE = 'clerum-image-proof-a1b2c3d4'+const OTHER_PROFILE = 'clerum-image-proof-deadbeef'+const HEAD = 'a'.repeat(40)+const OTHER_HEAD = 'b'.repeat(40)+const PROXY_BASE = BASE_IMAGES[0]+const WORKFLOW_BASE = BASE_IMAGES[1]+const PROXY_FIXTURE = DERIVED_FIXTURES[0].ref+const WORKFLOW_FIXTURE = DERIVED_FIXTURES[1].ref++const digest = seed => `sha256:${createHash('sha256').update(seed).digest('hex')}`++// The registry-qualified spelling a container runtime reports for these refs.+const canonical = ref => `docker.io/${ref}`++function recordedImages() {+ return Object.fromEntries(EXPECTED_IMAGES.map((ref, index) => [ref, digest(`${ref}#${index}`)]))+}++function manifestFor(images) {+ return {+ generated: '2026-09-15T00:00:00Z',+ profile: PROFILE,+ imageSource: 'local',+ imageTag: '',+ gitHead: HEAD,+ images: { ...images },+ sourceRevisions: Object.fromEntries(EXPECTED_IMAGES.map(ref => [ref, HEAD])),+ derivedFrom: Object.fromEntries(+ DERIVED_FIXTURES.map(({ ref, base }) => [ref, { ref: base, id: images[base] }])+ ),+ }+}++function observedEntries(images, spell = ref => ref) {+ return Object.entries(images).map(([ref, id]) => ({ ref: spell(ref), id }))+}++function certify({ profile = PROFILE, sourceHead = HEAD, manifest, images }) {+ return buildApprovedToolsImageProof({ profile, sourceHead, manifest, images })+}++test('certifies the three fixture refs and both base refs at the reviewed commit', () => {+ const images = recordedImages()+ const proof = certify({ manifest: manifestFor(images), images: observedEntries(images) })+ assert.deepEqual(Object.keys(proof.images).sort(), [...EXPECTED_IMAGES].sort())+ assert.equal(proof.profile, PROFILE)+ assert.equal(proof.sourceHead, HEAD)+ for (const ref of EXPECTED_IMAGES) {+ assert.equal(proof.images[ref], images[ref])+ assert.equal(proof.sourceRevisions[ref], HEAD)+ }+ for (const { ref, base } of DERIVED_FIXTURES)+ assert.deepEqual(proof.derivedFrom[ref], { ref: base, id: images[base] })+})++test('matches a registry-qualified inventory against the manifest spellings and keeps them', () => {+ const images = recordedImages()+ const proof = certify({+ manifest: manifestFor(images),+ images: observedEntries(images, canonical),+ })+ for (const ref of EXPECTED_IMAGES) assert.equal(proof.images[ref], images[ref])+ assert.deepEqual(Object.keys(proof.images).sort(), [...EXPECTED_IMAGES].sort())+})++test('matches a registry-qualified manifest against unqualified inventory spellings', () => {+ const images = recordedImages()+ const manifest = manifestFor(images)+ manifest.images = Object.fromEntries(+ Object.entries(images).map(([ref, id]) => [canonical(ref), id])+ )+ manifest.sourceRevisions = Object.fromEntries(EXPECTED_IMAGES.map(ref => [canonical(ref), HEAD]))+ const proof = certify({ manifest, images: observedEntries(images) })+ for (const ref of EXPECTED_IMAGES) assert.equal(proof.images[ref], images[ref])+})++test('accepts both spellings of one image only while they record the same digest', () => {+ const images = recordedImages()+ const manifest = manifestFor(images)+ manifest.images[canonical(PROXY_BASE)] = images[PROXY_BASE]+ manifest.sourceRevisions[canonical(PROXY_BASE)] = HEAD+ const proof = certify({ manifest, images: observedEntries(images) })+ assert.equal(proof.images[PROXY_BASE], images[PROXY_BASE])+})++test('refuses an inventory that spells one image twice with different digests', () => {+ const images = recordedImages()+ const conflicting = [+ ...observedEntries(images, canonical),+ { ref: PROXY_FIXTURE, id: digest('a-different-image') },+ ]+ assert.throws(+ () => certify({ manifest: manifestFor(images), images: conflicting }),+ /IMAGE_PROOF_IMAGE_MISMATCH/+ )+})++test('refuses a manifest that contradicts itself across two spellings', () => {+ const images = recordedImages()+ const manifest = manifestFor(images)+ manifest.images[canonical(PROXY_FIXTURE)] = digest('a-different-image')+ assert.throws(+ () => certify({ manifest, images: observedEntries(images) }),+ /IMAGE_PROOF_MANIFEST_INVALID/+ )+})++test('refuses a manifest that records one image twice with different source revisions', () => {+ const images = recordedImages()+ const manifest = manifestFor(images)+ manifest.sourceRevisions[canonical(WORKFLOW_BASE)] = OTHER_HEAD+ assert.throws(+ () => certify({ manifest, images: observedEntries(images) }),+ /IMAGE_PROOF_MANIFEST_INVALID/+ )+})++test('the manifest gitHead certifies nothing on its own', () => {+ const images = recordedImages()+ const manifest = { ...manifestFor(images), gitHead: OTHER_HEAD }+ const proof = certify({ manifest, images: observedEntries(images) })+ assert.equal(proof.sourceHead, HEAD)+ assert.equal(manifest.gitHead, OTHER_HEAD)+})++test('refuses a missing or unusable manifest', () => {+ const images = recordedImages()+ for (const manifest of [undefined, null, 'not-an-object', [], {}]) {+ assert.throws(+ () => certify({ manifest, images: observedEntries(images) }),+ /IMAGE_PROOF_(MANIFEST_INVALID|PROFILE_MISMATCH)/+ )+ }+})++test('refuses a manifest acquired for another profile', () => {+ const images = recordedImages()+ const manifest = { ...manifestFor(images), profile: OTHER_PROFILE }+ assert.throws(+ () => certify({ manifest, images: observedEntries(images) }),+ /IMAGE_PROOF_PROFILE_MISMATCH/+ )+})++test('refuses an unusable reviewed head', () => {+ const images = recordedImages()+ for (const sourceHead of ['', 'HEAD', 'A'.repeat(40), 'c'.repeat(39), HEAD.slice(1)]) {+ assert.throws(+ () => certify({ sourceHead, manifest: manifestFor(images), images: observedEntries(images) }),+ /IMAGE_PROOF_HEAD_MISMATCH/+ )+ }+ // Omitting the reviewed commit entirely takes the same refusal path.+ assert.throws(+ () =>+ buildApprovedToolsImageProof({+ profile: PROFILE,+ manifest: manifestFor(images),+ images: observedEntries(images),+ }),+ /IMAGE_PROOF_HEAD_MISMATCH/+ )+})++test('refuses a ref the manifest does not record or the profile does not hold', () => {+ const images = recordedImages()+ const withoutBase = manifestFor(images)+ delete withoutBase.images[PROXY_BASE]+ assert.throws(+ () => certify({ manifest: withoutBase, images: observedEntries(images) }),+ /IMAGE_PROOF_IMAGE_MISSING/+ )++ const absentFixture = observedEntries(images).filter(entry => entry.ref !== WORKFLOW_FIXTURE)+ assert.throws(+ () => certify({ manifest: manifestFor(images), images: absentFixture }),+ /IMAGE_PROOF_IMAGE_MISSING/+ )+})++test('refuses an observation that disagrees with the manifest or is unusable', () => {+ const images = recordedImages()+ const replaced = observedEntries(images).map(entry =>+ entry.ref === WORKFLOW_BASE ? { ref: entry.ref, id: digest('replaced') } : entry+ )+ assert.throws(+ () => certify({ manifest: manifestFor(images), images: replaced }),+ /IMAGE_PROOF_IMAGE_MISMATCH/+ )+ assert.throws(+ () => certify({ manifest: manifestFor(images), images: { ref: PROXY_BASE } }),+ /IMAGE_PROOF_MANIFEST_INVALID/+ )+ assert.throws(+ () =>+ certify({+ manifest: manifestFor(images),+ images: [{ ref: PROXY_BASE, id: 'sha256:not-a-digest' }],+ }),+ /IMAGE_PROOF_IMAGE_MISMATCH/+ )+ assert.throws(+ () => certify({ manifest: manifestFor(images), images: [{ ref: '', id: images[PROXY_BASE] }] }),+ /IMAGE_PROOF_MANIFEST_INVALID/+ )+})++test('refuses a ref with no recorded source revision', () => {+ const images = recordedImages()+ const manifest = manifestFor(images)+ delete manifest.sourceRevisions[FIXTURE_IMAGES[0]]+ assert.throws(+ () => certify({ manifest, images: observedEntries(images) }),+ /IMAGE_PROOF_HEAD_MISSING/+ )+})++test('refuses a fixture or a base that was built at a different commit', () => {+ const images = recordedImages()+ for (const stale of [PROXY_FIXTURE, WORKFLOW_FIXTURE, PROXY_BASE, WORKFLOW_BASE]) {+ const manifest = manifestFor(images)+ manifest.sourceRevisions[stale] = OTHER_HEAD+ assert.throws(+ () => certify({ manifest, images: observedEntries(images, canonical) }),+ /IMAGE_PROOF_HEAD_MISMATCH/,+ stale+ )+ }+})++test('refuses a fixture whose recorded base is absent, different or no longer current', () => {+ const images = recordedImages()++ const unbound = manifestFor(images)+ delete unbound.derivedFrom[PROXY_FIXTURE]+ assert.throws(+ () => certify({ manifest: unbound, images: observedEntries(images) }),+ /IMAGE_PROOF_BASE_BINDING_MISSING/+ )++ const wrongBase = manifestFor(images)+ wrongBase.derivedFrom[WORKFLOW_FIXTURE] = { ref: PROXY_BASE, id: images[PROXY_BASE] }+ assert.throws(+ () => certify({ manifest: wrongBase, images: observedEntries(images) }),+ /IMAGE_PROOF_BASE_BINDING_MISSING/+ )++ const staleBase = manifestFor(images)+ staleBase.derivedFrom[PROXY_FIXTURE] = { ref: PROXY_BASE, id: digest('the-base-before') }+ assert.throws(+ () => certify({ manifest: staleBase, images: observedEntries(images) }),+ /IMAGE_PROOF_BASE_MISMATCH/+ )+})diff --git a/scripts/e2e/approved-tools-restoration.test.mjs b/scripts/e2e/approved-tools-restoration.test.mjsnew file mode 100644index 000000000..49d11029c--- /dev/null+++ b/scripts/e2e/approved-tools-restoration.test.mjs@@ -0,0 +1,112 @@+import assert from 'node:assert/strict'+import { test } from 'node:test'+import { restoreOwnedFixture, runnerPhaseBudgetMs } from './prepare-codex-approved-tools.mjs'++const run = 'approved-tools-123456789abc'+function fixture() {+ const state = {+ profile: 'test-profile',+ worktree: 'test-worktree',+ head: 'old',+ run,+ proxyUid: 'owned',+ originalProxyImage: 'clerum/codex-llm-proxy:test',+ originalImagePullPolicy: 'IfNotPresent',+ forwards: [{ id: 1 }, { id: 2 }],+ restored: true,+ }+ const deployment = {+ metadata: { uid: 'owned', resourceVersion: '1' },+ spec: {+ template: {+ metadata: { annotations: { 'evenfire.ai/codex-tools-fixture-run': run } },+ spec: {+ containers: [+ { name: 'codex-llm-proxy', image: 'clerum/codex-approved-tools-proxy-e2e:test' },+ ],+ },+ },+ },+ }+ const events = []+ const options = {+ state,+ profile: state.profile,+ worktree: state.worktree,+ head: 'new',+ getDeployment: () => {+ events.push('read')+ return deployment+ },+ patchProxy: value => {+ events.push('patch')+ assert.deepEqual(value.metadata, deployment.metadata)+ },+ waitProxyRollout: () => events.push('ready'),+ cleanupForward: binding => events.push(`cleanup${binding.id}`),+ }+ return { state, deployment, events, options }+}++test('same owned run restores after a commit change and keeps an audit record', async () => {+ const f = fixture()+ await restoreOwnedFixture(f.options)+ assert.equal(f.state.restored, true)+ assert.deepEqual(f.events, ['read', 'patch', 'ready', 'cleanup1', 'cleanup2'])+ assert.deepEqual(f.state.headAudit, { createdHead: 'old', restoredAtHead: 'new' })+})++for (const mismatch of ['uid', 'run', 'missing-run', 'image']) {+ test(`a ${mismatch} mismatch refuses the proxy write but cleans every owned forward`, async () => {+ const f = fixture()+ if (mismatch === 'uid') f.deployment.metadata.uid = 'foreign'+ if (mismatch === 'run')+ f.deployment.spec.template.metadata.annotations['evenfire.ai/codex-tools-fixture-run'] =+ 'approved-tools-aaaaaaaaaaaa'+ if (mismatch === 'missing-run') f.deployment.spec.template.metadata.annotations = {}+ if (mismatch === 'image')+ f.deployment.spec.template.spec.containers[0].image = 'unrelated/image:test'+ await assert.rejects(() => restoreOwnedFixture(f.options))+ assert.equal(f.state.restored, false)+ assert.deepEqual(f.events, ['read', 'cleanup1', 'cleanup2'])+ })+}++test('rollout failure is not restored and does not prevent forward cleanup', async () => {+ const f = fixture()+ f.options.waitProxyRollout = () => {+ f.events.push('ready')+ throw new Error('not-ready')+ }+ await assert.rejects(() => restoreOwnedFixture(f.options), /not-ready/)+ assert.equal(f.state.restored, false)+ assert.deepEqual(f.events, ['read', 'patch', 'ready', 'cleanup1', 'cleanup2'])+})++test('cleanup failure cannot become success and later bindings are still attempted', async () => {+ const f = fixture()+ f.options.cleanupForward = binding => {+ f.events.push(`cleanup${binding.id}`)+ if (binding.id === 1) throw new Error('cleanup-failed')+ }+ await assert.rejects(() => restoreOwnedFixture(f.options), /cleanup-failed/)+ assert.equal(f.state.restored, false)+ assert.deepEqual(f.events.slice(-2), ['cleanup1', 'cleanup2'])+})++test('foreign profile ownership permits no read, patch or cleanup', async () => {+ const f = fixture()+ f.options.profile = 'other'+ await assert.rejects(() => restoreOwnedFixture(f.options), /ownership mismatch/)+ assert.deepEqual(f.events, [])+})++test('the outer deadline contains all runner phases and remains finite', () => {+ const b = runnerPhaseBudgetMs()+ assert(+ b.runnerTimeoutSeconds * 1000 >=+ b.playwright + b.electronCheck + b.staticAudit + b.playwrightSpawnGrace+ )+ assert(b.runnerOuterSpawnTimeoutMs > b.runnerTimeoutSeconds * 1000)+ assert(b.runnerOuterSpawnTimeoutMs < b.maxDeadlineMs)+})diff --git a/scripts/e2e/install-627-real-tools.mjs b/scripts/e2e/install-627-real-tools.mjsindex b564da44f..a7d123b8c 100644--- a/scripts/e2e/install-627-real-tools.mjs+++ b/scripts/e2e/install-627-real-tools.mjs@@ -12,7 +12,11 @@ if (!profile || context !== profile || !/^clerum-[a-z0-9-]+-[a-f0-9]{8}$/.test(pfunction run(executable, args, input, timeout = 30000) {const result = spawnSync(executable, args, {- cwd: root, input, encoding: 'utf8', timeout, maxBuffer: 4 * 1024 * 1024,+ cwd: root,+ input,+ encoding: 'utf8',+ timeout,+ maxBuffer: 4 * 1024 * 1024,})if (result.error || result.signal || result.status !== 0)throw new Error(`${executable} operation failed; no response bodies are logged`)@@ -24,22 +28,42 @@ function kubectl(args, value) {return run('kubectl', kc(args), value === undefined ? undefined : JSON.stringify(value))}function read(resource, namespace, name) {- const result = kubectl(['-n', namespace, 'get', resource, name, '--ignore-not-found', '-o', 'json'])+ const result = kubectl([+ '-n',+ namespace,+ 'get',+ resource,+ name,+ '--ignore-not-found',+ '-o',+ 'json',+ ])return result.trim() ? parseJson(result, 'Invalid Kubernetes JSON response') : undefined}function parseJson(value, message) {- try { return JSON.parse(value) } catch { throw new Error(message) }+ try {+ return JSON.parse(value)+ } catch {+ throw new Error(message)+ }}function source(repository, file, commit) {- const entry = parseJson(run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]), 'Invalid source response JSON')- if (!entry || typeof entry !== 'object' || Array.isArray(entry) ||- entry.encoding !== 'base64' || typeof entry.content !== 'string')+ const entry = parseJson(+ run('gh', ['api', `repos/keyper-labs/${repository}/contents/${file}?ref=${commit}`]),+ 'Invalid source response JSON'+ )+ if (+ !entry ||+ typeof entry !== 'object' ||+ Array.isArray(entry) ||+ entry.encoding !== 'base64' ||+ typeof entry.content !== 'string'+ )throw new Error('Invalid source content envelope')// GitHub wraps base64 in lines; require canonical encoding after removing only line breaks.const content = entry.content.replace(/[\r\n]/g, '')const decoded = Buffer.from(content, 'base64')- if (decoded.toString('base64') !== content)- throw new Error('Invalid source base64 content')+ if (decoded.toString('base64') !== content) throw new Error('Invalid source base64 content')return { commit, value: parseJson(decoded.toString('utf8'), 'Invalid source document JSON') }}@@ -47,8 +71,16 @@ const host = read('hosts', 'mcp-host', 'chatllm')if (host?.spec?.model?.provider !== 'codex-subscription' || !host.spec.contextRef)throw new Error('chatllm must already use the approved Codex subscription')// Install only the reviewed revisions; upstream HEAD is not an approved baseline.-const pluginSource = source('worktracker-plugin', 'recipe/worktracker.json', '2b2dbd403d817fcfc20b0051a6d8ef637ee072c5')-const wikiSource = source('mcp-servers-collection', 'mcp-wikipedia/registry.json', 'addf570cfcc8b2270b577f446fdb4c5d9d7b2d28')+const pluginSource = source(+ 'worktracker-plugin',+ 'recipe/worktracker.json',+ '2b2dbd403d817fcfc20b0051a6d8ef637ee072c5'+)+const wikiSource = source(+ 'mcp-servers-collection',+ 'mcp-wikipedia/registry.json',+ 'addf570cfcc8b2270b577f446fdb4c5d9d7b2d28'+)const recipe = pluginSource.valueconst expectedImages = new Set(['postgres:16-alpine',@@ -57,34 +89,63 @@ const expectedImages = new Set(['registry.evenfire.ai/evenfire/worktracker-ui:2.2.5',])const workloads = recipe?.spec?.workloads-if (recipe?.kind !== 'WorkflowRecipe' || recipe.metadata?.name !== 'worktracker' ||- !Array.isArray(workloads) || workloads.length !== 4 ||- workloads.some(workload => !expectedImages.has(workload?.image)) ||- new Set(workloads.map(workload => workload.image)).size !== expectedImages.size)+if (+ recipe?.kind !== 'WorkflowRecipe' ||+ recipe.metadata?.name !== 'worktracker' ||+ !Array.isArray(workloads) ||+ workloads.length !== 4 ||+ workloads.some(workload => !expectedImages.has(workload?.image)) ||+ new Set(workloads.map(workload => workload.image)).size !== expectedImages.size+)throw new Error('Worktracker source changed; review its images before installation')-if (wikiSource.value?.version !== '1.0.0' || !Array.isArray(wikiSource.value.tools) || wikiSource.value.tools.length !== 11)+if (+ wikiSource.value?.version !== '1.0.0' ||+ !Array.isArray(wikiSource.value.tools) ||+ wikiSource.value.tools.length !== 11+)throw new Error('Wikipedia catalog changed; review baseline before installation')const labels = { 'evenfire.ai/e2e-suite': 'issue-627-real-baseline' }-recipe.metadata = { ...recipe.metadata, namespace: 'sandbox-recipes', labels: { ...recipe.metadata.labels, ...labels } }+recipe.metadata = {+ ...recipe.metadata,+ namespace: 'sandbox-recipes',+ labels: { ...recipe.metadata.labels, ...labels },+}// Test data stays local; do not start optional meeting ingestion or background enrichment.for (const workload of workloads)for (const entry of workload.env ?? [])- if (['FEATURE_MEETINGS', 'FEATURE_PATTERNS_CRON', 'FEATURE_GITHUB', 'FEATURE_SEMANTIC_SEARCH'].includes(entry.name))+ if (+ [+ 'FEATURE_MEETINGS',+ 'FEATURE_PATTERNS_CRON',+ 'FEATURE_GITHUB',+ 'FEATURE_SEMANTIC_SEARCH',+ ].includes(entry.name)+ )entry.value = 'false'const wikipedia = {- apiVersion: 'clerum.io/v1alpha1', kind: 'McpServer',+ apiVersion: 'clerum.io/v1alpha1',+ kind: 'McpServer',metadata: { name: 'issue-627-wikipedia', namespace: 'mcp-server', labels },spec: {contextRef: host.spec.contextRef,image: 'registry.evenfire.ai/evenfire/mcp-wikipedia:1.0.0',- imagePullPolicy: 'IfNotPresent', enabled: true,+ imagePullPolicy: 'IfNotPresent',+ enabled: true,description: 'Issue 627 real Wikipedia baseline',- transport: { type: 'streamableHttp', port: 3000, url: 'http://issue-627-wikipedia.mcp-server.svc.cluster.local:3000/mcp' },- auth: { type: 'none' }, healthCheck: { port: 3000 },+ transport: {+ type: 'streamableHttp',+ port: 3000,+ url: 'http://issue-627-wikipedia.mcp-server.svc.cluster.local:3000/mcp',+ },+ auth: { type: 'none' },+ healthCheck: { port: 3000 },egressBindings: [{ egressClass: 'public-web' }],- resources: { requests: { cpu: '25m', memory: '64Mi' }, limits: { cpu: '500m', memory: '256Mi' } },+ resources: {+ requests: { cpu: '25m', memory: '64Mi' },+ limits: { cpu: '500m', memory: '256Mi' },+ },env: [{ name: 'PORT', value: '3000' }],},}@@ -92,25 +153,39 @@ const wikipedia = {const shared = randomBytes(32).toString('hex')const database = randomBytes(24).toString('hex')const bundles = ['sandbox-recipes', 'mcp-server'].map(namespace => ({- apiVersion: 'v1', kind: 'Secret', type: 'Opaque',- metadata: { name: 'worktracker-secrets', namespace, labels: { ...labels, 'clerum.io/owner-recipe': 'worktracker' } },- stringData: namespace === 'sandbox-recipes'- ? { 'pg-password': database, 'mcp-api-token': shared }- : { 'mcp-api-token': shared },+ apiVersion: 'v1',+ kind: 'Secret',+ type: 'Opaque',+ metadata: {+ name: 'worktracker-secrets',+ namespace,+ labels: { ...labels, 'clerum.io/owner-recipe': 'worktracker' },+ },+ stringData:+ namespace === 'sandbox-recipes'+ ? { 'pg-password': database, 'mcp-api-token': shared }+ : { 'mcp-api-token': shared },}))const objects = [recipe, ...bundles, wikipedia]for (const object of objects)if (read(object.kind, object.metadata.namespace, object.metadata.name))- throw new Error(`Existing ${object.kind}/${object.metadata.name}; refusing replacement or credential rotation`)+ throw new Error(+ `Existing ${object.kind}/${object.metadata.name}; refusing replacement or credential rotation`+ )// Validate resource shapes before creation. Secrets follow the owning recipe.for (const object of [recipe, wikipedia])kubectl(['create', '--dry-run=server', '-f', '-', '-o', 'name'], object)for (const object of objects) {kubectl(['create', '-f', '-', '-o', 'name'], object)- process.stdout.write(`CREATED ${object.kind}/${object.metadata.namespace}/${object.metadata.name}\n`)+ process.stdout.write(+ `CREATED ${object.kind}/${object.metadata.namespace}/${object.metadata.name}\n`+ )}-process.stdout.write(JSON.stringify({- status: 'resources-created-not-certified', profile,- revisions: { worktracker: pluginSource.commit, wikipedia: wikiSource.commit },- next: 'Verify workloads and effective tools/list; grant connector access through the platform before the real chat test.',-}) + '\n')+process.stdout.write(+ JSON.stringify({+ status: 'resources-created-not-certified',+ profile,+ revisions: { worktracker: pluginSource.commit, wikipedia: wikiSource.commit },+ next: 'Verify workloads and effective tools/list; grant connector access through the platform before the real chat test.',+ }) + '\n'+)diff --git a/scripts/e2e/prepare-codex-approved-tools.mjs b/scripts/e2e/prepare-codex-approved-tools.mjsindex 928602b10..ef4e5a64e 100644--- a/scripts/e2e/prepare-codex-approved-tools.mjs+++ b/scripts/e2e/prepare-codex-approved-tools.mjs@@ -4,11 +4,13 @@ import fs from 'node:fs'import net from 'node:net'import path from 'node:path'import { fileURLToPath, pathToFileURL } from 'node:url'+import { buildApprovedToolsImageProof } from './approved-tools-image-proof.mjs'const repo = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..')const sizes = [83, 150, 250]const fixtureImage = 'clerum/codex-approved-tools-mcp-e2e:test'const proxyImage = 'clerum/codex-approved-tools-proxy-e2e:test'+const proxyRunAnnotation = 'evenfire.ai/codex-tools-fixture-run'const workflowImage = 'clerum/workflow-custom-sdk-e2e:approved-tools-test'const workflowFeatureCheck = `process.stdout.write(JSON.stringify(process.env.WRC_ENABLE_CUSTOM_COORDINATOR_IMAGE === 'true' && process.env.WRC_REQUIRE_COORDINATOR_IMAGE_DIGEST === 'false' && (process.env.WRC_ALLOWED_COORDINATOR_IMAGE_PREFIXES || '').split(',').some(prefix => prefix.trim() && '${workflowImage}'.startsWith(prefix.trim()))))`const proxyFlags = [@@ -17,6 +19,40 @@ const proxyFlags = ['NODE_ENV',]+// Shared deadline budget: the outer prepare runner wraps Electron+// verification, the static audit, and the inner Playwright run (including its+// spawn grace). The outer deadline derives from those phases plus+// scheduling/reporting grace. Every bound stays finite.+export const APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS = 35 * 60_000+export const APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS = 30_000+export const APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS = 30_000+export const APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS = 15_000+export const APPROVED_TOOLS_RUNNER_KILL_GRACE_SECONDS = 5+export const APPROVED_TOOLS_RUNNER_SCHEDULING_GRACE_MS = 60_000+export const APPROVED_TOOLS_RUNNER_TIMEOUT_SECONDS = Math.ceil(+ (APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS ++ APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS ++ APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS ++ APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS ++ APPROVED_TOOLS_RUNNER_SCHEDULING_GRACE_MS) /+ 1000+)+export const APPROVED_TOOLS_RUNNER_OUTER_SPAWN_TIMEOUT_MS =+ APPROVED_TOOLS_RUNNER_TIMEOUT_SECONDS * 1000 + 20_000+export const APPROVED_TOOLS_MAX_DEADLINE_MS = 45 * 60_000+export function runnerPhaseBudgetMs() {+ return {+ playwright: APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS,+ electronCheck: APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS,+ staticAudit: APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS,+ playwrightSpawnGrace: APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS,+ schedulingGrace: APPROVED_TOOLS_RUNNER_SCHEDULING_GRACE_MS,+ runnerTimeoutSeconds: APPROVED_TOOLS_RUNNER_TIMEOUT_SECONDS,+ runnerOuterSpawnTimeoutMs: APPROVED_TOOLS_RUNNER_OUTER_SPAWN_TIMEOUT_MS,+ maxDeadlineMs: APPROVED_TOOLS_MAX_DEADLINE_MS,+ }+}+export function makeScenarios(run, ports, probePort) {if (!/^approved-tools-[a-f0-9]{12}$/.test(run)) throw new Error('Invalid fixture run identity')if (@@ -398,6 +434,23 @@ export function validateKubectlArgs(args, profile) {operation[5] === '-p') {const patch = JSON.parse(operation[6])+ if (+ patch.metadata !== undefined &&+ (Object.keys(patch.metadata).length !== 2 ||+ !/^[a-zA-Z0-9-]{1,128}$/.test(patch.metadata.uid ?? '') ||+ !/^[0-9]{1,32}$/.test(patch.metadata.resourceVersion ?? ''))+ )+ throw new Error('Invalid proxy object binding')+ const annotations = patch?.spec?.template?.metadata?.annotations+ if (+ annotations !== undefined &&+ (Object.keys(annotations).length !== 1 ||+ !(+ annotations[proxyRunAnnotation] === null ||+ /^approved-tools-[a-f0-9]{12}$/.test(annotations[proxyRunAnnotation] ?? '')+ ))+ )+ throw new Error('Invalid proxy run binding')const entries = patch?.spec?.template?.spec?.containersif (!Array.isArray(entries) || entries.length !== 1) throw new Error('Invalid proxy patch')const c = entries[0]@@ -427,8 +480,17 @@ export function validateKubectlArgs(args, profile) {)throw new Error('Invalid proxy environment')const normalized = {+ ...(patch.metadata === undefined+ ? {}+ : {+ metadata: {+ uid: patch.metadata.uid,+ resourceVersion: patch.metadata.resourceVersion,+ },+ }),spec: {template: {+ ...(annotations === undefined ? {} : { metadata: { annotations } }),spec: {containers: [{ name: c.name, image: c.image, imagePullPolicy: c.imagePullPolicy, env: c.env },@@ -493,7 +555,7 @@ function command(operation, args, { input, timeout = 60_000, inherit = false } =key.startsWith('BASH_FUNC_'))delete env[key]- if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > 25 * 60_000)+ if (!Number.isSafeInteger(timeout) || timeout < 1 || timeout > APPROVED_TOOLS_MAX_DEADLINE_MS)throw new Error('Invalid deadline')const options = {cwd: repo,@@ -512,6 +574,20 @@ function command(operation, args, { input, timeout = 60_000, inherit = false } =case 'head':result = spawnSync('git', ['rev-parse', 'HEAD'], options)break+ case 'image-inventory':+ if (args.length !== 0) throw new Error('Unexpected image inventory arguments')+ result = spawnSync(+ 'minikube',+ [+ '--profile',+ validateProfile(required('MINIKUBE_PROFILE')),+ 'image',+ 'ls',+ '--format=json',+ ],+ options+ )+ breakcase 'kubectl':result = spawnSync('kubectl',@@ -546,16 +622,16 @@ function command(operation, args, { input, timeout = 60_000, inherit = false } =['scripts/minikube/run-with-deadline.mjs','--timeout-seconds',- '1500',+ String(APPROVED_TOOLS_RUNNER_TIMEOUT_SECONDS),'--kill-grace-seconds',- '5',+ String(APPROVED_TOOLS_RUNNER_KILL_GRACE_SECONDS),'--label','approved-tools-runner','--',process.execPath,'scripts/e2e/run-codex-approved-tools.mjs',],- { ...options, timeout: 1_520_000 }+ { ...options, timeout: APPROVED_TOOLS_RUNNER_OUTER_SPAWN_TIMEOUT_MS })breakdefault:@@ -599,6 +675,80 @@ async function waitHttp(url, accept) {throw new Error('Fixture readiness deadline exceeded')}+// Independent forward cleanup with strict proxy-write guards.+//+// Profile/worktree ownership is required before any side effect. A HEAD+// change does not block cleanup of our own forwards: when profile and+// worktree still match and the live deployment identity (UID) plus image+// binding are valid, restoration proceeds and the old HEAD is recorded in+// state.headAudit. Proxy patching keeps its UID/image guards; forward cleanup+// uses the ownership-checked cleanup path and runs even when the proxy step+// fails. restored is set only when proxy and every cleanup complete.+export async function restoreOwnedFixture({+ state,+ profile,+ worktree,+ head,+ getDeployment,+ patchProxy,+ waitProxyRollout,+ cleanupForward,+}) {+ if (!state || state.profile !== profile || state.worktree !== worktree)+ throw new Error('Fixture ownership mismatch')+ state.restored = false+ if (state.head !== head) {+ state.headAudit = { createdHead: state.head, restoredAtHead: head }+ }+ let proxyError = null+ try {+ const deployment = await getDeployment()+ const container = deployment?.spec?.template?.spec?.containers?.find(+ c => c.name === 'codex-llm-proxy'+ )+ if (deployment?.metadata?.uid !== state.proxyUid || !container)+ throw new Error('Proxy deployment ownership changed')+ const runBinding = deployment.spec.template.metadata?.annotations?.[proxyRunAnnotation]+ if (runBinding !== undefined && runBinding !== state.run)+ throw new Error('Proxy belongs to another fixture run')+ if (container.image === proxyImage) {+ if (!/^approved-tools-[a-f0-9]{12}$/.test(state.run ?? '') || runBinding !== state.run)+ throw new Error('Proxy fixture run binding is missing')+ const env = proxyFlags.map(name => ({ name, $patch: 'delete' }))+ if (state.originalNodeEnv) env[2] = state.originalNodeEnv+ await patchProxy({+ metadata: {+ uid: deployment.metadata.uid,+ resourceVersion: deployment.metadata.resourceVersion,+ },+ image: state.originalProxyImage,+ imagePullPolicy: state.originalImagePullPolicy,+ env,+ annotations: { [proxyRunAnnotation]: null },+ })+ } else if (container.image !== state.originalProxyImage)+ throw new Error('Refusing to restore an unrelated proxy image')+ else if (runBinding !== undefined)+ throw new Error('Original proxy still has a fixture run binding')+ await waitProxyRollout()+ } catch (error) {+ proxyError = error+ }+ const cleanupErrors = []+ for (const binding of state.forwards ?? []) {+ try {+ await cleanupForward(binding)+ } catch (error) {+ cleanupErrors.push(error)+ }+ }+ if (proxyError || cleanupErrors.length > 0) {+ throw proxyError ?? cleanupErrors[0]+ }+ state.restored = true+ return state+}+export function validateRunDirectory(canonical, evidence) {const root = fs.realpathSync(path.join(canonical, '.local-notes/infra/runs'))if (path.relative(fs.realpathSync(canonical), root) !== '.local-notes/infra/runs')@@ -672,70 +822,69 @@ async function main() {fs.fsyncSync(stateFile.fd)}async function restore() {- if (!state || state.profile !== profile || state.worktree !== repo || state.head !== head)- throw new Error('Fixture ownership mismatch')- const deployment = JSON.parse(- kubectl(['-n', 'control-plane', 'get', 'deployment/codex-llm-proxy', '-o', 'json'])- )- const container = deployment.spec.template.spec.containers.find(- c => c.name === 'codex-llm-proxy'- )- if (deployment.metadata.uid !== state.proxyUid || !container)- throw new Error('Proxy deployment ownership changed')- if (container.image === proxyImage) {- const env = proxyFlags.map(name => ({ name, $patch: 'delete' }))- if (state.originalNodeEnv) env[2] = state.originalNodeEnv- kubectl([- '-n',- 'control-plane',- 'patch',- 'deployment/codex-llm-proxy',- '--type=strategic',- '-p',- JSON.stringify({- spec: {- template: {+ try {+ await restoreOwnedFixture({+ state,+ profile,+ worktree: repo,+ head,+ getDeployment: () =>+ JSON.parse(+ kubectl(['-n', 'control-plane', 'get', 'deployment/codex-llm-proxy', '-o', 'json'])+ ),+ patchProxy: ({ metadata, image, imagePullPolicy, env, annotations }) =>+ kubectl([+ '-n',+ 'control-plane',+ 'patch',+ 'deployment/codex-llm-proxy',+ '--type=strategic',+ '-p',+ JSON.stringify({+ metadata,spec: {- containers: [- {- name: 'codex-llm-proxy',- image: state.originalProxyImage,- imagePullPolicy: state.originalImagePullPolicy,- env,+ template: {+ metadata: { annotations },+ spec: {+ containers: [{ name: 'codex-llm-proxy', image, imagePullPolicy, env }],},- ],+ },},- },- },- }),- ])- kubectl(- [- '-n',- 'control-plane',- 'rollout',- 'status',- 'deployment/codex-llm-proxy',- '--timeout=180s',- ],- { timeout: 190_000 }- )- } else if (container.image !== state.originalProxyImage)- throw new Error('Refusing to restore an unrelated proxy image')- for (const binding of state.forwards) {- owner('pf_owner_cleanup_record', [- binding.record,- profile,- profile,- repo,- binding.namespace,- binding.service,- String(binding.localPort),- String(binding.remotePort),- ])+ }),+ ]),+ waitProxyRollout: () =>+ kubectl(+ [+ '-n',+ 'control-plane',+ 'rollout',+ 'status',+ 'deployment/codex-llm-proxy',+ '--timeout=180s',+ ],+ { timeout: 190_000 }+ ),+ cleanupForward: binding =>+ owner('pf_owner_cleanup_record', [+ binding.record,+ profile,+ profile,+ repo,+ binding.namespace,+ binding.service,+ String(binding.localPort),+ String(binding.remotePort),+ ]),+ })+ save()+ } catch (error) {+ try {+ if (state && state.profile === profile && state.worktree === repo) save()+ } catch {+ // Persisting the HEAD audit must not mask the original restore failure.+ }+ throw error}- state.restored = true- save()}if (action === 'restore') {stateFile = openOwnedFile(evidence, 'fixture-state.json', { writable: true })@@ -814,6 +963,24 @@ async function main() {throw new Error('Workflow fixture image is not permitted by the running Minikube coordinator policy')+ const imageManifest = JSON.parse(+ readOwnedFile(path.join(repo, 'deploy/minikube'), '.image-manifest.json', 4 * 1024 * 1024)+ )+ const inventory = JSON.parse(command('image-inventory', [], { timeout: 30_000 }))+ if (!Array.isArray(inventory)) throw new Error('IMAGE_PROOF_MANIFEST_INVALID: invalid inventory')+ const observedImages = inventory.flatMap(item => {+ if (item.repoTags == null) return []+ if (!Array.isArray(item.repoTags) || typeof item.id !== 'string')+ throw new Error('IMAGE_PROOF_MANIFEST_INVALID: invalid inventory item')+ const id = item.id.startsWith('sha256:') ? item.id : `sha256:${item.id}`+ return item.repoTags.map(ref => ({ ref, id }))+ })+ const imageProof = buildApprovedToolsImageProof({+ profile,+ sourceHead: head,+ manifest: imageManifest,+ images: observedImages,+ })const run = 'approved-tools-' + randomBytes(6).toString('hex')const ports = []while (ports.length < 5) {@@ -843,6 +1010,7 @@ async function main() {if (!container ||container.image === proxyImage ||+ deployment.spec.template.metadata?.annotations?.[proxyRunAnnotation] !== undefined ||(container.env ?? []).some(e => proxyFlags.slice(0, 2).includes(e.name)))throw new Error('Proxy already has test ownership')@@ -853,6 +1021,7 @@ async function main() {profile,worktree: repo,head,+ imageProof,run,scenarios,workflowScenario,@@ -933,8 +1102,13 @@ async function main() {'--type=strategic','-p',JSON.stringify({+ metadata: {+ uid: deployment.metadata.uid,+ resourceVersion: deployment.metadata.resourceVersion,+ },spec: {template: {+ metadata: { annotations: { [proxyRunAnnotation]: state.run } },spec: {containers: [{@@ -1044,7 +1218,7 @@ async function main() {process.env.APPROVED_TOOLS_SCENARIOS = JSON.stringify(scenarios)process.env.APPROVED_TOOLS_WORKFLOW_SCENARIO = JSON.stringify(workflowScenario)command('runner', [], {- timeout: 25 * 60_000,+ timeout: APPROVED_TOOLS_RUNNER_OUTER_SPAWN_TIMEOUT_MS,inherit: true,})await restore()diff --git a/scripts/e2e/run-codex-approved-tools.mjs b/scripts/e2e/run-codex-approved-tools.mjsindex 5177d90ce..b885d18ee 100644--- a/scripts/e2e/run-codex-approved-tools.mjs+++ b/scripts/e2e/run-codex-approved-tools.mjs@@ -3,6 +3,11 @@ import fs from 'node:fs'import path from 'node:path'import { fileURLToPath, pathToFileURL } from 'node:url'import {+ APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS,+ APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS,+ APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS,+ APPROVED_TOOLS_RUNNER_KILL_GRACE_SECONDS,+ APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS,openOwnedFile,readOwnedDescriptor,validateRunDirectory,@@ -20,6 +25,15 @@ export const expectedTitles = [),]+export function runPhaseTimeouts() {+ return {+ playwright: APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS,+ electronCheck: APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS,+ staticAudit: APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS,+ playwrightSpawnGrace: APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS,+ }+}+export function validateReport(report) {const specs = []function collect(suites) {@@ -72,7 +86,11 @@ function required(name) {}function runNode(args, cwd, env, timeout) {- if (!Number.isSafeInteger(timeout) || timeout < 1000 || timeout > 35 * 60_000)+ if (+ !Number.isSafeInteger(timeout) ||+ timeout < 1000 ||+ timeout > APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS+ )throw new Error('Invalid browser deadline')const result = spawnSync(process.execPath,@@ -81,7 +99,7 @@ function runNode(args, cwd, env, timeout) {'--timeout-seconds',String(Math.ceil(timeout / 1000)),'--kill-grace-seconds',- '5',+ String(APPROVED_TOOLS_RUNNER_KILL_GRACE_SECONDS),'--label','approved-tools-playwright','--',@@ -92,7 +110,7 @@ function runNode(args, cwd, env, timeout) {cwd,env,stdio: 'inherit',- timeout: timeout + 15_000,+ timeout: timeout + APPROVED_TOOLS_PLAYWRIGHT_SPAWN_GRACE_MS,killSignal: 'SIGTERM',})@@ -147,7 +165,7 @@ function main() {cwd: path.join(repo, 'desktop-app'),env,stdio: 'inherit',- timeout: 30_000,+ timeout: APPROVED_TOOLS_ELECTRON_CHECK_TIMEOUT_MS,})if (electronCheck.error || electronCheck.signal || electronCheck.status !== 0)throw new Error('Desktop Electron verification failed')@@ -162,7 +180,7 @@ function main() {'tests/e2e/playwright/helpers/approved-tools-workflow.ts','tests/e2e/playwright/helpers/approved-tools-subscription.ts',],- { cwd: repo, env, stdio: 'inherit', timeout: 30_000 }+ { cwd: repo, env, stdio: 'inherit', timeout: APPROVED_TOOLS_STATIC_AUDIT_TIMEOUT_MS })if (audit.error || audit.signal || audit.status !== 0)throw new Error('E2E Guardian audit failed')@@ -174,7 +192,7 @@ function main() {],playwright,env,- 35 * 60_000+ APPROVED_TOOLS_PLAYWRIGHT_TIMEOUT_MS)const verified = validateReport(JSON.parse(readOwnedDescriptor(reportFile)))process.stdout.write(diff --git a/scripts/minikube/build-images.sh b/scripts/minikube/build-images.shindex 74a828e11..f8c30067f 100755--- a/scripts/minikube/build-images.sh+++ b/scripts/minikube/build-images.sh@@ -32,7 +32,12 @@# --only=<svc> Build only the image(s) whose tag matches the substring <svc># (e.g. --only=control-api, --only=workflow-recipes). Skips# public-image pulls and preserves the recorded manifest-# acquisition mode/tag while refreshing image IDs.+# acquisition mode/tag while refreshing image IDs. A ref+# this run does not build is carried forward from the+# previous manifest only while that manifest names this+# profile and the daemon still resolves the ref to the+# recorded ID, with its own source revision preserved (see+# the manifest writer at the bottom).# --include-e2e-fixtures# --verify-only: also demand the two E2E-only fixtures# (workflow-custom-sdk-e2e, workflow-plugin-sdk-e2e) in ghcr@@ -478,6 +483,27 @@ require_inherited_mutation_lease() {bash "${SCRIPT_DIR}/require-t2-mutation-lock.sh"}+# The commit this run's images are built from, recorded in the manifest as+# gitHead.+#+# Without it the manifest can only claim that a ref exists, never which commit+# it was built from, and a reader asking "does this fixture image belong to the+# head under review?" has no answer beyond the image name. --verify-only+# acquires nothing and --public-only loads third-party images only, so neither+# has a source revision to record; every other path refuses to write a manifest+# it cannot stamp.+SOURCE_REVISION=""+resolve_source_revision() {+ local revision+ revision="$(git -C "$PROJECT_DIR" rev-parse --verify HEAD 2>/dev/null || true)"+ if [[ ! "$revision" =~ ^[0-9a-f]{40}$ ]]; then+ printf 'SOURCE_REVISION_UNRESOLVED: %s is not a Git checkout with a resolvable HEAD, so the image manifest could not record the commit these images were built from\n' \+ "$PROJECT_DIR" >&2+ return 1+ fi+ SOURCE_REVISION="$revision"+}+run_with_deadline() {local label="$1" timeout_seconds="$2"shift 2@@ -490,6 +516,9 @@ run_with_deadline() {validate_build_configurationrequire_inherited_mutation_lease+if [[ "$VERIFY_ONLY" == false && "$PUBLIC_ONLY" == false ]]; then+ resolve_source_revision+fiif ! command -v node >/dev/null 2>&1 || [[ ! -f "$DOCKER_CLI_DEADLINE_RUNNER" ]]; thenerr "The existing deadline runner and Node.js are required"@@ -940,6 +969,15 @@ build_section() {}# ---- Build function ----++# Refs this invocation actually built (docker build), and the base each built+# fixture was built FROM. Both feed the manifest: only a built ref may be+# stamped with this run's source revision, and a fixture records the exact base+# image it came from so a reader can prove the pair still matches instead of+# assuming that the base tag it names is still the image it was built from.+BUILT_IMAGE_REFS=()+DERIVED_BASE_RECORDS=()+build_image() {local name=$1 dir=$2 tag=$3 dockerfile=${4:-""}if [ -n "$ONLY_SVC" ] && [[ "$tag" != *"$ONLY_SVC"* ]] && [[ "$name" != *"$ONLY_SVC"* ]]; then@@ -950,7 +988,7 @@ build_image() {returnfilog "Building ${tag}..."- local docker_args=(-t "$tag")+ local docker_args=(-t "$tag") derived_base=""if [ -n "$dockerfile" ]; thendocker_args+=(-f "$dockerfile")fi@@ -958,9 +996,11 @@ build_image() {# The public target builds this production tag immediately beforehand under# the same profile lease. The test image never replaces that tag.docker_args+=(--build-arg CODEX_PROXY_IMAGE=clerum/codex-llm-proxy:test)+ derived_base="clerum/codex-llm-proxy:test"fiif [ "$name" = codex-approved-tools-workflow-e2e ]; thendocker_args+=(--build-arg WORKFLOW_CUSTOM_SDK_IMAGE=clerum/workflow-custom-sdk-e2e:test)+ derived_base="clerum/workflow-custom-sdk-e2e:test"filocal build_cmd=(docker build "${docker_args[@]}" "$dir")local build_status=0@@ -986,6 +1026,10 @@ build_image() {FAILED_IMAGES+=("$tag")return 1fi+ BUILT_IMAGE_REFS+=("$tag")+ if [ -n "$derived_base" ]; then+ DERIVED_BASE_RECORDS+=("${tag}|${derived_base}")+ fiok "${tag} (${sha:7:12})"}@@ -1024,6 +1068,7 @@ build_control_ui_image() {FAILED_IMAGES+=("$tag")return 1fi+ BUILT_IMAGE_REFS+=("$tag")ok "${tag} (${sha:7:12})"}@@ -1365,39 +1410,252 @@ fi# Docker's generic exit 1 must never trigger a pull or be rewritten as# NOT_BUILT. Staging the values also preserves any prior good manifest when an# inventory or inspect command fails partway through the list.-MANIFEST_SHAS=()+#+# One ref's current ID from the daemon this run built in: the digest, or+# nothing when a successful inventory proves the ref absent. A failed read+# prints the message the staging loop has always used and returns the child+# status, so the caller stops instead of rewriting an infrastructure failure as+# absence.+manifest_live_image_id() {+ local ref="$1" status=0 output="" ids=""+ if [[ "$MINIKUBE_MULTI_NODE" == true ]]; then+ output="$(minikube_image_id "$ref")" || status=$?+ if [[ "$status" -ne 0 ]]; then+ err "Could not inventory ${ref} while generating the image manifest" >&2+ return "$status"+ fi+ case "$output" in+ "" | NOT_FOUND) printf '' ;;+ *) printf '%s' "$output" ;;+ esac+ return 0+ fi+ ids="$(docker_local_image_query "$ref")" || status=$?+ if [[ "$status" -ne 0 ]]; then+ err "Could not inventory ${ref} while generating the image manifest" >&2+ return "$status"+ fi+ if [[ -z "$ids" ]]; then+ printf ''+ return 0+ fi+ output="$(docker_local_image_id "$ref")" || status=$?+ if [[ "$status" -ne 0 ]]; then+ err "Could not inspect ${ref} while generating the image manifest" >&2+ return "$status"+ fi+ printf '%s' "$output"+}++# The source revision THIS invocation built a ref at, or nothing when it did+# not build it. Only a build stamps a revision: an image that was merely+# observed keeps whatever the previous manifest recorded for it, and no run may+# stamp an image it did not build with its own head.+manifest_built_source_revision() {+ local ref="$1" built+ for built in ${BUILT_IMAGE_REFS[@]+"${BUILT_IMAGE_REFS[@]}"}; do+ if [[ "$built" == "$ref" ]]; then+ printf '%s' "$SOURCE_REVISION"+ return 0+ fi+ done+ printf ''+}++manifest_ref_is_staged() {+ local ref="$1" staged+ for staged in "${ALL_IMAGES[@]}"; do+ if [[ "$staged" == "$ref" ]]; then+ return 0+ fi+ done+ return 1+}++# ---- Previous manifest records ----+#+# Read once, before staging, because two callers need it: a ref this run+# rebuilds is stamped with this run's source revision, while a ref this run only+# observed keeps the revision the previous manifest recorded for it -- and only+# while that record still describes the same image (same profile, same digest).+# A run never rewrites an old source revision into a new one.+#+# The node reader validates every field it emits, so a hand-edited or truncated+# manifest cannot smuggle a ref, a digest, or a revision into this one.+PRIOR_MANIFEST_PROFILE=""+PRIOR_IMAGE_RECORDS=()+PRIOR_BINDING_RECORDS=()+load_previous_manifest_records() {+ local manifest="$1" status=0 fields="" kind="" a="" b="" c=""+ if [[ ! -f "$manifest" ]]; then+ return 0+ fi+ fields="$(node -e '+const fs = require("node:fs")+const refPattern = /^[A-Za-z0-9][A-Za-z0-9._/:-]*$/+const digestPattern = /^sha256:[0-9a-f]{64}$/+const revisionPattern = /^[0-9a-f]{40}$/+let parsed+try {+ parsed = JSON.parse(fs.readFileSync(process.argv[1], "utf8"))+} catch {+ process.exit(1)+}+if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) process.exit(1)+const profile = typeof parsed.profile === "string" && refPattern.test(parsed.profile) ? parsed.profile : ""+process.stdout.write("meta\t" + profile + "\n")+const images = parsed.images && typeof parsed.images === "object" ? parsed.images : {}+const revisions = parsed.sourceRevisions && typeof parsed.sourceRevisions === "object" ? parsed.sourceRevisions : {}+for (const [ref, id] of Object.entries(images)) {+ if (!refPattern.test(ref) || typeof id !== "string" || !digestPattern.test(id)) continue+ const revision = typeof revisions[ref] === "string" && revisionPattern.test(revisions[ref]) ? revisions[ref] : ""+ process.stdout.write("image\t" + ref + "\t" + id + "\t" + revision + "\n")+}+const derived = parsed.derivedFrom && typeof parsed.derivedFrom === "object" ? parsed.derivedFrom : {}+for (const [ref, entry] of Object.entries(derived)) {+ if (!refPattern.test(ref) || !entry || typeof entry !== "object") continue+ if (typeof entry.ref !== "string" || !refPattern.test(entry.ref)) continue+ if (typeof entry.id !== "string" || !digestPattern.test(entry.id)) continue+ process.stdout.write("base\t" + ref + "\t" + entry.ref + "\t" + entry.id + "\n")+}+' "$manifest")" || status=$?+ if [[ "$status" -ne 0 ]]; then+ warn "Ignoring an unreadable previous manifest; no record is carried forward"+ return 0+ fi+ while IFS=$'\t' read -r kind a b c || [[ -n "$kind" ]]; do+ case "$kind" in+ meta) PRIOR_MANIFEST_PROFILE="$a" ;;+ image) PRIOR_IMAGE_RECORDS+=("${a}|${b}|${c}") ;;+ base) PRIOR_BINDING_RECORDS+=("${a}|${b}|${c}") ;;+ esac+ done <<<"$fields"+ return 0+}++# The revision the previous manifest recorded for a ref, and only while that+# record still describes the digest this profile holds right now.+manifest_previous_source_revision() {+ local ref="$1" id="$2" record rest+ if [[ "$PRIOR_MANIFEST_PROFILE" != "$PROFILE" ]]; then+ printf ''+ return 0+ fi+ for record in ${PRIOR_IMAGE_RECORDS[@]+"${PRIOR_IMAGE_RECORDS[@]}"}; do+ [[ "${record%%|*}" == "$ref" ]] || continue+ rest="${record#*|}"+ [[ "${rest%%|*}" == "$id" ]] || continue+ printf '%s' "${rest#*|}"+ return 0+ done+ printf ''+}++load_previous_manifest_records "$MANIFEST_FILE"++MANIFEST_ENTRY_REFS=()+MANIFEST_ENTRY_SHAS=()+MANIFEST_ENTRY_REVISIONS=()+MANIFEST_BINDING_DERIVED=()+MANIFEST_BINDING_BASES=()+MANIFEST_BINDING_IDS=()if [[ "$MINIKUBE_MULTI_NODE" == true ]]; thencache_minikube_image_inventoryfifor img in "${ALL_IMAGES[@]}"; do- sha=""- if [[ "$MINIKUBE_MULTI_NODE" == true ]]; then- manifest_inventory_status=0- sha="$(minikube_image_id "$img")" || manifest_inventory_status=$?- if [[ "$manifest_inventory_status" -ne 0 ]]; then- err "Could not inventory ${img} while generating the image manifest" >&2- exit "$manifest_inventory_status"+ img_sha="$(manifest_live_image_id "$img")" || exit $?+ if [[ -z "$img_sha" ]]; then+ img_sha="NOT_BUILT"+ elif [[ ! "$img_sha" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then+ err "MINIKUBE_IMAGE_INVENTORY_INVALID: ${img} resolved to '${img_sha}', which is not an image digest" >&2+ exit 1+ fi+ MANIFEST_ENTRY_REFS+=("$img")+ MANIFEST_ENTRY_SHAS+=("$img_sha")+ img_revision="$(manifest_built_source_revision "$img")"+ if [[ -z "$img_revision" ]]; then+ img_revision="$(manifest_previous_source_revision "$img" "$img_sha")"+ fi+ MANIFEST_ENTRY_REVISIONS+=("$img_revision")+done++# A fixture built FROM another image this profile holds records that base and+# the ID it resolved to. Trusting the base TAG alone would let a fixture that+# was built from an older base keep certifying the newer one: the tag is the+# same string either way, the ID is not.+for record in ${DERIVED_BASE_RECORDS[@]+"${DERIVED_BASE_RECORDS[@]}"}; do+ derived="${record%%|*}"+ base="${record#*|}"+ base_id="$(manifest_live_image_id "$base")" || exit $?+ if [[ ! "$base_id" =~ ^sha256:[0-9a-fA-F]{64}$ ]]; then+ err "Could not resolve ${base}, the base image ${derived} was built from, in profile '${PROFILE}'" >&2+ exit 1+ fi+ MANIFEST_BINDING_DERIVED+=("$derived")+ MANIFEST_BINDING_BASES+=("$base")+ MANIFEST_BINDING_IDS+=("$base_id")+done++# ---- Carry verified records forward from the previous manifest ----+#+# This writer replaces the whole file, and a partial run stages only the refs in+# its own ALL_IMAGES. Without carrying, the fixture target -- five --only+# invocations in a row -- would leave a manifest holding only the last fixture,+# and the prepare step that swaps the proxy onto a fixture would have nothing to+# verify against.+#+# Carrying is never a re-stamp. The previous manifest's own source revision+# travels with the record, and only while the record is still true of THIS+# profile: the manifest must name this profile and the daemon must resolve the+# ref to exactly the recorded ID right now. A ref whose image was replaced, or a+# record acquired for another profile, is dropped -- an absent record makes the+# reader fail closed and name the acquisition command, which is the verdict a+# stale fixture deserves. The node reader below validates every field it emits,+# so a hand-edited or truncated manifest cannot smuggle a ref, digest, or+# revision into this one.+carry_previous_manifest_records() {+ local record="" rest="" ref="" id="" revision=""+ local binding="" base="" base_id="" live_id="" live_base_id=""+ if [[ "$PRIOR_MANIFEST_PROFILE" != "$PROFILE" ]]; then+ if [[ -n "$PRIOR_MANIFEST_PROFILE" ]]; then+ log "Not carrying image records acquired for profile '${PRIOR_MANIFEST_PROFILE}'"fi- else- manifest_inventory_status=0- manifest_image_ids="$(docker_local_image_query "$img")" || manifest_inventory_status=$?- if [[ "$manifest_inventory_status" -ne 0 ]]; then- err "Could not inventory ${img} while generating the image manifest" >&2- exit "$manifest_inventory_status"+ return 0+ fi+ for record in ${PRIOR_IMAGE_RECORDS[@]+"${PRIOR_IMAGE_RECORDS[@]}"}; do+ ref="${record%%|*}"+ rest="${record#*|}"+ id="${rest%%|*}"+ revision="${rest#*|}"+ if manifest_ref_is_staged "$ref"; then+ continuefi- if [[ -z "$manifest_image_ids" ]]; then- sha="NOT_BUILT"- else- manifest_inspect_status=0- sha="$(docker_local_image_id "$img")" || manifest_inspect_status=$?- if [[ "$manifest_inspect_status" -ne 0 ]]; then- err "Could not inspect ${img} while generating the image manifest" >&2- exit "$manifest_inspect_status"- fi+ live_id="$(manifest_live_image_id "$ref")" || return $?+ if [[ "$live_id" != "$id" ]]; then+ continuefi- fi- MANIFEST_SHAS+=("$sha")-done+ MANIFEST_ENTRY_REFS+=("$ref")+ MANIFEST_ENTRY_SHAS+=("$id")+ MANIFEST_ENTRY_REVISIONS+=("$revision")+ for binding in ${PRIOR_BINDING_RECORDS[@]+"${PRIOR_BINDING_RECORDS[@]}"}; do+ [[ "${binding%%|*}" == "$ref" ]] || continue+ rest="${binding#*|}"+ base="${rest%%|*}"+ base_id="${rest#*|}"+ live_base_id="$(manifest_live_image_id "$base")" || return $?+ if [[ "$live_base_id" != "$base_id" ]]; then+ log "Dropping the recorded base of ${ref}: ${base} is no longer the image it was built from"+ continue+ fi+ MANIFEST_BINDING_DERIVED+=("$ref")+ MANIFEST_BINDING_BASES+=("$base")+ MANIFEST_BINDING_IDS+=("$base_id")+ done+ done+ return 0+}++carry_previous_manifest_recordsmkdir -p "$(dirname "$MANIFEST_FILE")"echo -e "\n${BOLD}=== Generating Image Manifest ===${NC}"@@ -1407,16 +1665,53 @@ echo -e "\n${BOLD}=== Generating Image Manifest ===${NC}"echo " \"profile\": \"${PROFILE}\","echo " \"imageSource\": \"${MANIFEST_IMAGE_SOURCE}\","echo " \"imageTag\": \"${MANIFEST_IMAGE_TAG}\","+ # Informational only: the commit THIS invocation built from. It certifies+ # nothing about a ref this run did not build -- sourceRevisions below is the+ # per-ref answer, and a reader must decide freshness from that.+ echo " \"gitHead\": \"${SOURCE_REVISION}\","+ revision_total=0+ for image_index in "${!MANIFEST_ENTRY_REFS[@]}"; do+ if [ -n "${MANIFEST_ENTRY_REVISIONS[$image_index]}" ]; then+ revision_total=$((revision_total + 1))+ fi+ done+ if [ "$revision_total" -eq 0 ]; then+ echo " \"sourceRevisions\": {},"+ else+ echo " \"sourceRevisions\": {"+ count=0+ for image_index in "${!MANIFEST_ENTRY_REFS[@]}"; do+ revision="${MANIFEST_ENTRY_REVISIONS[$image_index]}"+ if [ -z "$revision" ]; then+ continue+ fi+ count=$((count + 1))+ comma=","+ if [ "$count" -eq "$revision_total" ]; then comma=""; fi+ echo " \"${MANIFEST_ENTRY_REFS[$image_index]}\": \"${revision}\"${comma}"+ done+ echo " },"+ fi+ if [ "${#MANIFEST_BINDING_DERIVED[@]}" -gt 0 ]; then+ echo " \"derivedFrom\": {"+ binding_total=${#MANIFEST_BINDING_DERIVED[@]}+ count=0+ for binding_index in "${!MANIFEST_BINDING_DERIVED[@]}"; do+ count=$((count + 1))+ comma=","+ if [ "$count" -eq "$binding_total" ]; then comma=""; fi+ echo " \"${MANIFEST_BINDING_DERIVED[$binding_index]}\": { \"ref\": \"${MANIFEST_BINDING_BASES[$binding_index]}\", \"id\": \"${MANIFEST_BINDING_IDS[$binding_index]}\" }${comma}"+ done+ echo " },"+ fiecho " \"images\": {"count=0- total=${#ALL_IMAGES[@]}- for image_index in "${!ALL_IMAGES[@]}"; do- img="${ALL_IMAGES[$image_index]}"- sha="${MANIFEST_SHAS[$image_index]}"+ total=${#MANIFEST_ENTRY_REFS[@]}+ for image_index in "${!MANIFEST_ENTRY_REFS[@]}"; docount=$((count + 1))comma=","if [ "$count" -eq "$total" ]; then comma=""; fi- echo " \"${img}\": \"${sha}\"${comma}"+ echo " \"${MANIFEST_ENTRY_REFS[$image_index]}\": \"${MANIFEST_ENTRY_SHAS[$image_index]}\"${comma}"doneecho " }"echo "}"diff --git a/scripts/minikube/t2.sh b/scripts/minikube/t2.shindex 336c03879..59430ef0b 100755--- a/scripts/minikube/t2.sh+++ b/scripts/minikube/t2.sh@@ -397,8 +397,17 @@ run_healthcheck_if_requested() {fi# Export the already-validated shell value to the child; the CLI argument# intentionally expands that same existing value before the assignment.+ # A user-facing journey may re-enter the canonical mutation wrapper for this+ # profile (for example a `make` fixture target). This parent owns the lease, so+ # pass its exact validated identity to the child and let the nested wrapper+ # validate the inherited token instead of acquiring a second lock for the same+ # profile. T2_SKIP_LOCK stays scoped to this invocation; the nested wrapper+ # still revalidates the token and owner identity and fails closed on a mismatch.# shellcheck disable=SC2097,SC2098T2_HEALTHCHECK_KILL_GRACE_SECONDS="$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \+ T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROFILE="$T2_PROFILE" T2_CONTEXT="$T2_CONTEXT" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" T2_LOCK_ROOT="$T2_LOCK_ROOT" \node "$T2_DEADLINE_RUNNER" \--timeout-seconds "$T2_HEALTHCHECK_TIMEOUT_SECONDS" \--heartbeat-seconds 20 --kill-grace-seconds "$T2_HEALTHCHECK_KILL_GRACE_SECONDS" \@@ -437,7 +446,13 @@ run_playwright_if_requested() {fireturn 0fi- if bash -c "$T2_PLAYWRIGHT_COMMAND"; then+ # See run_healthcheck_if_requested: the journey command may re-enter the+ # canonical mutation wrapper, so scope this parent's validated lease identity+ # to the child and keep T2_SKIP_LOCK out of the parent environment.+ if T2_SKIP_LOCK=true T2_LOCK_TOKEN="$T2_LOCK_TOKEN" \+ T2_PROFILE="$T2_PROFILE" T2_CONTEXT="$T2_CONTEXT" \+ T2_PROJECT_DIR="$T2_PROJECT_DIR" T2_LOCK_ROOT="$T2_LOCK_ROOT" \+ bash -c "$T2_PLAYWRIGHT_COMMAND"; thenT2_PLAYWRIGHT_STATUS=PASSt2_evidence_write Playwright PASS \"user-visible journey command passed; duration=$((SECONDS - phase_started_seconds))s"diff --git a/scripts/tests/test-codex-subscription-t0.sh b/scripts/tests/test-codex-subscription-t0.shindex 0b2dea58e..9ef3a42cd 100755--- a/scripts/tests/test-codex-subscription-t0.sh+++ b/scripts/tests/test-codex-subscription-t0.sh@@ -201,6 +201,8 @@ run_node_group "approved-tools-fixtures-and-runner" \"scripts/e2e/prepare-codex-approved-tools.test.mjs" \"scripts/e2e/run-codex-approved-tools.test.mjs" \"scripts/e2e/desktop-login-seed.test.mjs" \+ "scripts/e2e/approved-tools-image-proof.test.mjs" \+ "scripts/e2e/approved-tools-restoration.test.mjs" \"tests/e2e/fixtures/codex-subscription/approved-tools-workflow/index.test.mjs"diff --git a/scripts/tests/test-minikube-build-section-headers.sh b/scripts/tests/test-minikube-build-section-headers.shindex 06b7fdc03..539e9ebff 100755--- a/scripts/tests/test-minikube-build-section-headers.sh+++ b/scripts/tests/test-minikube-build-section-headers.sh@@ -24,6 +24,17 @@ FAIL=0# condition fails one direction or the other.REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+# shellcheck source=scripts/tests/lib/minikube-fixture-repo.sh+source "$REPO_ROOT/scripts/tests/lib/minikube-fixture-repo.sh"+verify_host_on_exit() {+ local result=$?+ trap - EXIT+ if [[ -n "${MINIKUBE_TEST_HOST_ROOT:-}" ]]; then+ minikube_test_assert_host_unchanged || result=1+ fi+ exit "$result"+}+trap verify_host_on_exit EXITpass() { echo "PASS: $1"; }fail() { echo "FAIL: $1"; FAIL=1; }@@ -98,6 +109,10 @@ STUB# so the script under test is the real one and reads the real manifest.prepare_repo() {local d=$1+ if [[ -n "${MINIKUBE_TEST_HOST_ROOT:-}" ]]; then+ minikube_test_assert_host_unchanged || return 1+ fi+ minikube_test_fixture_repo_init "$REPO_ROOT" "$d" || return 1make_stubs "$d"mkdir -p "$d/repo"cp -R "$REPO_ROOT/deploy" "$d/repo/deploy"diff --git a/scripts/tests/test-minikube-docker-cli-env.sh b/scripts/tests/test-minikube-docker-cli-env.shindex 355b97a98..5f7550cbe 100755--- a/scripts/tests/test-minikube-docker-cli-env.sh+++ b/scripts/tests/test-minikube-docker-cli-env.sh@@ -2,15 +2,22 @@set -euo pipefailROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"+# shellcheck source=scripts/tests/lib/minikube-fixture-repo.sh+source "$ROOT/scripts/tests/lib/minikube-fixture-repo.sh"TMP_DIR="$(mktemp -d)"# Invoked indirectly by the EXIT trap.# shellcheck disable=SC2329cleanup_test_tmp() {+ local result=$?+ if [[ -n "${MINIKUBE_TEST_HOST_ROOT:-}" ]]; then+ minikube_test_assert_host_unchanged || result=1+ fiif [[ "${KEEP_MINIKUBE_DOCKER_TEST_TMP:-false}" == true ]]; thenprintf 'Kept test fixtures at %s\n' "$TMP_DIR" >&2elserm -rf -- "$TMP_DIR"fi+ return "$result"}trap cleanup_test_tmp EXIT@@ -237,6 +244,10 @@ docker_log_has_ambient_runtime_operation() {prepare_fixture_repo() {local fixture="$1"+ if [[ -n "${MINIKUBE_TEST_HOST_ROOT:-}" ]]; then+ minikube_test_assert_host_unchanged+ fi+ minikube_test_fixture_repo_init "$ROOT" "$(dirname "$fixture")"mkdir -p "$fixture/scripts/minikube" "$fixture/scripts/release" \"$fixture/control-api" "$fixture/deploy/minikube"cp "$ROOT/scripts/minikube/build-images.sh" \@@ -394,7 +405,7 @@ assert_all_original_docker_env_is_restored() {}assert_real_build_script_isolated() {- local fixture="$TMP_DIR/repo" output="$TMP_DIR/build.out"+ local fixture="$TMP_DIR/repo/repo" output="$TMP_DIR/build.out"prepare_fixture_repo "$fixture": >"$DOCKER_LOG"if DOCKER_CONFIG="$AMBIENT_CONFIG" MINIKUBE_PRELOAD_BASE_IMAGES=false \@@ -422,7 +433,7 @@ assert_real_build_script_isolated() {}assert_public_pulls_are_isolated() {- local fixture="$TMP_DIR/public-repo" output="$TMP_DIR/public.out"+ local fixture="$TMP_DIR/public-repo/repo" output="$TMP_DIR/public.out"prepare_fixture_repo "$fixture": >"$DOCKER_LOG"if DOCKER_CONFIG="$AMBIENT_CONFIG" MINIKUBE_PRELOAD_BASE_IMAGES=false \@@ -443,7 +454,7 @@ assert_public_pulls_are_isolated() {assert_local_image_operations_preserve_status() {local fixture output status- fixture="$TMP_DIR/base-inspect-failure-repo"+ fixture="$TMP_DIR/base-inspect-failure-repo/repo"output="$TMP_DIR/base-inspect-failure.out"prepare_fixture_repo "$fixture": >"$DOCKER_LOG"@@ -462,7 +473,7 @@ assert_local_image_operations_preserve_status() {fail "base-image inventory status/fallback contract failed (status=$status)"fi- fixture="$TMP_DIR/public-query-failure-repo"+ fixture="$TMP_DIR/public-query-failure-repo/repo"output="$TMP_DIR/public-query-failure.out"prepare_fixture_repo "$fixture": >"$DOCKER_LOG"@@ -481,7 +492,7 @@ assert_local_image_operations_preserve_status() {fail "docker images query status/fallback contract failed (status=$status)"fi- fixture="$TMP_DIR/post-build-inspect-failure-repo"+ fixture="$TMP_DIR/post-build-inspect-failure-repo/repo"output="$TMP_DIR/post-build-inspect-failure.out"prepare_fixture_repo "$fixture"status=0@@ -499,7 +510,7 @@ assert_local_image_operations_preserve_status() {fail "post-build docker inspect status contract failed (status=$status)"fi- fixture="$TMP_DIR/manifest-inspect-failure-repo"+ fixture="$TMP_DIR/manifest-inspect-failure-repo/repo"output="$TMP_DIR/manifest-inspect-failure.out"prepare_fixture_repo "$fixture"status=0@@ -518,7 +529,7 @@ assert_local_image_operations_preserve_status() {fail "manifest docker inspect status contract failed (status=$status)"fi- fixture="$TMP_DIR/tag-failure-repo"+ fixture="$TMP_DIR/tag-failure-repo/repo"output="$TMP_DIR/tag-failure.out"prepare_fixture_repo "$fixture"status=0@@ -538,7 +549,7 @@ assert_local_image_operations_preserve_status() {}assert_local_minikube_load_timeout_kills_descendants() {- local fixture="$TMP_DIR/load-timeout-repo" output="$TMP_DIR/load-timeout.out"+ local fixture="$TMP_DIR/load-timeout-repo/repo" output="$TMP_DIR/load-timeout.out"local status=0 descendant=""prepare_fixture_repo "$fixture": >"$MINIKUBE_LOG"@@ -617,7 +628,7 @@ assert_startup_probes_are_isolated_and_bounded() {}assert_verify_inventory_is_read_only_and_bounded() {- local fixture="$TMP_DIR/verify-repo" output="$TMP_DIR/verify-timeout.out"+ local fixture="$TMP_DIR/verify-repo/repo" output="$TMP_DIR/verify-timeout.out"local status=0 descendant=""prepare_fixture_repo "$fixture"rm -f -- "$DESCENDANT_PID_FILE"@@ -680,7 +691,7 @@ assert_invalid_deadline_and_explicit_buildx_fail_closed() {}assert_private_registry_requires_explicit_config() {- local fixture="$TMP_DIR/private-repo" output="$TMP_DIR/private-missing.out" status=0+ local fixture="$TMP_DIR/private-repo/repo" output="$TMP_DIR/private-missing.out" status=0prepare_fixture_repo "$fixture": >"$DOCKER_LOG"unset MINIKUBE_DOCKER_AUTH_CONFIG FAKE_EXPECT_AUTH_CONFIGdiff --git a/scripts/tests/test-minikube-mutation-boundary.sh b/scripts/tests/test-minikube-mutation-boundary.shindex 490b5c6e6..98214483a 100755--- a/scripts/tests/test-minikube-mutation-boundary.sh+++ b/scripts/tests/test-minikube-mutation-boundary.sh@@ -265,6 +265,45 @@ NODEprintf 'PASS: WRC direct entry, pending journal and dedicated recovery enforce the real inherited lease\n'}+# The #627 real-tool targets are the only mutation targets whose recipes call a+# reviewed external builder directly. They must forward the same explicit+# profile/context/lease passthrough as every sibling target; without it a+# makefile-variable profile (`MINIKUBE_PROFILE ?=`, never exported) never+# reaches the wrapper and the transition fails before it can run.+assert_627_targets_forward_the_profile_and_lease() {+ local target plan problem=""+ for target in \+ minikube-install-627-real-tools \+ minikube-build-627-worktracker \+ minikube-build-627-github; do+ # Inspect the rendered boundary with a known Make value. Explicit recipe+ # assignments also support values defined by included Makefiles, which are+ # not automatically exported like command-line assignments are.+ plan="$(dry_run_make "${target}" MINIKUBE_PROFILE="${PROFILE}" 2>&1)" || {+ printf 'FAIL: dry-run %s failed:\n%s\n' "${target}" "${plan}" >&2+ exit 1+ }+ [[ "${plan}" == *"with-t2-mutation-lock.sh"* ]] \+ || problem+="${target} is not enclosed by the mutation wrapper; "+ [[ "${plan}" == *"T2_PROFILE=\"${PROFILE}\""* ]] \+ || problem+="${target} does not forward T2_PROFILE; "+ [[ "${plan}" == *"T2_CONTEXT=\"${PROFILE}\""* ]] \+ || problem+="${target} does not forward T2_CONTEXT; "+ [[ "${plan}" == *"T2_PROJECT_DIR="* ]] \+ || problem+="${target} does not forward T2_PROJECT_DIR; "+ [[ "${plan}" == *"T2_SKIP_LOCK="* ]] \+ || problem+="${target} does not forward T2_SKIP_LOCK; "+ [[ "${plan}" == *"T2_LOCK_TOKEN="* ]] \+ || problem+="${target} does not forward T2_LOCK_TOKEN; "+ done+ if [[ -n "${problem}" ]]; then+ printf 'FAIL: #627 mutation targets: %s\n' "${problem}" >&2+ exit 1+ fi+ printf 'PASS: the #627 real-tool targets forward the explicit profile, context and inherited lease\n'+}++run_child[[ "$(wc -l <"${LOG}" | tr -d ' ')" -eq 1 ]] || {echo 'FAIL: valid inherited lease did not reach the child boundary' >&2@@ -296,6 +335,7 @@ fi}assert_wrc_admission+assert_627_targets_forward_the_profile_and_leasefull_plan="$(dry_run_make minikube-build-images 2>&1)"full_body_plan="$(dry_run_make minikube-build-images-body 2>&1)"diff --git a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.mdindex de4d44602..d57f70efb 100644--- a/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md+++ b/tests/e2e/fixtures/codex-subscription/approved-tools-setup/README.md@@ -21,7 +21,9 @@ All commands use the verified `MINIKUBE_PROFILE`, matching explicitthe normal Codex proxy, the deterministic proxy fixture, the MCP fixture,the custom-coordinator SDK base, and the approved-tools workflow variant.Fixture tags do not replace production image tags. Run before reconcile- because image acquisition updates the manifest timestamp.+ because image acquisition updates the manifest timestamp. Preparation checks+ the live image IDs, each image's recorded source revision, and both derived+ base-image bindings before changing cluster resources.2. Complete the supported profile reconcile and exact-HEAD validation sequence.3. `make minikube-run-codex-approved-tools` prepares isolated resources, runs thevisible Playwright runner and restores the original proxy image/environment.@@ -53,7 +55,11 @@ Additional fixture ports are randomly allocated once per fresh run, persisted inits metadata, and registered with `port-forward-owner.sh` in the profile's ownedPID directory. A collision fails; no port mapping is regenerated. Readiness probesrevalidate the same live process identity. Existing profile `ports.env` stays-unchanged. Restoration rejects foreign worktrees, HEADs, deployments and images.+unchanged. Restoration rejects foreign worktrees, deployments, images and+fixture-run markers. A later commit in the same owned worktree is recorded in+the restoration audit; it does not prevent cleanup of verified owned forwards.+The proxy update checks its live object version and run marker. A refused proxy+restore still attempts owned-forward cleanup and never reports restored=true.Scenario metadata is written to `scenarios.json` and passed directly to the runnerin `run` mode. It contains no credentials. State records retain only fixturediff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex dbdce48ef..5c765e049 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.75",+ "version": "1.0.76","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.75",+ "version": "1.0.76","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex b46641e95..188471d1d 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.75",+ "version": "1.0.76","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",