Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
revert: "test(e2e): add the minikube legacy processing lease restart lane"
Alfredo Lopez3bfa59ae60d7
diff --git a/Makefile b/Makefileindex 46dbb3fcc..b74d68007 100644--- a/Makefile+++ b/Makefile@@ -321,25 +321,6 @@ minikube-run-document-upload: ## Run the visible Desktop document-upload journeyIMAGE_CAPABILITIES_LANE=document-upload \bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run-.PHONY: minikube-run-legacy-lease-restart minikube-run-legacy-lease-restart-vacuity-minikube-run-legacy-lease-restart: ## Run the legacy processing-lease restart scenarios (issue #1022) with the Desktop shell+GFS download journey and restore the Host- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \- IMAGE_CAPABILITIES_LANE=legacy-lease-restart \- bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run--minikube-run-legacy-lease-restart-vacuity: ## Prove the legacy-lease lane fails on the pre-fix Host (base 74e0d81d9): S-crash must stop at the shell with download_busy- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \- IMAGE_CAPABILITIES_LANE=legacy-lease-restart-vacuity \- bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run--.PHONY: minikube-build-legacy-lease-vacuity-fixture-minikube-build-legacy-lease-vacuity-fixture: ## Build the pre-fix Host (base 74e0d81d9) and its provider-fixture layer for the legacy-lease vacuity lane under the owned profile lease- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \- bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-legacy-lease-vacuity-image.sh-minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.tsindex 23dec419f..d70615959 100644--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts@@ -57,14 +57,6 @@ export const FIXTURE_RESPONSE_KIND = {// delivered.documentReadRequested: 'document-read-requested',documentAnswer: 'document-answer',- // Issue #1022: the legacy processing-lease restart journey. shell_exec first,- // then clerum__gfs_download, then an answer built from the download receipt;- // a failed step is answered with the store code it reported.- legacyLeaseShellRequested: 'legacy-lease-shell-requested',- legacyLeaseDownloadRequested: 'legacy-lease-download-requested',- legacyLeaseAnswer: 'legacy-lease-answer',- legacyLeaseShellFailed: 'legacy-lease-shell-failed',- legacyLeaseDownloadFailed: 'legacy-lease-download-failed',} as const/**@@ -93,33 +85,6 @@ export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze(['document-tool-output-unreadable','document-page-range-malformed','document-stream-unsupported',- 'legacy-lease-shell-not-offered',- 'legacy-lease-download-not-offered',- 'legacy-lease-tool-result-unexpected',- 'legacy-lease-tool-result-count',- 'legacy-lease-download-result-malformed',- 'legacy-lease-stream-unsupported',-])--/**- * Store failure codes a legacy-lease failure row may carry (the- * `GfsDownloadStoreError` codes plus `unrecognized`). Declared independently of- * the fixture; `test/imageCapabilityEvidence.test.ts` checks the lists agree.- */-export const FIXTURE_LEGACY_LEASE_FAILURE_CODES: readonly string[] = Object.freeze([- 'caller_mismatch',- 'caller_quota_exceeded',- 'corrupt_store_ledger',- 'download_expired',- 'download_missing',- 'download_busy',- 'host_quota_exceeded',- 'publication_cancelled',- 'storage_write_failed',- 'unsupported_store_schema',- 'workspace_unavailable',- 'writer_locked',- 'unrecognized',])/** The fixture's text-only answer. It names no image and no color. */@@ -347,12 +312,6 @@ export interface FixtureProviderAttempt {truncated?: boolean/** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */reason?: string- /** One of {@link FIXTURE_LEGACY_LEASE_FAILURE_CODES}. Legacy-lease failure rows only. */- failureCode?: string- /** sha256 the download receipt reported. Legacy-lease answer row only. */- downloadSha256?: string- /** Byte count the download receipt reported. Legacy-lease answer row only. */- downloadBytes?: number}export interface FixtureEvidenceCounters {@@ -367,10 +326,6 @@ export interface FixtureEvidenceCounters {documentReadRequests: numberdocumentAnswers: numberdocumentFailures: number- legacyLeaseShellRequests: number- legacyLeaseDownloadRequests: number- legacyLeaseAnswers: number- legacyLeaseFailures: number}export interface ImageCapabilityEvidenceSnapshot {@@ -397,10 +352,6 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = ['documentReadRequests','documentAnswers','documentFailures',- 'legacyLeaseShellRequests',- 'legacyLeaseDownloadRequests',- 'legacyLeaseAnswers',- 'legacyLeaseFailures',]function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {@@ -483,53 +434,6 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu)}- // failureCode is present on every legacy-lease failure row and on no other row.- const legacyFailure =- responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed ||- responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadFailed- if (legacyFailure) {- const failureCode = value.failureCode- if (- typeof failureCode !== 'string' ||- !FIXTURE_LEGACY_LEASE_FAILURE_CODES.includes(failureCode)- ) {- throw new Error(- `${source}: attempts[${index}].failureCode must be one of ` +- `${FIXTURE_LEGACY_LEASE_FAILURE_CODES.join(', ')} on a ${responseKind} row`- )- }- row.failureCode = failureCode- } else if ('failureCode' in value) {- throw new Error(- `${source}: attempts[${index}].failureCode is only valid on a legacy-lease failure row`- )- }-- // downloadSha256 and downloadBytes are present on every legacy-lease answer row only.- if (responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer) {- const downloadSha256 = value.downloadSha256- const downloadBytes = value.downloadBytes- if (typeof downloadSha256 !== 'string' || !SHA256_HEX.test(downloadSha256)) {- throw new Error(- `${source}: attempts[${index}].downloadSha256 must be a sha256 hex digest on a ` +- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`- )- }- if (!isByteCount(downloadBytes)) {- throw new Error(- `${source}: attempts[${index}].downloadBytes must be a non-negative integer on a ` +- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`- )- }- row.downloadSha256 = downloadSha256- row.downloadBytes = downloadBytes- } else if ('downloadSha256' in value || 'downloadBytes' in value) {- throw new Error(- `${source}: attempts[${index}].downloadSha256 and downloadBytes are only valid on a ` +- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`- )- }-// Present only on document rows; an image row must not grow the field.if ('documentSha256' in value) {const documentSha256 = value.documentSha256@@ -719,10 +623,7 @@ export function appendedAttempts(current.reason !== row.reason ||current.byteRange?.offset !== row.byteRange?.offset ||current.byteRange?.length !== row.byteRange?.length ||- current.truncated !== row.truncated ||- current.failureCode !== row.failureCode ||- current.downloadSha256 !== row.downloadSha256 ||- current.downloadBytes !== row.downloadBytes+ current.truncated !== row.truncated) {throw new Error(`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`diff --git a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.tsdeleted file mode 100644index 24f3df659..000000000--- a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts+++ /dev/null@@ -1,892 +0,0 @@-// desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts-//-// E2E_GUARDIAN_IPC_FLOW: this module drives no browser transition. It owns the-// cluster side of the legacy processing-lease restart lane (issue #1022): the-// workload stop/start scenarios, the seed of one legacy lease on the Host PVC,-// and read-only readers for the three signals the spec asserts after each-// scenario. The Desktop journey itself lives in the spec.-//-// Every kubectl call names the validated branch profile as its context and is-// bounded by a timeout. Arguments travel as a vector, never through a shell.-// Waits poll a live condition with a deadline; none of them is a fixed sleep.-import { execFileSync } from 'node:child_process'-import { randomUUID } from 'node:crypto'--const PROFILE_PATTERN = /^clerum-.+-[a-f0-9]{8}$/-const RUN_ID_PATTERN = /^image-capabilities-[a-f0-9]{12}$/-const SHA256_HEX = /^[a-f0-9]{64}$/--export const LEGACY_LEASE_LANE_MODES = ['fixed', 'vacuity'] as const-export type LegacyLeaseLaneMode = (typeof LEGACY_LEASE_LANE_MODES)[number]--/** The Host whose PVC carries the GFS download store (the fixture lane's Host). */-export const HOST = {- namespace: 'mcp-host',- deployment: 'chatllm',- container: 'mcp-host',- selector: 'app=chatllm',-} as const-export const HCC = {- namespace: 'control-plane',- deployment: 'host-context-controller',- selector: 'app=host-context-controller',-} as const-export const WRC = {- namespace: 'control-plane',- deployment: 'workflow-recipes',- selector: 'app=workflow-recipes',-} as const-export const GFS_WORKLOADS = [- {- namespace: 'gfs',- deployment: 'gfsc-writer',- selector: 'app=gfs-controller,clerum.io/gfsc-role=writer',- },- {- namespace: 'gfs',- deployment: 'gfsc-reader',- selector: 'app=gfs-controller,clerum.io/gfsc-role=reader',- },-] as const--/** The store ledger on the Host PVC, mounted at /workspace. */-export const STORE_LEDGER_PATH = '/workspace/.gfs-download-store/ledger-v1.json'-export const LEGACY_DISCARD_COUNTER = 'clerum_gfs_legacy_processing_leases_discarded_total'-const STORE_LOG_COMPONENT = 'GfsDownloadStore'-const DISCARD_MSG_PREFIX = 'GFS download store discarded '-const UNKNOWN_MSG_PREFIX = 'GFS download store could not confirm the discard of '--export const SEED_LABEL_KEY = 'evenfire.ai/legacy-lease-seed'--export interface LegacyLeaseLaneEnv {- profile: string- mode: LegacyLeaseLaneMode- runId: string-}--/** Reads and validates the runner-owned bindings of this lane. */-export function requireLegacyLeaseLaneEnv(- env: NodeJS.ProcessEnv = process.env-): LegacyLeaseLaneEnv {- const problems: string[] = []- const profile = (env.MINIKUBE_PROFILE ?? '').trim()- if (!PROFILE_PATTERN.test(profile))- problems.push(`MINIKUBE_PROFILE is not a branch profile ("${profile}")`)- if ((env.CONTROL_API_REAL_PG_CONTEXT ?? '').trim() !== profile)- problems.push('CONTROL_API_REAL_PG_CONTEXT must equal MINIKUBE_PROFILE')- if ((env.E2E_K8S_CONTEXT ?? '').trim() !== profile)- problems.push('E2E_K8S_CONTEXT must equal MINIKUBE_PROFILE (the GFS fixture helpers read it)')- const mode = (env.LEGACY_LEASE_LANE_MODE ?? '').trim()- if (!(LEGACY_LEASE_LANE_MODES as readonly string[]).includes(mode))- problems.push(`LEGACY_LEASE_LANE_MODE must be fixed or vacuity ("${mode}")`)- const runId = (env.LEGACY_LEASE_SEED_LABEL_VALUE ?? '').trim()- if (!RUN_ID_PATTERN.test(runId) || runId !== (env.IMAGE_CAPABILITIES_RUN_ID ?? '').trim())- problems.push('LEGACY_LEASE_SEED_LABEL_VALUE must equal IMAGE_CAPABILITIES_RUN_ID')- if (problems.length > 0) {- throw new Error(- 'legacy-lease-restart lane is not configured:\n' +- problems.map(problem => ` - ${problem}`).join('\n') +- '\nRun it through `make minikube-run-legacy-lease-restart` (or the -vacuity target).'- )- }- return { profile, mode: mode as LegacyLeaseLaneMode, runId }-}--export type Kubectl = (args: string[], options?: { timeoutMs?: number; input?: string }) => string--export function kubectlFor(profile: string): Kubectl {- if (!PROFILE_PATTERN.test(profile)) throw new Error(`refusing kubectl for "${profile}"`)- return (args, options = {}) => {- const timeoutMs = options.timeoutMs ?? 30_000- try {- return execFileSync(- 'kubectl',- ['--context', profile, `--request-timeout=${Math.ceil(timeoutMs / 1000)}s`, ...args],- {- encoding: 'utf8',- timeout: timeoutMs + 5_000,- input: options.input,- maxBuffer: 16 * 1024 * 1024,- stdio: ['pipe', 'pipe', 'pipe'],- }- )- } catch (error) {- const stderr =- typeof (error as { stderr?: unknown }).stderr === 'string'- ? (error as { stderr: string }).stderr.trim()- : ''- throw new Error(`kubectl ${args.join(' ')} failed: ${(error as Error).message}\n${stderr}`)- }- }-}--/** Polls `probe` until it returns a value, or fails with the last observation. */-export async function waitFor<T>(- label: string,- timeoutMs: number,- probe: () => { done: true; value: T } | { done: false; observed: string }-): Promise<T> {- const deadline = Date.now() + timeoutMs- let observed = 'not yet observed'- for (;;) {- const result = probe()- if (result.done) return result.value- observed = result.observed- if (Date.now() >= deadline)- throw new Error(`${label}: not reached within ${timeoutMs} ms (${observed})`)- await new Promise(resolve => setTimeout(resolve, 500))- }-}--export interface PodState {- name: string- uid: string- phase: string- terminating: boolean- ready: boolean- /** A container of this pod has started running at least once. */- started: boolean- image: string-}--export function readPods(kubectl: Kubectl, namespace: string, selector: string): PodState[] {- const list = JSON.parse(- kubectl(['-n', namespace, 'get', 'pod', '-l', selector, '-o', 'json'])- ) as {- items: Array<{- metadata: { name: string; uid: string; deletionTimestamp?: string }- spec: { containers: Array<{ image: string }> }- status: {- phase?: string- conditions?: Array<{ type: string; status: string }>- containerStatuses?: Array<{- started?: boolean- state?: { running?: unknown; terminated?: unknown }- lastState?: { running?: unknown; terminated?: unknown }- }>- }- }>- }- return list.items.map(item => ({- name: item.metadata.name,- uid: item.metadata.uid,- phase: item.status.phase ?? 'Unknown',- terminating: Boolean(item.metadata.deletionTimestamp),- ready: (item.status.conditions ?? []).some(c => c.type === 'Ready' && c.status === 'True'),- started: (item.status.containerStatuses ?? []).some(- c =>- c.started === true ||- Boolean(c.state?.running) ||- Boolean(c.state?.terminated) ||- Boolean(c.lastState?.running) ||- Boolean(c.lastState?.terminated)- ),- image: item.spec.containers[0]?.image ?? '',- }))-}--function describePods(pods: PodState[]): string {- return pods.length === 0- ? 'no pods'- : pods- .map(- p =>- `${p.name}(${p.uid.slice(0, 8)} ${p.phase}${p.terminating ? ' terminating' : ''}${p.ready ? ' ready' : ''})`- )- .join(', ')-}--export function deploymentReplicas(kubectl: Kubectl, namespace: string, name: string): number {- const replicas = JSON.parse(kubectl(['-n', namespace, 'get', 'deployment', name, '-o', 'json']))- .spec.replicas- if (!Number.isSafeInteger(replicas)) throw new Error(`${namespace}/${name} has no spec.replicas`)- return replicas as number-}--export function scale(kubectl: Kubectl, namespace: string, name: string, replicas: number): void {- kubectl(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])-}--export function rolloutStatus(kubectl: Kubectl, namespace: string, name: string): void {- kubectl(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=180s'], {- timeoutMs: 190_000,- })-}--export function rolloutRestart(kubectl: Kubectl, namespace: string, name: string): void {- kubectl(['-n', namespace, 'rollout', 'restart', `deployment/${name}`])-}--export async function waitNoPods(- kubectl: Kubectl,- namespace: string,- selector: string,- label: string-): Promise<void> {- await waitFor(`${label} has no pods`, 120_000, () => {- const pods = readPods(kubectl, namespace, selector)- return pods.length === 0- ? { done: true, value: undefined }- : { done: false, observed: describePods(pods) }- })-}--/** Exactly one Ready, non-terminating pod whose UID is not in `previous`. */-export async function waitSingleNewReadyPod(- kubectl: Kubectl,- namespace: string,- selector: string,- previous: ReadonlySet<string>,- label: string,- timeoutMs = 300_000-): Promise<PodState> {- return waitFor(`${label} has one new Ready pod`, timeoutMs, () => {- const pods = readPods(kubectl, namespace, selector)- const [only] = pods- if (pods.length === 1 && only && only.ready && !only.terminating && !previous.has(only.uid))- return { done: true, value: only }- return { done: false, observed: describePods(pods) }- })-}--export interface PvcIdentity {- name: string- uid: string-}--/** The PVC the Host deployment mounts as its workspace, by claim name and UID. */-export function hostPvcIdentity(kubectl: Kubectl): PvcIdentity {- const deployment = JSON.parse(- kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])- ) as {- spec: {- template: {- spec: { volumes?: Array<{ name: string; persistentVolumeClaim?: { claimName: string } }> }- }- }- }- const claims = (deployment.spec.template.spec.volumes ?? [])- .filter(volume => volume.name === 'workspace' && volume.persistentVolumeClaim)- .map(volume => volume.persistentVolumeClaim!.claimName)- if (claims.length !== 1) throw new Error(`Host deployment mounts ${claims.length} workspace PVCs`)- const name = claims[0]!- const uid = kubectl([- '-n',- HOST.namespace,- 'get',- 'pvc',- name,- '-o',- 'jsonpath={.metadata.uid}',- ]).trim()- if (!uid) throw new Error(`PVC ${name} has no UID`)- return { name, uid }-}--// ----------------------------------------------------------------------------// Seeding one legacy lease on the stopped Host's PVC.-// -----------------------------------------------------------------------------export interface LegacyLeaseRecord {- leaseId: string- callerIdentity: string- recordIds: string[]- acquiredAt: string- expiresAt: string-}--/**- * A valid, non-expired lease owned by a foreign caller, with no writer session- * (written by an earlier boot) and no records: exactly the shape a pre-#1019- * shell left behind when its Host died mid-command.- */-export function legacyLease(now = Date.now()): LegacyLeaseRecord {- const leaseId = randomUUID()- return {- leaseId,- callerIdentity: `legacy-lease-e2e-foreign-${leaseId.slice(0, 8)}`,- recordIds: [],- acquiredAt: new Date(now - 60_000).toISOString(),- expiresAt: new Date(now + 60 * 60_000).toISOString(),- }-}--/**- * Runs inside the seed pod as the Host's own uid/gid. Read-modify-write with- * the store's own publication sequence (temp file 0600, fsync, rename, fsync of- * the directory), then a re-read that must show the lease and the unchanged- * record set. It refuses a ledger that already carries leases.- */-const SEED_SCRIPT = `-const fs = require('node:fs')-const path = require('node:path')-const { randomUUID } = require('node:crypto')-const ledgerPath = ${JSON.stringify(STORE_LEDGER_PATH)}-const lease = JSON.parse(process.env.LEGACY_LEASE)-const before = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))-if (before.schemaVersion !== 1) throw new Error('ledger schemaVersion is not 1')-if (before.processingLeases !== undefined) throw new Error('ledger already carries processingLeases')-const recordIds = Object.keys(before.records || {}).sort()-const next = { ...before, processingLeases: { [lease.leaseId]: lease } }-const temporary = ledgerPath + '.tmp-' + randomUUID()-const fd = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)-try { fs.writeSync(fd, JSON.stringify(next)); fs.fsyncSync(fd) } finally { fs.closeSync(fd) }-fs.renameSync(temporary, ledgerPath)-const dir = fs.openSync(path.dirname(ledgerPath), 'r')-try { fs.fsyncSync(dir) } finally { fs.closeSync(dir) }-const after = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))-const afterIds = Object.keys(after.records || {}).sort()-const seeded = after.processingLeases && after.processingLeases[lease.leaseId]-if (!seeded || JSON.stringify(seeded) !== JSON.stringify(lease)) throw new Error('seeded lease not read back')-if (JSON.stringify(afterIds) !== JSON.stringify(recordIds)) throw new Error('record set changed')-const info = fs.statSync(ledgerPath)-process.stdout.write('LEGACY_LEASE_SEED ' + JSON.stringify({ leaseId: lease.leaseId, records: afterIds.length, mode: (info.mode & 0o777).toString(8), uid: info.uid, gid: info.gid }) + '\\n')-`--export interface SeedSummary {- leaseId: string- records: number- mode: string- uid: number- gid: number-}--export function parseSeedSummary(logs: string, leaseId: string): SeedSummary {- const lines = logs.split('\n').filter(line => line.startsWith('LEGACY_LEASE_SEED '))- if (lines.length !== 1) throw new Error(`seed pod printed ${lines.length} summaries`)- const summary = JSON.parse(lines[0]!.slice('LEGACY_LEASE_SEED '.length)) as SeedSummary- if (summary.leaseId !== leaseId) throw new Error('seed summary names another lease')- if (summary.mode !== '600' || summary.uid !== 1001 || summary.gid !== 1001)- throw new Error(`seeded ledger has mode ${summary.mode} owner ${summary.uid}:${summary.gid}`)- return summary-}--/**- * Seeds the lease while HCC and the Host are at zero and the Host pod is gone.- * The pod runs the Host's image as uid/gid 1001 (the Host's own identity) and- * mounts the same PVC. It is deleted before this returns, on every path.- */-export async function seedLegacyLease(- kubectl: Kubectl,- input: { runId: string; scenario: string; pvc: PvcIdentity; lease: LegacyLeaseRecord }-): Promise<SeedSummary> {- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 0)- throw new Error('refusing to seed: the Host is not scaled to zero')- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 0)- throw new Error('refusing to seed: HCC is not scaled to zero')- const hostPods = readPods(kubectl, HOST.namespace, HOST.selector)- if (hostPods.length !== 0)- throw new Error(`refusing to seed: Host pods remain (${describePods(hostPods)})`)- const image = JSON.parse(- kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])- ).spec.template.spec.containers.find((c: { name: string }) => c.name === HOST.container)?.image- if (typeof image !== 'string' || !image) throw new Error('Host container image not found')- const name = `legacy-lease-seed-${input.runId.slice(-12)}-${input.scenario.toLowerCase()}`- const pod = {- apiVersion: 'v1',- kind: 'Pod',- metadata: {- name,- namespace: HOST.namespace,- labels: { [SEED_LABEL_KEY]: input.runId },- },- spec: {- restartPolicy: 'Never',- automountServiceAccountToken: false,- securityContext: {- runAsUser: 1001,- runAsGroup: 1001,- fsGroup: 1001,- runAsNonRoot: true,- seccompProfile: { type: 'RuntimeDefault' },- },- containers: [- {- name: 'seed',- image,- imagePullPolicy: 'IfNotPresent',- command: ['node', '-e', SEED_SCRIPT],- env: [{ name: 'LEGACY_LEASE', value: JSON.stringify(input.lease) }],- securityContext: {- allowPrivilegeEscalation: false,- capabilities: { drop: ['ALL'] },- },- resources: {- limits: { cpu: '200m', memory: '128Mi' },- requests: { cpu: '50m', memory: '64Mi' },- },- volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],- },- ],- volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: input.pvc.name } }],- },- }- kubectl(['-n', HOST.namespace, 'create', '-f', '-'], { input: JSON.stringify(pod) })- try {- const phase = await waitFor(`seed pod ${name} finishes`, 120_000, () => {- const current = kubectl([- '-n',- HOST.namespace,- 'get',- 'pod',- name,- '-o',- 'jsonpath={.status.phase}',- ]).trim()- return current === 'Succeeded' || current === 'Failed'- ? { done: true, value: current }- : { done: false, observed: current || 'Pending' }- })- const logs = kubectl(['-n', HOST.namespace, 'logs', name, '-c', 'seed'])- if (phase !== 'Succeeded') throw new Error(`seed pod ${name} ${phase}:\n${logs.slice(-2000)}`)- return parseSeedSummary(logs, input.lease.leaseId)- } finally {- kubectl(- [- '-n',- HOST.namespace,- 'delete',- 'pod',- name,- '--ignore-not-found',- '--wait=true',- '--timeout=90s',- ],- {- timeoutMs: 100_000,- }- )- }-}--// ----------------------------------------------------------------------------// Scenarios.-// -----------------------------------------------------------------------------export const SCENARIOS = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'] as const-export type Scenario = (typeof SCENARIOS)[number]--export interface ScenarioResult {- scenario: Scenario- pvc: PvcIdentity- previousHostUid: string- host: PodState- seed: SeedSummary- lease: LegacyLeaseRecord- /** Workloads the scenario replaced, with their pod UIDs before and after. */- replaced: Array<{ workload: string; before: string[]; after: string }>-}--function singleReadyPod(- kubectl: Kubectl,- namespace: string,- selector: string,- label: string-): PodState {- const pods = readPods(kubectl, namespace, selector).filter(p => !p.terminating)- const [only] = pods- if (pods.length !== 1 || !only?.ready)- throw new Error(`${label} is not one Ready pod (${describePods(pods)})`)- return only-}--/**- * Stops the Host. `crash` kills the running pod without grace and scales the- * deployment to zero at once, so the replacement the ReplicaSet creates in- * between must never start a container: if one does, the crash window was- * lost and the scenario fails instead of silently becoming a graceful stop.- */-async function stopHost(- kubectl: Kubectl,- mode: 'crash' | 'graceful',- running: PodState-): Promise<void> {- if (mode === 'graceful') {- scale(kubectl, HOST.namespace, HOST.deployment, 0)- await waitNoPods(kubectl, HOST.namespace, HOST.selector, 'Host')- return- }- kubectl([- '-n',- HOST.namespace,- 'delete',- 'pod',- running.name,- '--force',- '--grace-period=0',- '--wait=false',- ])- scale(kubectl, HOST.namespace, HOST.deployment, 0)- const startedReplacements = new Set<string>()- await waitFor('Host has no pods after the crash', 120_000, () => {- const pods = readPods(kubectl, HOST.namespace, HOST.selector)- for (const pod of pods)- if (pod.uid !== running.uid && pod.started) startedReplacements.add(pod.name)- return pods.length === 0- ? { done: true, value: undefined }- : { done: false, observed: describePods(pods) }- })- if (startedReplacements.size > 0)- throw new Error(- `CRASH_WINDOW_LOST: a replacement Host pod started before the scale to zero (${[...startedReplacements].join(', ')})`- )-}--async function restartWorkload(- kubectl: Kubectl,- workload: { namespace: string; deployment: string; selector: string }-): Promise<{ workload: string; before: string[]; after: string }> {- const before = readPods(kubectl, workload.namespace, workload.selector).map(p => p.uid)- if (before.length === 0) throw new Error(`${workload.deployment} has no pod to restart`)- rolloutRestart(kubectl, workload.namespace, workload.deployment)- rolloutStatus(kubectl, workload.namespace, workload.deployment)- const pod = await waitSingleNewReadyPod(- kubectl,- workload.namespace,- workload.selector,- new Set(before),- workload.deployment- )- return { workload: workload.deployment, before, after: pod.uid }-}--/**- * Runs one scenario end to end: HCC to zero, Host stopped, lease seeded, the- * scenario's own restarts, then HCC and the Host back. Returns the new Host pod.- * Every step leaves HCC and the Host at their pre-scenario replicas on success;- * on failure, `ensureWorkloadsUp` (and the runner's restoration) bring them back.- */-export async function runScenario(- kubectl: Kubectl,- input: { scenario: Scenario; runId: string }-): Promise<ScenarioResult> {- const { scenario } = input- const replaced: ScenarioResult['replaced'] = []- const pvcBefore = hostPvcIdentity(kubectl)- const running = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')- const hccBefore = singleReadyPod(kubectl, HCC.namespace, HCC.selector, 'HCC')- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)- throw new Error('Host deployment is not at one replica before the scenario')- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)- throw new Error('HCC deployment is not at one replica before the scenario')-- // HCC first: it owns the Host deployment and would scale it back up.- scale(kubectl, HCC.namespace, HCC.deployment, 0)- await waitNoPods(kubectl, HCC.namespace, HCC.selector, 'HCC')- await stopHost(kubectl, scenario === 'S-crash' ? 'crash' : 'graceful', running)-- const lease = legacyLease()- const seed = await seedLegacyLease(kubectl, {- runId: input.runId,- scenario,- pvc: pvcBefore,- lease,- })-- const startHostThenHcc = async () => {- scale(kubectl, HOST.namespace, HOST.deployment, 1)- scale(kubectl, HCC.namespace, HCC.deployment, 1)- }- switch (scenario) {- case 'S-crash':- await startHostThenHcc()- break- case 'S-graceful':- // A new template while stopped: the next pod is a fresh rollout, the way- // a graceful `rollout restart` replaces the Host.- rolloutRestart(kubectl, HOST.namespace, HOST.deployment)- await startHostThenHcc()- break- case 'S-hcc':- await startHostThenHcc()- break- case 'S-gfs':- for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))- await startHostThenHcc()- break- case 'S-update':- // Upgrade order: GFS, WRC, HCC, then the Host.- for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))- replaced.push(await restartWorkload(kubectl, WRC))- rolloutRestart(kubectl, HOST.namespace, HOST.deployment)- scale(kubectl, HCC.namespace, HCC.deployment, 1)- rolloutStatus(kubectl, HCC.namespace, HCC.deployment)- scale(kubectl, HOST.namespace, HOST.deployment, 1)- break- }-- const hccPod = await waitSingleNewReadyPod(- kubectl,- HCC.namespace,- HCC.selector,- new Set([hccBefore.uid]),- 'HCC'- )- replaced.push({ workload: HCC.deployment, before: [hccBefore.uid], after: hccPod.uid })- const host = await waitSingleNewReadyPod(- kubectl,- HOST.namespace,- HOST.selector,- new Set([running.uid]),- 'Host'- )- rolloutStatus(kubectl, HOST.namespace, HOST.deployment)- // HCC's first reconcile must not roll the Host again: the pod that booted on- // the seeded ledger is the one whose signals are read.- const settled = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')- if (settled.uid !== host.uid)- throw new Error(`HCC replaced the Host pod after it booted (${host.uid} -> ${settled.uid})`)- const pvcAfter = hostPvcIdentity(kubectl)- if (pvcAfter.name !== pvcBefore.name || pvcAfter.uid !== pvcBefore.uid)- throw new Error(- `Host PVC changed across ${scenario}: ${JSON.stringify(pvcBefore)} -> ${JSON.stringify(pvcAfter)}`- )- return { scenario, pvc: pvcAfter, previousHostUid: running.uid, host, seed, lease, replaced }-}--/** Brings HCC and the Host back to one replica after a failed scenario. */-export function ensureWorkloadsUp(kubectl: Kubectl): void {- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)- scale(kubectl, HOST.namespace, HOST.deployment, 1)- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)- scale(kubectl, HCC.namespace, HCC.deployment, 1)- rolloutStatus(kubectl, HCC.namespace, HCC.deployment)- rolloutStatus(kubectl, HOST.namespace, HOST.deployment)-}--// ----------------------------------------------------------------------------// Signal readers (read-only) and pure verdicts.-// -----------------------------------------------------------------------------export function readStoreLedger(kubectl: Kubectl, podName: string): Record<string, unknown> {- const raw = kubectl([- '-n',- HOST.namespace,- 'exec',- podName,- '-c',- HOST.container,- '--',- 'cat',- STORE_LEDGER_PATH,- ])- const parsed = JSON.parse(raw) as unknown- if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))- throw new Error('store ledger is not a JSON object')- return parsed as Record<string, unknown>-}--export function readHostLogLines(kubectl: Kubectl, podName: string): string[] {- return kubectl(['-n', HOST.namespace, 'logs', podName, '-c', HOST.container], {- timeoutMs: 60_000,- })- .split('\n')- .filter(line => line.trim().length > 0)-}--export function readHostMetrics(kubectl: Kubectl, podName: string): string {- return kubectl([- '-n',- HOST.namespace,- 'exec',- podName,- '-c',- HOST.container,- '--',- 'node',- '-e',- "fetch('http://127.0.0.1:8080/metrics').then(r=>{if(!r.ok)throw new Error('metrics '+r.status);return r.text()}).then(t=>process.stdout.write(t))",- ])-}--/**- * The value of an unlabelled metric, or null when the exposition has no such- * sample. A metric with labels, or more than one sample, is refused.- */-export function parseMetricValue(text: string, name: string): number | null {- const samples = text- .split('\n')- .filter(- line => !line.startsWith('#') && (line.startsWith(`${name} `) || line.startsWith(`${name}{`))- )- if (samples.length === 0) return null- if (samples.length !== 1) throw new Error(`${name} has ${samples.length} samples`)- const match = new RegExp(`^${name}(?:\\{\\})? ([0-9.eE+-]+)$`).exec(samples[0]!.trim())- if (!match) throw new Error(`${name} sample is not an unlabelled number: ${samples[0]}`)- const value = Number(match[1])- if (!Number.isFinite(value)) throw new Error(`${name} sample is not finite`)- return value-}--interface StoreLogLine {- component?: unknown- msg?: unknown- discarded?: unknown- legacyProcessingLeases?: unknown- outcome?: unknown-}--function storeLogLines(logLines: string[]): StoreLogLine[] {- const parsed: StoreLogLine[] = []- for (const line of logLines) {- if (!line.startsWith('{')) continue- let value: unknown- try {- value = JSON.parse(line)- } catch {- continue- }- if (- typeof value === 'object' &&- value !== null &&- (value as StoreLogLine).component === STORE_LOG_COMPONENT- )- parsed.push(value as StoreLogLine)- }- return parsed-}--export type RecoveryOutcome = 'discarded' | 'unknown-outcome'--/**- * Signal 2 on the fixed Host. The ledger must have no `processingLeases`, and- * exactly one of the two outcomes must be visible: one confirmed discard of- * one lease with the counter at 1, or one unknown-outcome warning with the- * counter at 0. The counter is read from a pod that just booted, so its value- * is the delta. A missing counter is a failure: it is the liveness witness- * that the metric registry of the fixed build is the one that answered.- */-export function legacyLeaseRecoveryVerdict(input: {- ledger: Record<string, unknown>- logLines: string[]- counter: number | null-}): RecoveryOutcome {- if (Object.hasOwn(input.ledger, 'processingLeases'))- throw new Error('the ledger still carries processingLeases after the restart')- if (input.counter === null)- throw new Error(`${LEGACY_DISCARD_COUNTER} is not exposed by the Host`)- const lines = storeLogLines(input.logLines)- const discards = lines.filter(- line =>- typeof line.msg === 'string' &&- line.msg.startsWith(DISCARD_MSG_PREFIX) &&- typeof line.discarded === 'number'- )- const unknown = lines.filter(- line =>- line.outcome === 'unknown' &&- typeof line.msg === 'string' &&- line.msg.startsWith(UNKNOWN_MSG_PREFIX) &&- typeof line.legacyProcessingLeases === 'number'- )- if (- discards.length === 1 &&- unknown.length === 0 &&- discards[0]!.discarded === 1 &&- input.counter === 1- )- return 'discarded'- if (- unknown.length === 1 &&- discards.length === 0 &&- unknown[0]!.legacyProcessingLeases === 1 &&- input.counter === 0- )- return 'unknown-outcome'- throw new Error(- `no recovery outcome: ${discards.length} discard line(s) ` +- `(${discards.map(line => String(line.discarded)).join(',') || '-'}), ` +- `${unknown.length} unknown-outcome line(s), counter ${input.counter}`- )-}--/**- * The vacuity witness on the pre-fix Host: the seeded lease is still in the- * ledger, nothing was discarded, and the counter does not exist. Together with- * the journey failing at the shell with download_busy, this shows the lane can- * tell the fixed build from the broken one.- */-export function vacuityStoreVerdict(input: {- ledger: Record<string, unknown>- logLines: string[]- counter: number | null- leaseId: string-}): 'lease-retained' {- const leases = input.ledger.processingLeases as Record<string, unknown> | undefined- if (!leases || typeof leases !== 'object' || !Object.hasOwn(leases, input.leaseId))- throw new Error('the pre-fix Host no longer carries the seeded lease')- const discards = storeLogLines(input.logLines).filter(- line => typeof line.msg === 'string' && line.msg.startsWith(DISCARD_MSG_PREFIX)- )- if (discards.length !== 0) throw new Error('the pre-fix Host logged a legacy-lease discard')- if (input.counter !== null) throw new Error(`the pre-fix Host exposes ${LEGACY_DISCARD_COUNTER}`)- return 'lease-retained'-}--export interface StoreRecord {- id: string- hostPath: string- sizeBytes: number- sha256?: string- state: string-}--export function storeRecords(ledger: Record<string, unknown>): StoreRecord[] {- const records = ledger.records- if (typeof records !== 'object' || records === null || Array.isArray(records))- throw new Error('store ledger has no records object')- return Object.values(records as Record<string, StoreRecord>)-}--/** The one completed record this journey added, matching the GFS source bytes. */-export function downloadedRecord(- before: Record<string, unknown>,- after: Record<string, unknown>,- source: { sha256: string; bytes: number }-): StoreRecord {- if (!SHA256_HEX.test(source.sha256)) throw new Error('source sha256 is not a digest')- const known = new Set(storeRecords(before).map(record => record.id))- const added = storeRecords(after).filter(record => !known.has(record.id))- const matching = added.filter(- record =>- record.state === 'completed' &&- record.sha256 === source.sha256 &&- record.sizeBytes === source.bytes- )- if (added.length !== 1 || matching.length !== 1)- throw new Error(- `expected one new completed record for ${source.sha256}/${source.bytes} bytes; ` +- `added ${JSON.stringify(added.map(r => ({ state: r.state, sha256: r.sha256, sizeBytes: r.sizeBytes })))}`- )- return matching[0]!-}--/** sha256 and size of a file on the Host PVC, computed in the Host container. */-export function hostFileDigest(- kubectl: Kubectl,- podName: string,- hostPath: string-): { sha256: string; bytes: number } {- if (path_is_unsafe(hostPath)) throw new Error(`refusing to read ${hostPath}`)- const out = kubectl([- '-n',- HOST.namespace,- 'exec',- podName,- '-c',- HOST.container,- '--',- 'node',- '-e',- "const b=require('node:fs').readFileSync(process.argv[1]);process.stdout.write(require('node:crypto').createHash('sha256').update(b).digest('hex')+' '+b.length)",- `/workspace/${hostPath}`,- ]).trim()- const [sha256, bytes] = out.split(' ')- if (!sha256 || !SHA256_HEX.test(sha256) || !/^\d+$/.test(bytes ?? ''))- throw new Error(`unexpected digest output: ${out}`)- return { sha256, bytes: Number(bytes) }-}--function path_is_unsafe(hostPath: string): boolean {- return hostPath.startsWith('/') || hostPath.split('/').some(part => part === '..' || part === '')-}diff --git a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.tsdeleted file mode 100644index 41fb82cd4..000000000--- a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts+++ /dev/null@@ -1,59 +0,0 @@-import { defineConfig } from '@playwright/test'-import path from 'node:path'-import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'-import { requireLegacyLeaseLaneEnv } from './helpers/legacyLeaseRestart'--// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns-// visible IPC transitions and reads the external provider ledger as its wire oracle.--/*- * Desktop App legacy processing-lease restart lane (issue #1022).- *- * It reuses the image-capabilities fixture stack (derived Host image, provider- * fixture, seeded catalog and login) and adds the cluster side: before each- * scenario the Host and HCC are stopped, one legacy foreign processing lease is- * seeded on the Host PVC, the scenario's restarts run, and the Desktop journey- * (approved `shell_exec`, then `clerum__gfs_download`) must succeed on the Host- * that booted on that ledger.- *- * - It loads NO env file. Every binding is exported by the runner- * (`make minikube-run-legacy-lease-restart` or the -vacuity target) and a- * missing or inconsistent one fails here, at config load.- * - The runner selects the scenarios with --grep: the fixed lane runs the five- * `legacy-lease-restart fixture:` tests, the vacuity lane runs the one- * `legacy-lease-restart vacuity:` test against the pre-fix Host.- */--const repoRoot = path.resolve(__dirname, '../../..')--if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {- throw new Error(- `NODE_ENV must be "test" (or unset) for the legacy-lease-restart fixture lane; ` +- `received "${process.env.NODE_ENV}".`- )-}-process.env.NODE_ENV = 'test'--// Fail at config load, with every missing binding listed at once.-requireImageCapabilitiesFixtureEnv(process.env)-requireLegacyLeaseLaneEnv(process.env)--const recorderRoot = process.env.QA_RECORDER_ROOT- ? path.resolve(process.env.QA_RECORDER_ROOT)- : path.join(repoRoot, '.local-notes', 'qa-recorder')--export default defineConfig({- testDir: '.',- testMatch: /qa-recorder-legacy-lease-restart\.spec\.ts/,- // Per scenario: HCC and Host stop, seed pod, the scenario's own rollouts- // (S-update replaces GFS, WRC, HCC and the Host), one Electron launch and a- // two-tool journey with an approval.- timeout: 660_000,- expect: { timeout: 30_000 },- retries: 0,- workers: 1,- fullyParallel: false,- reporter: [['list']],- outputDir: path.join(recorderRoot, 'runs', 'desktop-app-legacy-lease-restart'),- preserveOutput: 'always',-})diff --git a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.tsdeleted file mode 100644index e0e1a5dcb..000000000--- a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts+++ /dev/null@@ -1,440 +0,0 @@-import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'-import { createHash, randomBytes } from 'node:crypto'-import {- GFS_E2E_DRIVE,- type GfsFileFixture,- cleanupGfsFixture,- getE2EUserId,- gfsWriterPod,- kubectlOut,- ridOf,- seedGfsFileFixture,- seedGfsGrant,- uniqueGfsFixtureName,-} from '../../../tests/e2e/gfsUiFixtures'-import { discoverManagedGfsAgent } from './helpers/gfsAgentDiscovery'-import {- FIXTURE_RESPONSE_KIND,- IMAGE_CAPABILITY_FIXTURE_MODELS,- type ImageCapabilityEvidenceSnapshot,- appendedAttempts,- readImageCapabilityEvidence,- requireImageCapabilitiesFixtureEnv,-} from './helpers/imageCapabilityEvidence'-import {- HOST,- type Kubectl,- LEGACY_DISCARD_COUNTER,- type LegacyLeaseLaneMode,- SCENARIOS,- type Scenario,- type ScenarioResult,- downloadedRecord,- ensureWorkloadsUp,- hostFileDigest,- kubectlFor,- legacyLeaseRecoveryVerdict,- parseMetricValue,- readHostLogLines,- readHostMetrics,- readPods,- readStoreLedger,- requireLegacyLeaseLaneEnv,- runScenario,- vacuityStoreVerdict,-} from './helpers/legacyLeaseRestart'-import {- EXTERNAL_REST_API_BASE_URL,- RPC_PROXY_BASE_URL,- assertAllowedTarget,- desktopCredentials,- finalizeRecording,- launchDesktopApp,- login,- screenshotAndLog,-} from './qa-recorder-helpers'--/*- * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the agent switch, the model popover and- * the approval card all travel over Electron IPC, so the renderer has no HTTP- * request to await for any transition. The oracles are the visible chat (the- * fixture's answer line and the approval card), the provider-boundary ledger of- * the derived in-cluster fixture, and read-only reads of the Host's own store- * ledger, logs, metrics and workspace file.- *- * Issue #1022: a processing lease written by a Host build before #1019 has no- * writer session. A Host that boots on such a ledger must discard it at- * initialize (log `discarded 1`, counter 1) or report that it could not confirm- * the discard; either way the ledger it serves carries no `processingLeases`,- * and a shell command plus a GFS download work. The pre-fix build treats the- * lease as an inherited executor, so its shell refuses with `download_busy`.- *- * Each scenario stops HCC and the Host, seeds one foreign lease on the Host PVC- * from a pod running as the Host's uid/gid, performs its own restarts, and- * brings HCC and the Host back. Then the Desktop journey runs: the user sends a- * journey line, approves `shell_exec` by clicking Approve, and the fixture asks- * for `clerum__gfs_download` of a seeded GFS file. The answer on screen carries- * the sha256 and byte count of the Host's receipt, which must equal the GFS- * source, as must the file the Host wrote to its workspace.- *- * Setup shortcuts (named, outside the behaviour under test): the GFS file and- * its grants are seeded with SQL and the writer blob with kubectl cp, and the- * legacy lease is written by the seed pod. Only the external provider peer is- * simulated; nothing is mocked inside the cluster or the app.- */--const SCENARIO_TITLES: Record<Scenario, string> = {- 'S-crash':- 'S-crash: a Host killed without grace boots on a legacy lease and serves shell then download',- 'S-graceful':- 'S-graceful: a Host stopped and rolled out boots on a legacy lease and serves shell then download',- 'S-hcc':- 'S-hcc: an HCC restart with the Host boots on a legacy lease and serves shell then download',- 'S-gfs':- 'S-gfs: each GFS pod restarts while the Host boots on a legacy lease, then shell and download work',- 'S-update':- 'S-update: GFS, WRC, HCC then Host restart in upgrade order on a legacy lease, then shell and download work',-}--const JOURNEY_RESPONSE_TIMEOUT_MS = 240_000-const ANY_TERMINAL_ANSWER = /LEGACY-LEASE-FIXTURE-(?:OK|SHELL-FAILED|DOWNLOAD-FAILED)/--interface LaneContext {- mode: LegacyLeaseLaneMode- runId: string- kubectl: Kubectl- env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>- file: GfsFileFixture- source: { sha256: string; bytes: number }-}--let lane: LaneContext | undefined--function requireLane(expected: LegacyLeaseLaneMode): LaneContext {- if (!lane) throw new Error('legacy-lease-restart lane context was not prepared')- // A mode mismatch is a runner defect: it fails, it never skips.- if (lane.mode !== expected)- throw new Error(- `this test belongs to the ${expected} lane, but LEGACY_LEASE_LANE_MODE is ${lane.mode}`- )- return lane-}--test.describe.configure({ mode: 'serial' })--test.beforeAll(async () => {- const env = requireImageCapabilitiesFixtureEnv()- const laneEnv = requireLegacyLeaseLaneEnv()- await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)- await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)- const kubectl = kubectlFor(laneEnv.profile)-- // The GFS subject of the Host this lane drives: exactly the chatllm Host.- const agent = discoverManagedGfsAgent()- if (agent.namespace !== HOST.namespace || agent.name !== HOST.deployment)- throw new Error(`GFS agent discovery returned ${agent.subjectId}, not the lane's Host`)- const ownerUserId = getE2EUserId(desktopCredentials().email)-- const file = seedGfsFileFixture(uniqueGfsFixtureName('e2e-gfs-legacy-lease'))- try {- seedGfsGrant({- resourceId: file.fileResourceId,- subjectType: 'user',- subjectId: ownerUserId,- permissions: ['read'],- inherit: true,- grantedBy: 'e2e:legacy-lease-restart',- })- seedGfsGrant({- resourceId: file.fileResourceId,- subjectType: 'host',- subjectId: agent.subjectId,- permissions: ['read'],- inherit: true,- grantedBy: 'e2e:legacy-lease-restart',- })- // The source the journey must reproduce: the bytes the fixture seeded, and- // the same digest read back from the writer's blob.- const content = Buffer.from(`E2E GFS file fixture: ${file.name}\n`, 'utf8')- const sha256 = createHash('sha256').update(content).digest('hex')- const blob = kubectlOut([- '-n',- 'gfs',- 'exec',- gfsWriterPod(),- '--',- 'sha256sum',- `/data/gfs/${ridOf(file.fileResourceId)}`,- ]).trim()- if (blob.split(/\s+/)[0] !== sha256)- throw new Error(`writer blob digest ${blob} differs from the seeded ${sha256}`)- lane = {- mode: laneEnv.mode,- runId: laneEnv.runId,- kubectl,- env,- file,- source: { sha256, bytes: content.length },- }- } catch (error) {- cleanupGfsFixture(file.name)- throw error- }-})--test.afterAll(async () => {- if (lane) cleanupGfsFixture(lane.file.name)-})--/** Reach the authenticated shell through the real sign-in form. */-async function signIn(page: Page) {- await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })- const emailInput = page.locator('#email-input')- const settingsMenu = page.getByTestId('nav-settings-menu')- await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })- if (await settingsMenu.isVisible()) {- if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') await settingsMenu.click()- await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')- await page.getByTestId('logout-btn').click()- }- await expect(emailInput).toBeVisible({ timeout: 20_000 })- await expect(page.getByTestId('nav-chat')).toHaveCount(0)- await login(page, desktopCredentials())- await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })-}--async function openFixtureChat(page: Page, hostRef: string) {- const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')- await page.getByTestId('nav-chat').click()- const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })- await expect(switchAgent).toBeVisible({ timeout: 30_000 })- await switchAgent.click()- const item = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })- await expect(item).toHaveCount(1)- await item.click()- await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))-- await page.getByTestId('nav-new-chat').click()- const composer = page.getByRole('textbox', { name: 'Agent message composer' })- await expect(composer).toBeVisible({ timeout: 30_000 })- await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })-- const model = IMAGE_CAPABILITY_FIXTURE_MODELS.supported- const chip = page.getByTestId('selected-chat-model')- await expect(chip).toBeVisible({ timeout: 30_000 })- await chip.click()- await expect(page.getByRole('menu', { name: 'Select model' })).toBeVisible({ timeout: 20_000 })- const row = page.getByTestId(`model-option-${model}`)- await expect(row).toHaveCount(1)- await row.click()- await expect(chip).toHaveAttribute('data-model-id', model)- return composer-}--interface JourneyOutcome {- answer: string- approvals: string[]- before: ImageCapabilityEvidenceSnapshot- after: ImageCapabilityEvidenceSnapshot-}--/**- * Sends the journey line and approves every tool approval the Host raises by- * clicking Approve, until the fixture's terminal answer is on screen. Returns- * the answer text and the input preview of each approval clicked.- */-async function runJourney(page: Page, ctx: LaneContext, marker: string): Promise<JourneyOutcome> {- const composer = await openFixtureChat(page, ctx.env.hostRef)- const line = `LEGACY_LEASE_JOURNEY marker=${marker} drive=${GFS_E2E_DRIVE} resourceId=${ridOf(ctx.file.fileResourceId)}`- const before = readImageCapabilityEvidence(ctx.env)- expect(before.runId).toBe(ctx.env.runId)- await composer.fill(line)- const send = page.getByTestId('send-button')- await expect(send).toBeEnabled({ timeout: 20_000 })- await send.click()-- const approve = page.getByTestId('approval-approve-btn')- const preview = page.getByTestId('approval-input-preview')- const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')- const terminal = response.filter({ hasText: ANY_TERMINAL_ANSWER })- const approvals: string[] = []- // The journey has at most two approvable tools (shell, then download).- for (let round = 0; round < 3; round += 1) {- await expect(approve.or(terminal)).toBeVisible({ timeout: JOURNEY_RESPONSE_TIMEOUT_MS })- if ((await terminal.count()) > 0) break- if (round === 2) throw new Error('more approvals than the journey has tools')- await expect(preview).toBeVisible({ timeout: 20_000 })- const shown = await preview.innerText()- approvals.push(shown)- await approve.click()- // The clicked card leaves, or is replaced by the next tool's approval.- await expect- .poll(- async () =>- (await approve.count()) === 0 ||- ((await preview.count()) > 0 && (await preview.innerText()) !== shown),- { timeout: 60_000 }- )- .toBe(true)- }- await expect(terminal).toHaveCount(1)- const answer = (await terminal.innerText()).trim()- const after = readImageCapabilityEvidence(ctx.env)- return { answer, approvals, before, after }-}--/** The shell approval the user clicked is the fixture's command for this marker. */-function expectShellApproved(approvals: string[], marker: string) {- expect(approvals.length).toBeGreaterThanOrEqual(1)- expect(approvals[0]).toContain('printf')- expect(approvals[0]).toContain(marker)-}--async function withDesktop(- testInfo: TestInfo,- label: string,- body: (page: Page) => Promise<void>-): Promise<void> {- let app: ElectronApplication | undefined- let recordedPage: Page | undefined- try {- const launched = await launchDesktopApp(- testInfo,- `issue1022-${label}-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`- )- app = launched.app- recordedPage = launched.page- await signIn(launched.page)- await body(launched.page)- await screenshotAndLog(launched.page, testInfo, `desktop-legacy-lease-${label}`)- } catch (error) {- if (recordedPage && !recordedPage.isClosed()) {- await recordedPage- .screenshot({ path: testInfo.outputPath('failure.png') })- .catch(() => undefined)- }- throw error- } finally {- await finalizeRecording(app, recordedPage)- }-}--/** The Host pod that booted on the seeded ledger is still the only Ready one. */-function expectHostUnchanged(ctx: LaneContext, result: ScenarioResult) {- const pods = readPods(ctx.kubectl, HOST.namespace, HOST.selector).filter(p => !p.terminating)- expect(pods.map(p => p.uid)).toEqual([result.host.uid])- expect(pods[0]?.ready).toBe(true)- expect(result.host.uid).not.toBe(result.previousHostUid)-}--function attachScenario(testInfo: TestInfo, data: unknown) {- return testInfo.attach('scenario.json', {- body: JSON.stringify(data, null, 2),- contentType: 'application/json',- })-}--for (const scenario of SCENARIOS) {- test(`legacy-lease-restart fixture: ${SCENARIO_TITLES[scenario]}`, async ({}, testInfo) => {- const ctx = requireLane('fixed')- try {- const result = await runScenario(ctx.kubectl, { scenario, runId: ctx.runId })- expect(result.seed.leaseId).toBe(result.lease.leaseId)-- // Signal 2: the store the new Host serves has no legacy lease, and it- // reports one of the two recovery outcomes.- const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)- const metrics = readHostMetrics(ctx.kubectl, result.host.name)- const outcome = legacyLeaseRecoveryVerdict({- ledger: ledgerAtBoot,- logLines: readHostLogLines(ctx.kubectl, result.host.name),- counter: parseMetricValue(metrics, LEGACY_DISCARD_COUNTER),- })- await attachScenario(testInfo, { ...result, outcome })-- // Signal 3: the Desktop journey on that Host.- const marker = `legacy-lease-${randomBytes(8).toString('hex')}`- await withDesktop(testInfo, scenario.toLowerCase(), async page => {- const journey = await runJourney(page, ctx, marker)- expectShellApproved(journey.approvals, marker)- expect(journey.answer).toContain(- `LEGACY-LEASE-FIXTURE-OK marker=${marker} sha256=${ctx.source.sha256} bytes=${ctx.source.bytes}`- )- const rows = appendedAttempts(journey.before, journey.after)- const kinds = rows.map(row => row.responseKind)- expect(- kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)- ).toHaveLength(1)- expect(- kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)- ).toHaveLength(1)- const answers = rows.filter(- row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer- )- expect(answers).toHaveLength(1)- expect(answers[0]?.downloadSha256).toBe(ctx.source.sha256)- expect(answers[0]?.downloadBytes).toBe(ctx.source.bytes)- const delta = (key: keyof ImageCapabilityEvidenceSnapshot['counters']) =>- journey.after.counters[key] - journey.before.counters[key]- expect(delta('legacyLeaseShellRequests')).toBe(1)- expect(delta('legacyLeaseDownloadRequests')).toBe(1)- expect(delta('legacyLeaseAnswers')).toBe(1)- expect(delta('legacyLeaseFailures')).toBe(0)- })-- // The store recorded exactly this download, and the workspace file holds- // the GFS source bytes.- const ledgerAfter = readStoreLedger(ctx.kubectl, result.host.name)- const record = downloadedRecord(ledgerAtBoot, ledgerAfter, ctx.source)- expect(hostFileDigest(ctx.kubectl, result.host.name, record.hostPath)).toEqual(ctx.source)- expect(Object.hasOwn(ledgerAfter, 'processingLeases')).toBe(false)- expectHostUnchanged(ctx, result)- } finally {- ensureWorkloadsUp(ctx.kubectl)- }- })-}--test('legacy-lease-restart vacuity: S-crash on base 74e0d81d9 fails at the shell with download_busy', async ({}, testInfo) => {- const ctx = requireLane('vacuity')- try {- const result = await runScenario(ctx.kubectl, { scenario: 'S-crash', runId: ctx.runId })- const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)- const verdict = vacuityStoreVerdict({- ledger: ledgerAtBoot,- logLines: readHostLogLines(ctx.kubectl, result.host.name),- counter: parseMetricValue(- readHostMetrics(ctx.kubectl, result.host.name),- LEGACY_DISCARD_COUNTER- ),- leaseId: result.lease.leaseId,- })- await attachScenario(testInfo, { ...result, verdict })-- const marker = `legacy-lease-${randomBytes(8).toString('hex')}`- await withDesktop(testInfo, 'vacuity', async page => {- const journey = await runJourney(page, ctx, marker)- expectShellApproved(journey.approvals, marker)- expect(journey.answer).toContain('LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy')- const rows = appendedAttempts(journey.before, journey.after)- const shellFailed = rows.filter(- row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed- )- // Positive witness first: the shell step ran and failed with the store code.- expect(shellFailed).toHaveLength(1)- expect(shellFailed[0]?.failureCode).toBe('download_busy')- expect(- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)- ).toHaveLength(1)- expect(- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)- ).toHaveLength(0)- expect(- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer)- ).toHaveLength(0)- })- expectHostUnchanged(ctx, result)- } finally {- ensureWorkloadsUp(ctx.kubectl)- }-})diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.tsindex 31c621059..7912bc0d6 100644--- a/desktop-app/test/imageCapabilityEvidence.test.ts+++ b/desktop-app/test/imageCapabilityEvidence.test.ts@@ -44,24 +44,6 @@ const rows = {truncated: true,}),textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),- shellRequested: (): Row => ({- model: 'glm-5.3-flash',- imageSha256: null,- responseKind: 'legacy-lease-shell-requested',- }),- legacyAnswer: (): Row => ({- model: 'glm-5.3-flash',- imageSha256: null,- responseKind: 'legacy-lease-answer',- downloadSha256: DIGEST,- downloadBytes: 42,- }),- shellFailed: (): Row => ({- model: 'glm-5.3-flash',- imageSha256: null,- responseKind: 'legacy-lease-shell-failed',- failureCode: 'download_busy',- }),}function ledger(attempts: Row[]): string {@@ -81,10 +63,6 @@ function ledger(attempts: Row[]): string {documentReadRequests: 0,documentAnswers: 0,documentFailures: 0,- legacyLeaseShellRequests: 0,- legacyLeaseDownloadRequests: 0,- legacyLeaseAnswers: 0,- legacyLeaseFailures: 0,},attempts,})@@ -142,47 +120,6 @@ describe('imageCapabilityEvidence ledger rows', () => {})})-describe('legacy-lease ledger rows', () => {- it('keeps the failure code and the download receipt on the rows that carry them', () => {- const snapshot = parse([rows.shellRequested(), rows.legacyAnswer(), rows.shellFailed()])- expect(snapshot.attempts).toEqual([- rows.shellRequested(),- rows.legacyAnswer(),- rows.shellFailed(),- ])- })-- it('requires a closed-set failure code on failure rows and nowhere else', () => {- // Witness: the well-formed failure row parses with its code.- expect(parse([rows.shellFailed()]).attempts[0]?.failureCode).toBe('download_busy')- const { failureCode: _code, ...withoutCode } = rows.shellFailed()- for (const [label, row] of [- ['missing failureCode', withoutCode],- ['free-text failureCode', { ...rows.shellFailed(), failureCode: 'disk on fire' }],- ['failureCode on an answer row', { ...rows.legacyAnswer(), failureCode: 'download_busy' }],- ['failureCode on a text row', { ...rows.textOnly(), failureCode: 'download_busy' }],- ] as const) {- expect(() => parse([row]), label).toThrow(/failureCode/)- }- })-- it('requires the download receipt on every answer row and nowhere else', () => {- // Witness: the well-formed answer row parses with its receipt.- expect(parse([rows.legacyAnswer()]).attempts[0]?.downloadBytes).toBe(42)- const { downloadSha256: _sha, ...withoutSha } = rows.legacyAnswer()- const { downloadBytes: _bytes, ...withoutBytes } = rows.legacyAnswer()- for (const [label, row] of [- ['missing downloadSha256', withoutSha],- ['short downloadSha256', { ...rows.legacyAnswer(), downloadSha256: 'abc' }],- ['missing downloadBytes', withoutBytes],- ['negative downloadBytes', { ...rows.legacyAnswer(), downloadBytes: -1 }],- ['receipt on a shell row', { ...rows.shellRequested(), downloadSha256: DIGEST }],- ] as const) {- expect(() => parse([row]), label).toThrow(/downloadSha256|downloadBytes/)- }- })-})-describe('appendedAttempts', () => {it('refuses a row whose reason or delivered page changed between reads', () => {const before = parse([rows.rejected(), rows.answer()])@@ -199,21 +136,6 @@ describe('appendedAttempts', () => {expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)}})-- it('refuses a legacy-lease row whose receipt or failure code changed between reads', () => {- const before = parse([rows.legacyAnswer(), rows.shellFailed()])- // Witness: an unchanged prefix yields the appended row.- expect(- appendedAttempts(before, parse([rows.legacyAnswer(), rows.shellFailed(), rows.textOnly()]))- ).toEqual([rows.textOnly()])- for (const [label, changed] of [- ['sha', [{ ...rows.legacyAnswer(), downloadSha256: 'c'.repeat(64) }, rows.shellFailed()]],- ['bytes', [{ ...rows.legacyAnswer(), downloadBytes: 43 }, rows.shellFailed()]],- ['code', [rows.legacyAnswer(), { ...rows.shellFailed(), failureCode: 'writer_locked' }]],- ] as const) {- expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)- }- })})describe('rejection reason parity', () => {@@ -231,17 +153,4 @@ describe('rejection reason parity', () => {expect((declared as readonly string[]).length).toBeGreaterThan(0)expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')})-- it('declares the same legacy-lease failure codes as the fixture, independently', async () => {- const providerModule = (await import(PROVIDER_MODULE_URL)) as {- LEGACY_LEASE_FAILURE_CODES: readonly string[]- }- const declared = (evidenceModule as Record<string, unknown>).FIXTURE_LEGACY_LEASE_FAILURE_CODES- expect(Array.isArray(declared)).toBe(true)- expect(declared).not.toBe(providerModule.LEGACY_LEASE_FAILURE_CODES)- expect([...(declared as readonly string[])].sort()).toEqual(- [...providerModule.LEGACY_LEASE_FAILURE_CODES].sort()- )- expect(declared as readonly string[]).toContain('download_busy')- })})diff --git a/desktop-app/test/legacyLeaseRestart.test.ts b/desktop-app/test/legacyLeaseRestart.test.tsdeleted file mode 100644index 9c9d358e1..000000000--- a/desktop-app/test/legacyLeaseRestart.test.ts+++ /dev/null@@ -1,238 +0,0 @@-import { describe, expect, it } from 'vitest'-import {- LEGACY_DISCARD_COUNTER,- downloadedRecord,- legacyLease,- legacyLeaseRecoveryVerdict,- parseMetricValue,- parseSeedSummary,- requireLegacyLeaseLaneEnv,- vacuityStoreVerdict,-} from './e2e-playwright/helpers/legacyLeaseRestart'--const PROFILE = 'clerum-fix-issue-1022-processing-lease-lifecycle-2e0feeb9'-const RUN_ID = 'image-capabilities-0123456789ab'-const SHA = 'a'.repeat(64)--const discardLine = JSON.stringify({- level: 40,- component: 'GfsDownloadStore',- discarded: 1,- msg: 'GFS download store discarded 1 legacy processing lease(s) at initialize',-})-const unknownLine = JSON.stringify({- level: 40,- component: 'GfsDownloadStore',- legacyProcessingLeases: 1,- outcome: 'unknown',- msg: 'GFS download store could not confirm the discard of 1 legacy processing lease(s)',-})-const unrelatedLine = JSON.stringify({ level: 30, component: 'Server', msg: 'listening' })--describe('requireLegacyLeaseLaneEnv', () => {- const valid = {- MINIKUBE_PROFILE: PROFILE,- CONTROL_API_REAL_PG_CONTEXT: PROFILE,- E2E_K8S_CONTEXT: PROFILE,- LEGACY_LEASE_LANE_MODE: 'fixed',- LEGACY_LEASE_SEED_LABEL_VALUE: RUN_ID,- IMAGE_CAPABILITIES_RUN_ID: RUN_ID,- }-- it('accepts the runner bindings', () => {- expect(requireLegacyLeaseLaneEnv(valid)).toEqual({- profile: PROFILE,- mode: 'fixed',- runId: RUN_ID,- })- })-- it('lists every inconsistent binding at once', () => {- let message = ''- try {- requireLegacyLeaseLaneEnv({- ...valid,- E2E_K8S_CONTEXT: 'minikube',- LEGACY_LEASE_LANE_MODE: 'skip',- LEGACY_LEASE_SEED_LABEL_VALUE: 'image-capabilities-ffffffffffff',- })- } catch (error) {- message = (error as Error).message- }- expect(message).toContain('E2E_K8S_CONTEXT')- expect(message).toContain('LEGACY_LEASE_LANE_MODE')- expect(message).toContain('LEGACY_LEASE_SEED_LABEL_VALUE')- })-})--describe('legacyLease', () => {- it('is a valid, unexpired, foreign lease with no records and no writer session', () => {- const now = Date.parse('2026-10-08T10:00:00.000Z')- const lease = legacyLease(now)- expect(lease.leaseId).toMatch(/^[0-9a-f-]{36}$/)- expect(lease.recordIds).toEqual([])- expect(lease.callerIdentity).not.toBe('')- expect(Date.parse(lease.expiresAt)).toBeGreaterThan(now)- expect(Date.parse(lease.expiresAt)).toBeGreaterThan(Date.parse(lease.acquiredAt))- expect(Object.hasOwn(lease, 'writerSessionId')).toBe(false)- })-})--describe('parseSeedSummary', () => {- const summary = (fields: Record<string, unknown>) =>- `LEGACY_LEASE_SEED ${JSON.stringify({ leaseId: 'l1', records: 0, mode: '600', uid: 1001, gid: 1001, ...fields })}\n`-- it('reads the one summary of the seeded lease', () => {- expect(parseSeedSummary(summary({}), 'l1').records).toBe(0)- })-- it('refuses a ledger left with another mode or owner', () => {- expect(() => parseSeedSummary(summary({ mode: '644' }), 'l1')).toThrow(/mode 644/)- expect(() => parseSeedSummary(summary({ uid: 0 }), 'l1')).toThrow(/owner 0:1001/)- })-- it('refuses no summary, two summaries, or another lease', () => {- expect(() => parseSeedSummary('', 'l1')).toThrow(/0 summaries/)- expect(() => parseSeedSummary(summary({}) + summary({}), 'l1')).toThrow(/2 summaries/)- expect(() => parseSeedSummary(summary({}), 'l2')).toThrow(/another lease/)- })-})--describe('parseMetricValue', () => {- const exposition = [- `# HELP ${LEGACY_DISCARD_COUNTER} Legacy processing leases discarded.`,- `# TYPE ${LEGACY_DISCARD_COUNTER} counter`,- `${LEGACY_DISCARD_COUNTER} 1`,- 'process_cpu_seconds_total 3.2',- ].join('\n')-- it('reads an unlabelled sample', () => {- expect(parseMetricValue(exposition, LEGACY_DISCARD_COUNTER)).toBe(1)- })-- it('returns null when the metric is absent', () => {- expect(parseMetricValue('process_cpu_seconds_total 3.2\n', LEGACY_DISCARD_COUNTER)).toBeNull()- })-- it('refuses duplicate or labelled samples', () => {- expect(() =>- parseMetricValue(`${exposition}\n${LEGACY_DISCARD_COUNTER} 2`, LEGACY_DISCARD_COUNTER)- ).toThrow(/2 samples/)- expect(() =>- parseMetricValue(`${LEGACY_DISCARD_COUNTER}{a="b"} 1`, LEGACY_DISCARD_COUNTER)- ).toThrow(/not an unlabelled number/)- })-})--describe('legacyLeaseRecoveryVerdict', () => {- const ledger = { schemaVersion: 1, records: {} }-- it('accepts one confirmed discard with the counter at 1', () => {- expect(- legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine, discardLine], counter: 1 })- ).toBe('discarded')- })-- it('accepts one unknown-outcome warning with the counter at 0', () => {- expect(legacyLeaseRecoveryVerdict({ ledger, logLines: [unknownLine], counter: 0 })).toBe(- 'unknown-outcome'- )- })-- it('refuses a ledger that still carries processingLeases', () => {- expect(() =>- legacyLeaseRecoveryVerdict({- ledger: { ...ledger, processingLeases: {} },- logLines: [discardLine],- counter: 1,- })- ).toThrow(/still carries processingLeases/)- })-- it('refuses when no outcome was logged (the Host never ran the discard)', () => {- expect(() =>- legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine], counter: 0 })- ).toThrow(/0 discard line\(s\).*0 unknown-outcome/)- })-- it('refuses a discard whose counter disagrees, and a missing counter', () => {- expect(() =>- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: 0 })- ).toThrow(/counter 0/)- expect(() =>- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: null })- ).toThrow(/not exposed/)- })-- it('refuses both outcomes at once, and a discard of more than one lease', () => {- expect(() =>- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine, unknownLine], counter: 1 })- ).toThrow(/no recovery outcome/)- const two = discardLine.replace('"discarded":1', '"discarded":2')- expect(() => legacyLeaseRecoveryVerdict({ ledger, logLines: [two], counter: 2 })).toThrow(- /no recovery outcome/- )- })-})--describe('vacuityStoreVerdict', () => {- const ledger = { schemaVersion: 1, records: {}, processingLeases: { l1: {} } }-- it('accepts the pre-fix Host keeping the lease without a counter', () => {- expect(- vacuityStoreVerdict({ ledger, logLines: [unrelatedLine], counter: null, leaseId: 'l1' })- ).toBe('lease-retained')- })-- it('refuses a discarded lease, a discard line, or an exposed counter', () => {- expect(() =>- vacuityStoreVerdict({- ledger: { schemaVersion: 1, records: {} },- logLines: [],- counter: null,- leaseId: 'l1',- })- ).toThrow(/no longer carries/)- expect(() =>- vacuityStoreVerdict({ ledger, logLines: [discardLine], counter: null, leaseId: 'l1' })- ).toThrow(/logged a legacy-lease discard/)- expect(() => vacuityStoreVerdict({ ledger, logLines: [], counter: 0, leaseId: 'l1' })).toThrow(- /exposes/- )- })-})--describe('downloadedRecord', () => {- const record = (id: string, fields: Record<string, unknown>) => ({- id,- hostPath: `downloads/${id}`,- sizeBytes: 10,- sha256: SHA,- state: 'completed',- ...fields,- })- const before = { schemaVersion: 1, records: { old: record('old', {}) } }-- it('returns the one new completed record with the source digest and size', () => {- const after = { schemaVersion: 1, records: { ...before.records, fresh: record('fresh', {}) } }- expect(downloadedRecord(before, after, { sha256: SHA, bytes: 10 }).id).toBe('fresh')- })-- it('refuses no new record, a different digest, or an unfinished transfer', () => {- expect(() => downloadedRecord(before, before, { sha256: SHA, bytes: 10 })).toThrow(/added \[\]/)- const other = {- schemaVersion: 1,- records: { fresh: record('fresh', { sha256: 'b'.repeat(64) }) },- }- expect(() => downloadedRecord(before, other, { sha256: SHA, bytes: 10 })).toThrow(- /expected one/- )- const partial = {- schemaVersion: 1,- records: { fresh: record('fresh', { state: 'transferring' }) },- }- expect(() => downloadedRecord(before, partial, { sha256: SHA, bytes: 10 })).toThrow(- /expected one/- )- })-})diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.mdindex 1c7b4d75a..97103dfb3 100644--- a/docs/testing/minikube-t2-runbook.md+++ b/docs/testing/minikube-t2-runbook.md@@ -237,105 +237,6 @@ real-subscription interoperability or authorize real account use. The fourpublic targets and the guarded build body have hermetic boundary coverage in`scripts/tests/test-minikube-mutation-boundary.sh`.-### Legacy processing-lease restart lane--Issue #1022. This lane proves that a Host booting on a GFS download store-ledger that still carries a processing lease written before #1019 (no writer-session) discards that lease and keeps serving shell commands and GFS downloads.-It is development-only, uses the image-capabilities provider fixture, and has-the same requirements as that lane: Node 24, a clean tree, the branch-owned-`MINIKUBE_PROFILE` with a matching `CONTROL_API_REAL_PG_CONTEXT`, the mutation-lease, and the single-Host (`mcp-host/chatllm`) profile.--```sh-make minikube-build-image-capabilities-fixture-make minikube-t2-make minikube-run-legacy-lease-restart-```--The Playwright config is-`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts`.-The lane runs five serial scenarios. Each one scales HCC and then the Host-to 0. It waits until the Host pod is gone, and then seeds one legacy lease. The-lease is valid and not expired, belongs to a foreign caller, has-`recordIds: []` and has no `writerSessionId`. The seed pod is labelled-`evenfire.ai/legacy-lease-seed=<run id>`. It runs the Host image as uid/gid-1001 and mounts the Host PVC. It does a read-modify-write using the store's own-publication sequence: a temp file with mode 0600, then fsync, rename, and an-fsync of the directory. It refuses a ledger that already has leases. The pod-reads the ledger back and is deleted before the scenario continues.--| Scenario | How the Host is stopped | What runs after the seed |-| --- | --- | --- |-| `S-crash` | `delete pod --force --grace-period=0`, then an immediate scale to 0 | The Host and HCC return. The scenario fails with `CRASH_WINDOW_LOST` if a replacement pod started a container in between. |-| `S-graceful` | Scale to 0 (SIGTERM, full grace) | `rollout restart` of the Host, then the Host and HCC return |-| `S-hcc` | Scale to 0 | HCC and the Host return, so HCC is a new pod |-| `S-gfs` | Scale to 0 | Each GFS pod restarts in turn (`gfsc-writer`, then `gfsc-reader`), then the Host and HCC return |-| `S-update` | Scale to 0 | Upgrade order: GFS, then WRC, then HCC, then the Host |--After each scenario, the lane checks the following:--- **Pods.** The Host has a new pod UID and is Ready. HCC does not replace it- after it boots. The Host PVC has the same name and UID before and after.-- **Store.** The served ledger has no `processingLeases`. The lane then needs- one of two outcomes:- - a `GfsDownloadStore` warning that 1 legacy lease was discarded, with- `clerum_gfs_legacy_processing_leases_discarded_total` at 1;- - one warning that the outcome is unknown, with that counter at 0.-- **Journey.** The Desktop user sends the journey line and approves- `shell_exec` by clicking Approve. The fixture then requests- `clerum__gfs_download` for a seeded GFS file.- - The answer on screen and the fixture ledger must both carry the GFS- source's sha256 and byte count.- - The Host store must record exactly one new completed download with that- digest.- - The file in the Host workspace must hash to the same value.- - The journey makes no direct RPC calls and uses no storage shortcuts or- in-cluster mocks.--**Vacuity.** Run the vacuity lane once, and run it before the fixed lane. The-pre-fix Host quarantines every ledger record when it sees an inherited executor.-Those records stay on the PVC and keep counting against the caller's quota. The-fixed Host then discards the seeded lease when the runner restores it.--```sh-make minikube-build-legacy-lease-vacuity-fixture-make minikube-run-legacy-lease-restart-vacuity-```--The build archives base `74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e` and checks-that its store has the inherited-executor guard and no legacy-lease discard. It-builds `clerum/mcp-host:legacy-lease-vacuity` from that archive and puts this-HEAD's fixture layer on top as-`clerum/image-capabilities-mcp-host:legacy-lease-vacuity`. It then writes-`.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json` in the-canonical checkout. The runner checks the profile, the base revision, the-fixture-layer revision and the image identities against that manifest before it-patches anything.--The vacuity lane runs one `S-crash` scenario. It passes only when all of these-hold:--- the seeded lease is still in the ledger;-- no discard line was logged and the counter does not exist;-- the journey stops at the shell with- `LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy`, with no download- request.--**Restoration.** The runner journals the original HCC and Host replica counts-before Playwright starts. Zero is never a restored state. Restoration does the-following:--- deletes any seed pod of the run;-- scales the Host and then HCC back;-- waits for GFS, WRC, HCC and the Host to become Ready;-- restores the production Host image;-- waits until the Host's ledger has no `processingLeases`;-- fails when a live `e2e-gfs-legacy-lease-*` GFS fixture remains.--`make minikube-restore-image-capabilities` resumes an interrupted run the same-way.-### Orphaned lock recoveryIf `PROFILE_BUSY` reports that the lock has no valid owner PID, or that a stalediff --git a/scripts/e2e/build-legacy-lease-vacuity-image.sh b/scripts/e2e/build-legacy-lease-vacuity-image.shdeleted file mode 100755index 54ceac335..000000000--- a/scripts/e2e/build-legacy-lease-vacuity-image.sh+++ /dev/null@@ -1,143 +0,0 @@-#!/usr/bin/env bash-# Build the two images of the legacy processing-lease vacuity lane (issue #1022)-# inside the owned minikube profile, under the inherited T2 mutation lease:-#-# clerum/mcp-host:legacy-lease-vacuity-# mcp-host built from the pre-fix base revision (74e0d81d9, before #1019),-# whose store treats a legacy lease as an inherited executor;-# clerum/image-capabilities-mcp-host:legacy-lease-vacuity-# the provider-fixture layer of THIS HEAD on top of that Host.-#-# It then writes the manifest `make minikube-run-legacy-lease-restart-vacuity`-# proves before patching anything:-# <canonical>/.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json-#-# Nothing is published. Run it through-# `make minikube-build-legacy-lease-vacuity-fixture`.-set -euo pipefail--root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"-profile="${MINIKUBE_PROFILE:?MINIKUBE_PROFILE is required}"-base_revision=74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e-base_image=clerum/mcp-host:legacy-lease-vacuity-fixture_image=clerum/image-capabilities-mcp-host:legacy-lease-vacuity-deadline="${root}/scripts/minikube/run-with-deadline.mjs"--bash "${root}/scripts/minikube/require-t2-mutation-lock.sh"--[[ "${profile}" =~ ^clerum-.+-[a-f0-9]{8}$ ]] || {- printf 'LEGACY_LEASE_VACUITY: %s is not a branch profile\n' "${profile}" >&2- exit 1-}-[[ -z "$(git -C "${root}" status --porcelain)" ]] || {- printf 'LEGACY_LEASE_VACUITY: the worktree must be clean (the fixture layer is built from HEAD)\n' >&2- exit 1-}-head_revision="$(git -C "${root}" rev-parse --verify HEAD)"-[[ "$(git -C "${root}" cat-file -t "${base_revision}")" == commit ]] || {- printf 'LEGACY_LEASE_VACUITY: base revision %s is not in this repository\n' "${base_revision}" >&2- exit 1-}-common_dir="$(git -C "${root}" rev-parse --path-format=absolute --git-common-dir)"-canonical="$(cd -- "$(dirname -- "${common_dir}")" && pwd -P)"-[[ "$(basename -- "${canonical}")" == evenfire ]] || {- printf 'LEGACY_LEASE_VACUITY: canonical Evenfire checkout required\n' >&2- exit 1-}--run_dir="${canonical}/.local-notes/infra/runs/legacy-lease-vacuity/${profile}"-source_dir="${run_dir}/base-${base_revision:0:12}"--# Docker runs through the repository's isolated CLI environment (an empty task-# config, no ambient credentials), pointed at the profile's own daemon with-# `minikube docker-env`, exactly as scripts/minikube/build-images.sh does.-# shellcheck source=scripts/minikube/docker-cli-env.sh-source "${root}/scripts/minikube/docker-cli-env.sh"-cleanup() {- local status=$? cleanup_status=0- trap - EXIT INT TERM- docker_cli_env_cleanup || cleanup_status=$?- rm -rf -- "${source_dir}"- if [[ "${status}" -eq 0 && "${cleanup_status}" -ne 0 ]]; then status="${cleanup_status}"; fi- exit "${status}"-}-trap cleanup EXIT-trap 'exit 130' INT-trap 'exit 143' TERM--mkdir -p "${run_dir}"-rm -rf -- "${source_dir}"-mkdir -p "${source_dir}"-git -C "${root}" archive --format=tar "${base_revision}" | tar -x -C "${source_dir}"--# The archive must be the pre-fix store: it keeps the inherited-executor guard-# and has no legacy-lease discard. Either check failing means a wrong base.-store="${source_dir}/mcp-host/src/internalTools/gfsDownloadStore.ts"-grep -q 'hasInheritedExecutors' "${store}" || {- printf 'LEGACY_LEASE_VACUITY: base store has no inherited-executor guard\n' >&2- exit 1-}-if grep -q 'discardLegacyProcessingLeases' "${store}"; then- printf 'LEGACY_LEASE_VACUITY: base store already discards legacy leases\n' >&2- exit 1-fi--docker_cli_env_prepare-docker_env_output="$(node "${deadline}" --timeout-seconds 60 --kill-grace-seconds 5 --label minikube-docker-env -- \- minikube -p "${profile}" docker-env --shell bash)"-[[ -n "${docker_env_output}" ]] || {- printf 'LEGACY_LEASE_VACUITY: minikube returned an empty Docker environment\n' >&2- exit 1-}-eval "${docker_env_output}"-unset DOCKER_API_VERSION--inspect() {- docker_cli_run_public "inspect-$1" 60 docker image inspect "$1" --format "$2" | tr -d '\r\n'-}--# The base Host image from the archived pre-fix tree. Its build context is the-# archive root, which carries that revision's own root .dockerignore.-docker_cli_run_public legacy-lease-vacuity-host-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \- docker build --tag "${base_image}" --file "${source_dir}/mcp-host/Dockerfile" "${source_dir}"-base_id="$(inspect "${base_image}" '{{.Id}}')"--# The provider-fixture layer of THIS HEAD on top of that Host.-docker_cli_run_public legacy-lease-vacuity-fixture-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \- docker build --tag "${fixture_image}" \- --build-arg "MCP_HOST_IMAGE=${base_image}" \- --file "${root}/tests/e2e/fixtures/image-capabilities/Dockerfile" "${root}"-fixture_id="$(inspect "${fixture_image}" '{{.Id}}')"--for id in "${base_id}" "${fixture_id}"; do- [[ "${id}" =~ ^sha256:[a-f0-9]{64}$ ]] || {- printf 'LEGACY_LEASE_VACUITY: unexpected image id "%s"\n' "${id}" >&2- exit 1- }-done-# The fixture image must extend exactly the base just built.-parent_layers="$(inspect "${base_image}" '{{json .RootFS.Layers}}')"-fixture_layers="$(inspect "${fixture_image}" '{{json .RootFS.Layers}}')"-[[ "${fixture_layers}" == "${parent_layers%]},"* ]] || {- printf 'LEGACY_LEASE_VACUITY: the fixture layer does not extend the pre-fix Host image\n' >&2- exit 1-}--manifest="${run_dir}/manifest.json"-temporary="$(mktemp "${run_dir}/manifest.json.XXXXXX")"-node -e '-const [file, profile, baseRevision, head, baseImage, baseId, fixtureImage, fixtureId] = process.argv.slice(1)-const manifest = {- kind: "legacy-lease-vacuity",- profile,- baseRevision,- fixtureLayerRevision: head,- images: { [baseImage]: baseId, [fixtureImage]: fixtureId },- derivedFrom: { [fixtureImage]: { ref: baseImage, id: baseId } },- builtAt: new Date().toISOString(),-}-require("node:fs").writeFileSync(file, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 })-' "${temporary}" "${profile}" "${base_revision}" "${head_revision}" \- "${base_image}" "${base_id}" "${fixture_image}" "${fixture_id}"-mv -f -- "${temporary}" "${manifest}"-printf 'LEGACY_LEASE_VACUITY_BUILT manifest=%s base=%s fixture=%s\n' "${manifest}" "${base_id}" "${fixture_id}"diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjsindex 3544bc021..91594cc2f 100644--- a/scripts/e2e/image-capabilities-fixture.mjs+++ b/scripts/e2e/image-capabilities-fixture.mjs@@ -42,69 +42,6 @@ export const playwrightLanes = {label: 'document-upload-playwright',grep: 'document-upload fixture: ',},- // Issue #1022: legacy processing-lease restart scenarios on the fixed Host.- 'legacy-lease-restart': {- config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',- label: 'legacy-lease-restart-playwright',- grep: 'legacy-lease-restart fixture: ',- // S-crash, S-graceful, S-hcc, S-gfs and S-update: a shorter list is a lost scenario.- expectedSpecs: 5,- },- // The same journey on a Host built from the pre-fix base revision: it must- // fail at the shell step with download_busy, or the lane proves nothing.- 'legacy-lease-restart-vacuity': {- config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',- label: 'legacy-lease-restart-vacuity-playwright',- grep: 'legacy-lease-restart vacuity: ',- expectedSpecs: 1,- },-}--/**- * The pre-fix revision the vacuity lane runs (#1022 base, before #1019), and- * the two images `scripts/e2e/build-legacy-lease-vacuity-image.sh` builds from- * it. They never share a tag with the HEAD images.- */-export const legacyLeaseVacuity = Object.freeze({- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',- baseImage: 'clerum/mcp-host:legacy-lease-vacuity',- fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',- manifestKind: 'legacy-lease-vacuity',-})--export function legacyLeaseVacuityManifestPath(canonical, profile) {- return path.join(- canonical,- '.local-notes/infra/runs/legacy-lease-vacuity',- profile,- 'manifest.json'- )-}--/** Label of the seed pods the legacy-lease spec creates; restoration deletes them. */-export const legacyLeaseSeedLabel = 'evenfire.ai/legacy-lease-seed'--/**- * Runtime of each lane: the Host image HCC is pointed at, the Playwright- * deadline, and whether the lane restarts workloads (legacy-lease lanes scale- * HCC and the Host to zero and restart GFS and WRC, so restoration must bring- * them back).- */-const laneRuntimes = {- image: { fixtureImage, timeoutSeconds: 900, legacyLease: null },- 'document-upload': { fixtureImage, timeoutSeconds: 900, legacyLease: null },- // Five scenarios, each with workload stop/start cycles and one Desktop launch.- 'legacy-lease-restart': { fixtureImage, timeoutSeconds: 3600, legacyLease: 'fixed' },- 'legacy-lease-restart-vacuity': {- fixtureImage: legacyLeaseVacuity.fixtureImage,- timeoutSeconds: 900,- legacyLease: 'vacuity',- },-}--export function laneRuntime(lane) {- if (!Object.hasOwn(laneRuntimes, lane)) throw new Error(`Unknown lane ${JSON.stringify(lane)}`)- return laneRuntimes[lane]}export function playwrightLaneConfig(env) {@@ -136,7 +73,7 @@ const isCount = value => Number.isSafeInteger(value) && value >= 0* test and no unexpected, skipped or flaky outcome.*/export function playwrightVerdict(report, { lane, configFile }) {- const { grep, expectedSpecs } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })const reported = report?.config?.configFileif (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))throw new Error('Playwright report config mismatch')@@ -165,10 +102,6 @@ export function playwrightVerdict(report, { lane, configFile }) {)if (titles.some(title => !title.startsWith(grep)))throw new Error(`Playwright report has specs outside the ${lane} lane`)- if (expectedSpecs !== undefined && titles.length !== expectedSpecs)- throw new Error(- `Playwright report lists ${titles.length} ${lane} specs, expected ${expectedSpecs}`- )return 'PASS'}@@ -204,53 +137,6 @@ export function proveImages(manifest, observed, head, profile) {return { head, profile, images: observed }}-/**- * Proves the vacuity images: the Host layer was built from the pre-fix base- * revision, the fixture layer from this HEAD, both for this profile, and the- * fixture layer derives from the exact base image present in the node.- */-export function proveVacuityImages(manifest, observed, head, profile) {- const { baseRevision, baseImage: vacuityBase, fixtureImage: vacuityFixture } = legacyLeaseVacuity- if (- !/^[a-f0-9]{40}$/.test(head) ||- manifest?.kind !== legacyLeaseVacuity.manifestKind ||- manifest.profile !== profile- )- throw new Error('Vacuity image ownership mismatch')- if (manifest.baseRevision !== baseRevision)- throw new Error('Vacuity Host image was not built from the pre-fix base revision')- if (manifest.fixtureLayerRevision !== head)- throw new Error('Vacuity fixture layer was not built from this HEAD')- for (const ref of [vacuityBase, vacuityFixture]) {- const id = manifest.images?.[ref]- if (!/^sha256:[a-f0-9]{64}$/.test(id ?? '') || id !== observed[ref])- throw new Error('Vacuity image identity mismatch')- }- const binding = manifest.derivedFrom?.[vacuityFixture]- if (binding?.ref !== vacuityBase || binding.id !== observed[vacuityBase])- throw new Error('Vacuity fixture derived-base mismatch')- return { head, profile, baseRevision, images: observed }-}--/**- * The legacy-lease lanes scale HCC and the Host. Each must be back at its- * recorded replica count; zero is never a restored state.- */-export function legacyLeaseReplicaPlan(recorded, live) {- const plan = []- // The Host first: HCC sets the Host replicas itself, and a Host already at- // its count makes HCC's first reconcile a no-op.- for (const key of ['host', 'hcc']) {- const want = recorded?.[key]- if (!Number.isSafeInteger(want) || want < 1)- throw new Error(`Recorded ${key} replicas must be a positive integer`)- if (!Number.isSafeInteger(live?.[key]) || live[key] < 0)- throw new Error(`Live ${key} replicas are unreadable`)- if (live[key] !== want) plan.push({ key, replicas: want })- }- return plan-}-export function modelInputs(runId) {if (!/^image-capabilities-[a-f0-9]{12}$/.test(runId)) throw new Error('Invalid fixture run')return [@@ -370,7 +256,6 @@ async function main() {label: laneLabel,grep: laneGrep,} = selectFixtureAction(process.argv[2], process.env)- const runtime = action === 'run' ? laneRuntime(lane) : nullconst head = command('git', ['rev-parse', 'HEAD']).trim()const branch = command('git', ['branch', '--show-current']).trim()if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')@@ -454,8 +339,6 @@ async function main() {profile,head,root,- fixtureImage: runtime.fixtureImage,- legacyLease: runtime.legacyLease,rows: [],changes: {},restored: false,@@ -467,10 +350,6 @@ async function main() {(action === 'run' && state.head !== head))throw new Error('Run state identity mismatch')- // Restore reads the lane's Host image from the persisted state, never from the- // environment, so a stale lane variable cannot point restoration elsewhere.- if (typeof state.fixtureImage !== 'string' || !state.fixtureImage)- throw new Error('Run state has no fixture image; restore it with the runner that created it')const save = () => {const destination = path.join(evidence, 'state.json')if (fs.existsSync(destination)) {@@ -545,86 +424,8 @@ async function main() {if (!response.ok) throw new Error(`Fixture admin operation failed (${response.status})`)return response.status === 204 ? null : response.json()}- const deploymentReplicas = (namespace, name) => get(namespace, 'deployment', name).spec.replicas- const restoreLegacyLeaseWorkloads = () => {- // Seed pods mount the Host PVC; none may outlive the run.- kc([- '-n',- 'mcp-host',- 'delete',- 'pod',- '-l',- `${legacyLeaseSeedLabel}=${state.runId}`,- '--ignore-not-found',- '--wait=true',- '--timeout=90s',- ])- const remaining = kc([- '-n',- 'mcp-host',- 'get',- 'pod',- '-l',- legacyLeaseSeedLabel,- '-o',- 'jsonpath={.items[*].metadata.name}',- ]).trim()- if (remaining) throw new Error('Legacy-lease seed pods remain in mcp-host')- const liveReplicas = () => ({- host: deploymentReplicas('mcp-host', 'chatllm'),- hcc: deploymentReplicas('control-plane', 'host-context-controller'),- })- for (const { key, replicas } of legacyLeaseReplicaPlan(- state.legacyLeaseReplicas,- liveReplicas()- )) {- const [namespace, name] =- key === 'host' ? ['mcp-host', 'chatllm'] : ['control-plane', 'host-context-controller']- kc(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])- }- for (const [namespace, name] of [- ['gfs', 'gfsc-writer'],- ['gfs', 'gfsc-reader'],- ['control-plane', 'workflow-recipes'],- ['control-plane', 'host-context-controller'],- ['mcp-host', 'chatllm'],- ])- kc(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=150s'])- if (legacyLeaseReplicaPlan(state.legacyLeaseReplicas, liveReplicas()).length)- throw new Error('HCC or the Host is not back at its recorded replicas')- }- // The Host discards a legacy lease at initialize, which runs at boot and can- // wait out writer contention, so the ledger is polled with a bound.- const waitLedgerWithoutLegacyLeases = async () => {- const until = Date.now() + 240_000- let last = 'unread'- while (Date.now() < until) {- try {- const ledger = JSON.parse(- kc([- '-n',- 'mcp-host',- 'exec',- 'deployment/chatllm',- '-c',- 'mcp-host',- '--',- 'cat',- '/workspace/.gfs-download-store/ledger-v1.json',- ])- )- if (!Object.hasOwn(ledger, 'processingLeases')) return- last = 'processingLeases present'- } catch (error) {- last = error.message- }- await new Promise(resolve => setTimeout(resolve, 2_000))- }- throw new Error(`Restored Host ledger still carries legacy processing leases (${last})`)- }const restoreRuntime = async () => {if (state.restored) return- if (state.legacyLease && state.legacyLeaseReplicas) restoreLegacyLeaseWorkloads()// Restore the model before removing its fixture-only catalog rows.if (state.changes.host) {const current = get('mcp-host', 'host', 'chatllm')@@ -699,7 +500,7 @@ async function main() {path: state.hcc.path.replace(/value$/, 'name'),value: 'CONTEXT_MAPPER_HOST_IMAGE',},- { op: 'test', path: state.hcc.path, value: state.fixtureImage },+ { op: 'test', path: state.hcc.path, value: fixtureImage },{ op: 'replace', path: state.hcc.path, value: state.hcc.image },unmark,])@@ -781,9 +582,6 @@ async function main() {'process.stdout.write(String(!process.env.EVENFIRE_IMAGE_CAPABILITIES_FIXTURE && !require("node:fs").existsSync("/tmp/image-capabilities-evidence.json")))',]).trim()if (predicate !== 'true') throw new Error('Fixture state remains in Host')- // The vacuity lane leaves its seeded lease on the PVC (the pre-fix Host- // cannot discard it); the restored fixed Host must discard it at boot.- if (state.legacyLease) await waitLedgerWithoutLegacyLeases()state.restored = truesave()}@@ -798,20 +596,11 @@ async function main() {}let failuretry {- const vacuity = runtime.legacyLease === 'vacuity'const manifest = JSON.parse(- fs.readFileSync(- vacuity- ? legacyLeaseVacuityManifestPath(canonical, profile)- : path.join(root, 'deploy/minikube/.image-manifest.json'),- 'utf8'- )+ fs.readFileSync(path.join(root, 'deploy/minikube/.image-manifest.json'), 'utf8'))- const proofRefs = vacuity- ? [legacyLeaseVacuity.baseImage, legacyLeaseVacuity.fixtureImage]- : [baseImage, fixtureImage]const observed = Object.fromEntries(- proofRefs.map(ref => [+ [baseImage, fixtureImage].map(ref => [ref,command('minikube', ['-p',@@ -827,9 +616,7 @@ async function main() {]).trim(),]))- state.imageProof = vacuity- ? proveVacuityImages(manifest, observed, head, profile)- : proveImages(manifest, observed, head, profile)+ state.imageProof = proveImages(manifest, observed, head, profile)save()const host = get('mcp-host', 'host', 'chatllm')const hosts = JSON.parse(kc(['-n', 'mcp-host', 'get', 'hosts', '-o', 'json']))@@ -985,7 +772,7 @@ async function main() {value: 'CONTEXT_MAPPER_HOST_IMAGE',},{ op: 'test', path: state.hcc.path, value: state.hcc.image },- { op: 'replace', path: state.hcc.path, value: state.fixtureImage },+ { op: 'replace', path: state.hcc.path, value: fixtureImage },])kc(['-n',@@ -995,7 +782,7 @@ async function main() {'deployment/host-context-controller','--timeout=120s',])- await waitHostImage(state.fixtureImage)+ await waitHostImage(fixtureImage)const wire = JSON.parse(kc(['-n',@@ -1024,26 +811,6 @@ async function main() {QA_RECORDER_IMAGE_MODEL_UNSUPPORTED: 'glm-5.3',QA_RECORDER_IMAGE_MODEL_UNKNOWN: models[2].model,QA_RECORDER_ROOT: evidence,- ...(runtime.legacyLease- ? {- // The GFS fixture helpers name the cluster through this variable.- E2E_K8S_CONTEXT: profile,- LEGACY_LEASE_LANE_MODE: runtime.legacyLease,- LEGACY_LEASE_SEED_LABEL_VALUE: state.runId,- }- : {}),- }- if (runtime.legacyLease) {- // The spec scales these to zero; restoration needs the values it found.- const replicas = {- host: deploymentReplicas('mcp-host', 'chatllm'),- hcc: deploymentReplicas('control-plane', 'host-context-controller'),- }- // Validates the recorded counts: a workload already at zero is refused- // here, before the spec runs, instead of becoming the restore target.- legacyLeaseReplicaPlan(replicas, replicas)- state.legacyLeaseReplicas = replicas- save()}command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)@@ -1059,7 +826,7 @@ async function main() {['scripts/minikube/run-with-deadline.mjs','--timeout-seconds',- String(runtime.timeoutSeconds),+ '900','--heartbeat-seconds','20','--kill-grace-seconds',@@ -1080,7 +847,7 @@ async function main() {...environment,PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),},- timeout: (runtime.timeoutSeconds + 30) * 1000,+ timeout: 930_000,})report(output)@@ -1108,9 +875,7 @@ async function main() {try {const current = get('mcp-host', 'deployment', 'chatllm')if (- current.spec.template.spec.containers.some(- container => container.image === state.fixtureImage- )+ current.spec.template.spec.containers.some(container => container.image === fixtureImage)) {const logs = kc(['-n', 'mcp-host', 'logs', 'deployment/chatllm', '--tail=60'])state.startupDiagnostic = [@@ -1137,32 +902,6 @@ async function main() {)}uninstallSignalRestore()- if (!failure && state.legacyLease) {- // The spec removes its GFS fixtures; a live row left behind fails the run.- try {- const live = kc([- '-n',- 'control-plane',- 'exec',- 'deployment/control-postgres',- '--',- 'psql',- '-v',- 'ON_ERROR_STOP=1',- '-U',- 'postgres',- '-d',- 'profiles',- '-t',- '-A',- '-c',- "SELECT count(*) FROM gfs_resources WHERE drive = 'main' AND deleted_at IS NULL AND path_cache LIKE '/e2e-gfs-legacy-lease-%';",- ]).trim()- if (live !== '0') failure = new Error(`Legacy-lease GFS fixture rows remain (${live})`)- } catch (error) {- failure = new Error(`Legacy-lease GFS fixture check failed: ${error.message}`)- }- }if (failure) throw failureprocess.stdout.write(fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjsindex 0e81c46f5..13ae290e9 100644--- a/scripts/e2e/image-capabilities-fixture.test.mjs+++ b/scripts/e2e/image-capabilities-fixture.test.mjs@@ -8,15 +8,10 @@ import {fixtureImage,fixturePassLine,installSignalRestore,- laneRuntime,- legacyLeaseReplicaPlan,- legacyLeaseVacuity,- legacyLeaseVacuityManifestPath,modelInputs,playwrightLaneConfig,playwrightVerdict,proveImages,- proveVacuityImages,requireOwnedResource,runAnnotation,sanitizeFixtureReport,@@ -357,190 +352,3 @@ test('the playwright log redacts the admin password, the session cookie and bearassert.ok(!report.includes('opaque-bearer-value'))assert.match(report, /3 passed \(41\.2s\)/)})--// ----------------------------------------------------------------------------// Issue #1022: legacy processing-lease restart lanes.-// -----------------------------------------------------------------------------test('the legacy-lease lanes share one config and select disjoint journeys', () => {- const fixed = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart' })- const vacuity = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart-vacuity' })- assert.equal(- fixed.config,- 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts'- )- assert.equal(vacuity.config, fixed.config)- assert.equal(fixed.grep, 'legacy-lease-restart fixture: ')- assert.equal(vacuity.grep, 'legacy-lease-restart vacuity: ')- assert.equal(vacuity.grep.startsWith(fixed.grep), false)- assert.equal(fixed.grep.startsWith(vacuity.grep), false)- const options = {- lane: 'legacy-lease-restart',- configFile: laneConfigFile('legacy-lease-restart'),- }- const scenarioTitles = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'].map(- scenario => `legacy-lease-restart fixture: ${scenario}`- )- // Witness: the lane's own five scenario titles are PASS.- assert.equal(- playwrightVerdict(- playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles }),- options- ),- 'PASS'- )- // A clean report that lost scenarios is not the lane.- assert.throws(- () =>- playwrightVerdict(- playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles.slice(0, 2) }),- options- ),- /lists 2 legacy-lease-restart specs, expected 5/- )- // The vacuity lane is exactly its one S-crash journey.- assert.equal(- playwrightVerdict(- playwrightReport({- lane: 'legacy-lease-restart-vacuity',- titles: ['legacy-lease-restart vacuity: S-crash'],- }),- {- lane: 'legacy-lease-restart-vacuity',- configFile: laneConfigFile('legacy-lease-restart-vacuity'),- }- ),- 'PASS'- )- // A vacuity title in the fixed lane's report is outside that lane.- assert.throws(- () =>- playwrightVerdict(- playwrightReport({- lane: 'legacy-lease-restart',- titles: ['legacy-lease-restart vacuity: S-crash'],- }),- options- ),- /outside the legacy-lease-restart lane/- )-})--test('each lane runtime names its Host image and deadline; existing lanes are unchanged', () => {- for (const lane of ['image', 'document-upload']) {- assert.deepEqual(laneRuntime(lane), { fixtureImage, timeoutSeconds: 900, legacyLease: null })- }- assert.deepEqual(laneRuntime('legacy-lease-restart'), {- fixtureImage,- timeoutSeconds: 3600,- legacyLease: 'fixed',- })- assert.deepEqual(laneRuntime('legacy-lease-restart-vacuity'), {- fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',- timeoutSeconds: 900,- legacyLease: 'vacuity',- })- // The vacuity images never reuse a HEAD tag.- assert.notEqual(legacyLeaseVacuity.fixtureImage, fixtureImage)- assert.notEqual(legacyLeaseVacuity.baseImage, baseImage)- for (const lane of ['typo', '__proto__', '']) {- assert.throws(() => laneRuntime(lane), /Unknown lane/, lane)- }-})--test('the vacuity manifest lives under the canonical run root, per profile', () => {- assert.equal(- legacyLeaseVacuityManifestPath('/repo/evenfire', profile),- `/repo/evenfire/.local-notes/infra/runs/legacy-lease-vacuity/${profile}/manifest.json`- )-})--const vacuityIds = {- [legacyLeaseVacuity.baseImage]: `sha256:${'e'.repeat(64)}`,- [legacyLeaseVacuity.fixtureImage]: `sha256:${'f'.repeat(64)}`,-}-function vacuityManifest() {- return {- kind: 'legacy-lease-vacuity',- profile,- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',- fixtureLayerRevision: head,- images: { ...vacuityIds },- derivedFrom: {- [legacyLeaseVacuity.fixtureImage]: {- ref: legacyLeaseVacuity.baseImage,- id: vacuityIds[legacyLeaseVacuity.baseImage],- },- },- }-}--test('vacuity images must come from the pre-fix base and this HEAD, for this profile', () => {- // Witness: the well-formed manifest is proven.- assert.deepEqual(proveVacuityImages(vacuityManifest(), vacuityIds, head, profile), {- head,- profile,- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',- images: vacuityIds,- })- for (const [label, mutate, pattern] of [- ['other kind', value => (value.kind = 'image-manifest'), /ownership/],- ['other profile', value => (value.profile = 'clerum-other-12345678'), /ownership/],- ['built from HEAD', value => (value.baseRevision = head), /pre-fix base/],- ['stale fixture layer', value => (value.fixtureLayerRevision = 'd'.repeat(40)), /this HEAD/],- [- 'other base id',- value => (value.images[legacyLeaseVacuity.baseImage] = `sha256:${'1'.repeat(64)}`),- /identity/,- ],- [- 'fixture from another base',- value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].id = `sha256:${'2'.repeat(64)}`),- /derived-base/,- ],- [- 'fixture from the HEAD Host',- value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].ref = baseImage),- /derived-base/,- ],- ]) {- const changed = vacuityManifest()- mutate(changed)- assert.throws(() => proveVacuityImages(changed, vacuityIds, head, profile), pattern, label)- }- // The node holds a different image than the manifest recorded.- assert.throws(- () =>- proveVacuityImages(- vacuityManifest(),- { ...vacuityIds, [legacyLeaseVacuity.fixtureImage]: `sha256:${'3'.repeat(64)}` },- head,- profile- ),- /identity/- )- // A HEAD image manifest is never accepted as a vacuity proof.- assert.throws(() => proveVacuityImages(manifest(), ids, head, profile), /ownership/)-})--test('restoration scales the Host before HCC and never accepts zero as restored', () => {- const recorded = { host: 1, hcc: 1 }- // Witness: both at zero yields a plan, Host first.- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 0, hcc: 0 }), [- { key: 'host', replicas: 1 },- { key: 'hcc', replicas: 1 },- ])- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 0 }), [- { key: 'hcc', replicas: 1 },- ])- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 1 }), [])- for (const [label, bad] of [- ['zero host', { host: 0, hcc: 1 }],- ['zero hcc', { host: 1, hcc: 0 }],- ['missing', {}],- ['fractional', { host: 1.5, hcc: 1 }],- ]) {- assert.throws(() => legacyLeaseReplicaPlan(bad, { host: 1, hcc: 1 }), /Recorded/, label)- }- assert.throws(() => legacyLeaseReplicaPlan(recorded, { host: undefined, hcc: 1 }), /unreadable/)-})diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.mdindex 624c1216b..b3a207519 100644--- a/tests/e2e/fixtures/image-capabilities/README.md+++ b/tests/e2e/fixtures/image-capabilities/README.md@@ -55,39 +55,6 @@ file chip, sends, and requires the digest of the attached bytes in the answer,one `document-read-requested` row, one `document-answer` row carrying thatdigest, no image row and no `documentFailures`.-## Legacy processing-lease restart journey (issue #1022)--`make minikube-run-legacy-lease-restart` is the same runner with-`IMAGE_CAPABILITIES_LANE=legacy-lease-restart`; the config is-`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts` and-the cluster side lives in `helpers/legacyLeaseRestart.ts`. The user message-carries one line:--```-LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>-```--The fixture then scripts the turns from the message list alone: it asks for-`shell_exec` with a command whose output is `<marker>-shell-ok` (a token absent-from the command), then for `clerum__gfs_download` of the named resource once-the shell result echoes the token, and answers-`LEGACY-LEASE-FIXTURE-OK marker=… sha256=… bytes=…` from the download receipt.-A shell or download result without what the next step needs is answered with-`LEGACY-LEASE-FIXTURE-SHELL-FAILED code=<code>` or-`LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=<code>`, where the code is a GFS-download store error code (for example `download_busy`) or `unrecognized`.-`download_busy` is what the pre-fix Host returns for a legacy lease. The ledger-rows are `legacy-lease-shell-requested`, `legacy-lease-download-requested`,-`legacy-lease-answer` (with `downloadSha256` and `downloadBytes`),-`legacy-lease-shell-failed` and `legacy-lease-download-failed` (with-`failureCode`). It never records command output.--The vacuity lane, `make minikube-run-legacy-lease-restart-vacuity`, runs the same-fixture layer on a Host built from the pre-fix base by-`make minikube-build-legacy-lease-vacuity-fixture`. The procedure and the-signals checked after each scenario are in-`docs/testing/minikube-t2-runbook.md` (Legacy processing-lease restart lane).-## Local executionUse Node 24 and Bash 5, a clean development branch based on current `origin/dev`,diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjsindex 1ecd30441..85bf71b60 100644--- a/tests/e2e/fixtures/image-capabilities/provider.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs@@ -160,12 +160,6 @@ export const FIXTURE_REJECTION_REASONS = Object.freeze(['document-tool-output-unreadable','document-page-range-malformed','document-stream-unsupported',- 'legacy-lease-shell-not-offered',- 'legacy-lease-download-not-offered',- 'legacy-lease-tool-result-unexpected',- 'legacy-lease-tool-result-count',- 'legacy-lease-download-result-malformed',- 'legacy-lease-stream-unsupported',])/** A refusal with a stable machine-readable reason code for the evidence tally. */@@ -518,187 +512,6 @@ function classifyDocumentTurn(body) {}}-/*- * Legacy processing-lease restart journey (issue #1022, `make- * minikube-run-legacy-lease-restart`). The user message carries one line- *- * LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>- *- * and the fixture scripts two tool calls from the message list alone:- *- * 1. no tool result yet: call `shell_exec` with a command whose output is- * `<marker>-shell-ok`. That token does not appear in the command, so only- * a command that really ran can echo it back;- * 2. one shell result carrying that token: call `clerum__gfs_download` for the- * named resource;- * 3. a shell result and a download result: answer with the sha256 and byte- * count the Host's download receipt reports.- *- * A shell or download result that does not carry what step 2 or 3 needs is- * answered with a failure line naming a closed-set code (a store error code such- * as `download_busy`, or `unrecognized`), so the journey fails on screen at the- * step that broke. The ledger records only the step, the code, and for the- * answer the receipt's digest and byte count, never the command output.- */-const SHELL_TOOL = 'shell_exec'-const GFS_DOWNLOAD_TOOL = 'clerum__gfs_download'-const LEGACY_LEASE_JOURNEY_LINE =- /LEGACY_LEASE_JOURNEY marker=(legacy-lease-[a-f0-9]{16}) drive=([a-z0-9][a-z0-9-]{0,62}) resourceId=([a-f0-9]{32})/-const LEGACY_LEASE_SHELL_TOKEN_SUFFIX = '-shell-ok'-export const LEGACY_LEASE_ANSWER_PREFIX = 'LEGACY-LEASE-FIXTURE-OK'-export const LEGACY_LEASE_SHELL_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-SHELL-FAILED'-export const LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED'--/**- * Codes a failed shell or download step may report: the GFS download store's- * error codes (`GfsDownloadStoreErrorCode`), plus `unrecognized` for a result- * that names none of them. A closed set of fixed strings, never tool output.- */-export const LEGACY_LEASE_FAILURE_CODES = Object.freeze([- 'caller_mismatch',- 'caller_quota_exceeded',- 'corrupt_store_ledger',- 'download_expired',- 'download_missing',- 'download_busy',- 'host_quota_exceeded',- 'publication_cancelled',- 'storage_write_failed',- 'unsupported_store_schema',- 'workspace_unavailable',- 'writer_locked',- 'unrecognized',-])--/** The store error text (`GfsDownloadStoreError`): `GFS download store failed (<code>)`. */-const STORE_FAILURE = /GFS download store failed \(([a-z_]+)\)/--function toolOutputPattern(name) {- return new RegExp(- `^<tool_output name="${name}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`- )-}-const SHELL_TOOL_OUTPUT = toolOutputPattern(SHELL_TOOL)-const GFS_DOWNLOAD_TOOL_OUTPUT = toolOutputPattern(GFS_DOWNLOAD_TOOL)--class FixtureLegacyLeaseError extends Error {- constructor(code, message) {- super(message)- this.name = 'FixtureLegacyLeaseError'- this.code = code- }-}--function failureCode(content) {- const code = STORE_FAILURE.exec(content)?.[1]- return code && LEGACY_LEASE_FAILURE_CODES.includes(code) ? code : 'unrecognized'-}--function offeredTool(body, name) {- const tools = Array.isArray(body.tools) ? body.tools : []- return tools.some(tool => tool?.function?.name === name)-}--/**- * Classifies a legacy-lease journey turn.- *- * - `null`: the last user message carries no journey line, or no tool is offered- * (a title or summary request), so the request is not part of the journey.- * - `{ stage: 'shell', marker }`: ask for the shell command.- * - `{ stage: 'download', drive, resourceId }`: the shell echoed the token.- * - `{ stage: 'shell-failed', code }`: the shell result lacks the token.- * - `{ stage: 'answer', marker, sha256, sizeBytes }`: the download receipt.- * - `{ stage: 'download-failed', code }`: the download result is an error.- */-function classifyLegacyLeaseTurn(body) {- if (!Array.isArray(body.tools) || body.tools.length === 0) return null- const userIndex = lastUserIndex(body.messages)- if (userIndex < 0) return null- const user = body.messages[userIndex]- if (typeof user.content !== 'string') return null- const journey = LEGACY_LEASE_JOURNEY_LINE.exec(user.content)- if (!journey) return null- const [, marker, drive, resourceId] = journey-- const toolMessages = body.messages- .slice(userIndex + 1)- .filter(message => message?.role === 'tool')- if (toolMessages.length > 2)- throw new FixtureLegacyLeaseError(- 'legacy-lease-tool-result-count',- `the journey makes two tool calls, received ${toolMessages.length} results`- )- const contents = toolMessages.map(message => message.content)- if (contents.some(content => typeof content !== 'string'))- throw new FixtureLegacyLeaseError(- 'legacy-lease-tool-result-unexpected',- 'a tool result is not a string'- )-- if (contents.length === 0) {- if (!offeredTool(body, SHELL_TOOL))- throw new FixtureLegacyLeaseError(- 'legacy-lease-shell-not-offered',- 'the Host did not offer shell_exec for the journey'- )- return { stage: 'shell', marker }- }-- const shell = SHELL_TOOL_OUTPUT.exec(contents[0])- if (!shell)- throw new FixtureLegacyLeaseError(- 'legacy-lease-tool-result-unexpected',- 'the first tool result is not a shell_exec result'- )- const token = `${marker}${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}`- const echoed = shell[1].split('\n').some(line => line.trim() === token)- if (contents.length === 1) {- if (!echoed) return { stage: 'shell-failed', code: failureCode(shell[1]) }- if (!offeredTool(body, GFS_DOWNLOAD_TOOL))- throw new FixtureLegacyLeaseError(- 'legacy-lease-download-not-offered',- 'the Host did not offer clerum__gfs_download after the shell step'- )- return { stage: 'download', drive, resourceId }- }- if (!echoed)- throw new FixtureLegacyLeaseError(- 'legacy-lease-tool-result-unexpected',- 'a download result follows a shell result without the token'- )-- const download = GFS_DOWNLOAD_TOOL_OUTPUT.exec(contents[1])- if (!download)- throw new FixtureLegacyLeaseError(- 'legacy-lease-tool-result-unexpected',- 'the second tool result is not a clerum__gfs_download result'- )- let receipt- try {- receipt = JSON.parse(download[1])- } catch {- return { stage: 'download-failed', code: failureCode(download[1]) }- }- if (- !receipt ||- typeof receipt !== 'object' ||- receipt.delivery !== 'workspace_file' ||- typeof receipt.sha256 !== 'string' ||- !SHA256_HEX.test(receipt.sha256) ||- !isByteCount(receipt.sizeBytes)- )- throw new FixtureLegacyLeaseError(- 'legacy-lease-download-result-malformed',- 'the download result is not a workspace_file receipt with sha256 and sizeBytes'- )- return {- stage: 'answer',- marker,- sha256: receipt.sha256.toLowerCase(),- sizeBytes: receipt.sizeBytes,- }-}-const isByteCount = value => Number.isSafeInteger(value) && value >= 0const INTERNAL_HOST_SUFFIXES = [@@ -945,10 +758,6 @@ export function createImageFixtureFetch(documentReadRequests: 0,documentAnswers: 0,documentFailures: 0,- legacyLeaseShellRequests: 0,- legacyLeaseDownloadRequests: 0,- legacyLeaseAnswers: 0,- legacyLeaseFailures: 0,},attempts: [],}@@ -1102,74 +911,6 @@ export function createImageFixtureFetch(})}-- let legacyTurn- try {- legacyTurn = classifyLegacyLeaseTurn(body)- } catch (error) {- if (!(error instanceof FixtureLegacyLeaseError)) throw error- state.counters.legacyLeaseFailures += 1- return refuse(model, error.code, { code: error.code })- }- if (legacyTurn) {- if (stream) {- // A tool call cannot be answered in this fixture's SSE, exactly as for- // the document read.- state.counters.legacyLeaseFailures += 1- return refuse(model, 'legacy-lease-stream-unsupported', {- code: 'legacy_lease_stream_unsupported',- })- }- sequence += 1- const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`- const reply = (responseKind, content, fields = {}) => {- recordCall(model, null, responseKind, fields)- publish()- return jsonResponse(completionBody(completionId, model, content))- }- const call = (responseKind, name, args) => {- recordCall(model, null, responseKind)- publish()- return jsonResponse(toolCallCompletionBody(completionId, model, name, args))- }- switch (legacyTurn.stage) {- case 'shell':- state.counters.legacyLeaseShellRequests += 1- return call('legacy-lease-shell-requested', SHELL_TOOL, {- command: `printf '%s${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}\\n' ${legacyTurn.marker}`,- })- case 'download':- state.counters.legacyLeaseDownloadRequests += 1- return call('legacy-lease-download-requested', GFS_DOWNLOAD_TOOL, {- drive: legacyTurn.drive,- resourceId: legacyTurn.resourceId,- })- case 'shell-failed':- state.counters.legacyLeaseFailures += 1- return reply(- 'legacy-lease-shell-failed',- `${LEGACY_LEASE_SHELL_FAILED_PREFIX} code=${legacyTurn.code}`,- { failureCode: legacyTurn.code }- )- case 'download-failed':- state.counters.legacyLeaseFailures += 1- return reply(- 'legacy-lease-download-failed',- `${LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX} code=${legacyTurn.code}`,- { failureCode: legacyTurn.code }- )- case 'answer':- state.counters.legacyLeaseAnswers += 1- return reply(- 'legacy-lease-answer',- `${LEGACY_LEASE_ANSWER_PREFIX} marker=${legacyTurn.marker} ` +- `sha256=${legacyTurn.sha256} bytes=${legacyTurn.sizeBytes}`,- { downloadSha256: legacyTurn.sha256, downloadBytes: legacyTurn.sizeBytes }- )- default:- throw new Error(`Unhandled legacy-lease stage ${String(legacyTurn.stage)}`)- }- }state.counters.textOnlyResponses += 1return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })}diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjsindex 3beca6da5..a527950d2 100644--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs@@ -1091,12 +1091,6 @@ const EXPECTED_REJECTION_REASONS = ['image-png-malformed','image-png-unsupported-form','image-tile-grid-mismatch',- 'legacy-lease-download-not-offered',- 'legacy-lease-download-result-malformed',- 'legacy-lease-shell-not-offered',- 'legacy-lease-stream-unsupported',- 'legacy-lease-tool-result-count',- 'legacy-lease-tool-result-unexpected','provider-auth-mismatch','provider-body-invalid','provider-body-unsupported',@@ -1233,308 +1227,3 @@ test('every rejected row records its closed-set reason and no request content',assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)})--// ----------------------------------------------------------------------------// Legacy processing-lease restart journey (issue #1022): the fixture scripts-// shell_exec, then clerum__gfs_download, then answers from the receipt.-// -----------------------------------------------------------------------------const LEGACY_MARKER = 'legacy-lease-0123456789abcdef'-const LEGACY_RESOURCE = 'a'.repeat(32)-const LEGACY_SHA = 'b'.repeat(64)-const LEGACY_TOKEN = `${LEGACY_MARKER}-shell-ok`--const toolDefinition = name => ({- type: 'function',- function: { name, description: name, parameters: { type: 'object' } },-})-const LEGACY_TOOLS = [toolDefinition('shell_exec'), toolDefinition('clerum__gfs_download')]--function legacyUserMessage(marker = LEGACY_MARKER) {- return {- role: 'user',- content:- '<turn-context>\ndate: 2026-10-08\n</turn-context>\n\n' +- `Run the journey. LEGACY_LEASE_JOURNEY marker=${marker} drive=main resourceId=${LEGACY_RESOURCE}`,- }-}--function legacyMessages(results) {- const messages = [legacyUserMessage()]- for (const [index, [name, content]] of results.entries()) {- messages.push({- role: 'assistant',- content: null,- tool_calls: [{ id: `call-${index}`, type: 'function', function: { name, arguments: '{}' } }],- })- messages.push({ role: 'tool', tool_call_id: `call-${index}`, content })- }- return messages-}--const wrapped = (name, content) =>- `<tool_output name="${name}" sanitized="false">\n${content}\n</tool_output>`-const shellResult = content => ['shell_exec', wrapped('shell_exec', content)]-const downloadResult = content => ['clerum__gfs_download', wrapped('clerum__gfs_download', content)]-const receipt = (overrides = {}) =>- JSON.stringify({- delivery: 'workspace_file',- id: '11111111-2222-4333-8444-555555555555',- path: '.gfs-downloads/input-1/report.pdf',- sizeBytes: 42,- sha256: LEGACY_SHA,- ...overrides,- })--test('legacy-lease step 1: asks for a shell command whose output token is not in the command', async () => {- const h = harness()- const response = await call(- h,- chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS })- )- assert.equal(response.status, 200)- const body = await response.json()- assert.equal(body.choices[0].finish_reason, 'tool_calls')- const [toolCall] = body.choices[0].message.tool_calls- assert.equal(toolCall.function.name, 'shell_exec')- const { command } = JSON.parse(toolCall.function.arguments)- // The approval preview shows the marker, but only an executed command can- // produce the joined token.- assert.ok(command.includes(LEGACY_MARKER))- assert.equal(command.includes(LEGACY_TOKEN), false)- assert.equal(command, `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`)- const evidence = h.getEvidence()- assert.equal(evidence.counters.legacyLeaseShellRequests, 1)- assert.deepEqual(evidence.attempts, [- { model: VISUAL_MODEL, imageSha256: null, responseKind: 'legacy-lease-shell-requested' },- ])-})--test('legacy-lease step 2: a shell result with the token asks for the GFS download', async () => {- const h = harness()- const response = await call(- h,- chatBody(VISUAL_MODEL, legacyMessages([shellResult(`${LEGACY_TOKEN}\n`)]), {- tools: LEGACY_TOOLS,- })- )- const body = await response.json()- const [toolCall] = body.choices[0].message.tool_calls- assert.equal(toolCall.function.name, 'clerum__gfs_download')- assert.deepEqual(JSON.parse(toolCall.function.arguments), {- drive: 'main',- resourceId: LEGACY_RESOURCE,- })- assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 1)- assert.equal(h.getEvidence().attempts[0].responseKind, 'legacy-lease-download-requested')-})--test('legacy-lease step 2: a lease failure before command start is answered with its store code', async () => {- const h = harness()- const response = await call(- h,- chatBody(- VISUAL_MODEL,- legacyMessages([- shellResult(- 'Processing lease failed before command start: GFS download store failed (download_busy)'- ),- ]),- { tools: LEGACY_TOOLS }- )- )- const body = await response.json()- assert.equal(body.choices[0].finish_reason, 'stop')- assert.equal(- body.choices[0].message.content,- 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy'- )- const evidence = h.getEvidence()- assert.equal(evidence.counters.legacyLeaseFailures, 1)- assert.equal(evidence.counters.legacyLeaseDownloadRequests, 0)- assert.deepEqual(evidence.attempts, [- {- model: VISUAL_MODEL,- imageSha256: null,- responseKind: 'legacy-lease-shell-failed',- failureCode: 'download_busy',- },- ])-})--test('legacy-lease step 2: output without the token or a known code is unrecognized, never a download', async () => {- for (const content of [- // The command echoed back is not the token.- `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`,- 'GFS download store failed (not_a_store_code)',- `${LEGACY_TOKEN}-extra`,- ]) {- const h = harness()- const response = await call(- h,- chatBody(VISUAL_MODEL, legacyMessages([shellResult(content)]), { tools: LEGACY_TOOLS })- )- const body = await response.json()- assert.equal(- body.choices[0].message.content,- 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=unrecognized',- content- )- assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 0, content)- }-})--test('legacy-lease step 3: the answer carries the receipt sha256 and byte count', async () => {- const h = harness()- const response = await call(- h,- chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt())]), {- tools: LEGACY_TOOLS,- })- )- const body = await response.json()- assert.equal(- body.choices[0].message.content,- `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${LEGACY_SHA} bytes=42`- )- const evidence = h.getEvidence()- assert.equal(evidence.counters.legacyLeaseAnswers, 1)- assert.deepEqual(evidence.attempts, [- {- model: VISUAL_MODEL,- imageSha256: null,- responseKind: 'legacy-lease-answer',- downloadSha256: LEGACY_SHA,- downloadBytes: 42,- },- ])-})--test('legacy-lease step 3: a different receipt gives a different answer (not a constant)', async () => {- const other = 'c'.repeat(64)- const h = harness()- const response = await call(- h,- chatBody(- VISUAL_MODEL,- legacyMessages([- shellResult(LEGACY_TOKEN),- downloadResult(receipt({ sha256: other, sizeBytes: 7 })),- ]),- { tools: LEGACY_TOOLS }- )- )- const body = await response.json()- assert.equal(- body.choices[0].message.content,- `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${other} bytes=7`- )-})--test('legacy-lease step 3: a download error is answered with its store code', async () => {- const h = harness()- const response = await call(- h,- chatBody(- VISUAL_MODEL,- legacyMessages([- shellResult(LEGACY_TOKEN),- downloadResult('Error: GFS download store failed (workspace_unavailable)'),- ]),- { tools: LEGACY_TOOLS }- )- )- const body = await response.json()- assert.equal(- body.choices[0].message.content,- 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=workspace_unavailable'- )- assert.deepEqual(h.getEvidence().attempts, [- {- model: VISUAL_MODEL,- imageSha256: null,- responseKind: 'legacy-lease-download-failed',- failureCode: 'workspace_unavailable',- },- ])-})--test('legacy-lease turns the fixture cannot use are refused with a closed-set reason', async () => {- const cases = [- [- 'shell not offered',- chatBody(VISUAL_MODEL, [legacyUserMessage()], {- tools: [toolDefinition('clerum__gfs_download')],- }),- 'legacy-lease-shell-not-offered',- ],- [- 'download not offered',- chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN)]), {- tools: [toolDefinition('shell_exec')],- }),- 'legacy-lease-download-not-offered',- ],- [- 'first result from another tool',- chatBody(VISUAL_MODEL, legacyMessages([downloadResult(receipt())]), { tools: LEGACY_TOOLS }),- 'legacy-lease-tool-result-unexpected',- ],- [- 'three results',- chatBody(- VISUAL_MODEL,- legacyMessages([- shellResult(LEGACY_TOKEN),- downloadResult(receipt()),- downloadResult(receipt()),- ]),- { tools: LEGACY_TOOLS }- ),- 'legacy-lease-tool-result-count',- ],- [- 'receipt without sha256',- chatBody(- VISUAL_MODEL,- legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt({ sha256: 'x' }))]),- { tools: LEGACY_TOOLS }- ),- 'legacy-lease-download-result-malformed',- ],- [- 'streamed turn',- chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS, stream: true }),- 'legacy-lease-stream-unsupported',- ],- ]- for (const [label, body, reason] of cases) {- const h = harness()- const response = await call(h, body)- assert.ok(response.status >= 400, label)- const evidence = h.getEvidence()- assert.equal(evidence.counters.legacyLeaseFailures, 1, label)- assert.deepEqual(- evidence.attempts,- [{ model: VISUAL_MODEL, imageSha256: null, responseKind: 'rejected', reason }],- label- )- }-})--test('legacy-lease: a journey line without offered tools is an ordinary text-only request', async () => {- const h = harness()- const response = await call(h, chatBody(VISUAL_MODEL, [legacyUserMessage()]))- const body = await response.json()- assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)- // Witness: the request reached the fixture and was answered as text.- assert.equal(h.getEvidence().counters.textOnlyResponses, 1)- assert.equal(h.getEvidence().counters.legacyLeaseShellRequests, 0)-})--test('legacy-lease failure codes are a frozen closed set that includes download_busy', () => {- const codes = providerModule.LEGACY_LEASE_FAILURE_CODES- assert.equal(Object.isFrozen(codes), true)- assert.ok(codes.includes('download_busy'))- assert.ok(codes.includes('unrecognized'))-})diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 5153d8811..86c865e52 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.108",+ "version": "1.0.107","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.108",+ "version": "1.0.107","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex a1819b160..18d0e2843 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.108",+ "version": "1.0.107","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",