Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
test(e2e): add the minikube legacy processing lease restart lane
Alfredo Lopeza06a2adc4139
diff --git a/Makefile b/Makefileindex b74d68007..46dbb3fcc 100644--- a/Makefile+++ b/Makefile@@ -321,6 +321,25 @@ minikube-run-document-upload: ## Run the visible Desktop document-upload journeyIMAGE_CAPABILITIES_LANE=document-upload \bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run+.PHONY: minikube-run-legacy-lease-restart minikube-run-legacy-lease-restart-vacuity+minikube-run-legacy-lease-restart: ## Run the legacy processing-lease restart scenarios (issue #1022) with the Desktop shell+GFS download journey and restore the Host+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ IMAGE_CAPABILITIES_LANE=legacy-lease-restart \+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run++minikube-run-legacy-lease-restart-vacuity: ## Prove the legacy-lease lane fails on the pre-fix Host (base 74e0d81d9): S-crash must stop at the shell with download_busy+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ IMAGE_CAPABILITIES_LANE=legacy-lease-restart-vacuity \+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run++.PHONY: minikube-build-legacy-lease-vacuity-fixture+minikube-build-legacy-lease-vacuity-fixture: ## Build the pre-fix Host (base 74e0d81d9) and its provider-fixture layer for the legacy-lease vacuity lane under the owned profile lease+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-legacy-lease-vacuity-image.sh+minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.tsindex d70615959..23dec419f 100644--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts@@ -57,6 +57,14 @@ export const FIXTURE_RESPONSE_KIND = {// delivered.documentReadRequested: 'document-read-requested',documentAnswer: 'document-answer',+ // Issue #1022: the legacy processing-lease restart journey. shell_exec first,+ // then clerum__gfs_download, then an answer built from the download receipt;+ // a failed step is answered with the store code it reported.+ legacyLeaseShellRequested: 'legacy-lease-shell-requested',+ legacyLeaseDownloadRequested: 'legacy-lease-download-requested',+ legacyLeaseAnswer: 'legacy-lease-answer',+ legacyLeaseShellFailed: 'legacy-lease-shell-failed',+ legacyLeaseDownloadFailed: 'legacy-lease-download-failed',} as const/**@@ -85,6 +93,33 @@ export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze(['document-tool-output-unreadable','document-page-range-malformed','document-stream-unsupported',+ 'legacy-lease-shell-not-offered',+ 'legacy-lease-download-not-offered',+ 'legacy-lease-tool-result-unexpected',+ 'legacy-lease-tool-result-count',+ 'legacy-lease-download-result-malformed',+ 'legacy-lease-stream-unsupported',+])++/**+ * Store failure codes a legacy-lease failure row may carry (the+ * `GfsDownloadStoreError` codes plus `unrecognized`). Declared independently of+ * the fixture; `test/imageCapabilityEvidence.test.ts` checks the lists agree.+ */+export const FIXTURE_LEGACY_LEASE_FAILURE_CODES: readonly string[] = Object.freeze([+ 'caller_mismatch',+ 'caller_quota_exceeded',+ 'corrupt_store_ledger',+ 'download_expired',+ 'download_missing',+ 'download_busy',+ 'host_quota_exceeded',+ 'publication_cancelled',+ 'storage_write_failed',+ 'unsupported_store_schema',+ 'workspace_unavailable',+ 'writer_locked',+ 'unrecognized',])/** The fixture's text-only answer. It names no image and no color. */@@ -312,6 +347,12 @@ export interface FixtureProviderAttempt {truncated?: boolean/** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */reason?: string+ /** One of {@link FIXTURE_LEGACY_LEASE_FAILURE_CODES}. Legacy-lease failure rows only. */+ failureCode?: string+ /** sha256 the download receipt reported. Legacy-lease answer row only. */+ downloadSha256?: string+ /** Byte count the download receipt reported. Legacy-lease answer row only. */+ downloadBytes?: number}export interface FixtureEvidenceCounters {@@ -326,6 +367,10 @@ export interface FixtureEvidenceCounters {documentReadRequests: numberdocumentAnswers: numberdocumentFailures: number+ legacyLeaseShellRequests: number+ legacyLeaseDownloadRequests: number+ legacyLeaseAnswers: number+ legacyLeaseFailures: number}export interface ImageCapabilityEvidenceSnapshot {@@ -352,6 +397,10 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = ['documentReadRequests','documentAnswers','documentFailures',+ 'legacyLeaseShellRequests',+ 'legacyLeaseDownloadRequests',+ 'legacyLeaseAnswers',+ 'legacyLeaseFailures',]function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {@@ -434,6 +483,53 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu)}+ // failureCode is present on every legacy-lease failure row and on no other row.+ const legacyFailure =+ responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed ||+ responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadFailed+ if (legacyFailure) {+ const failureCode = value.failureCode+ if (+ typeof failureCode !== 'string' ||+ !FIXTURE_LEGACY_LEASE_FAILURE_CODES.includes(failureCode)+ ) {+ throw new Error(+ `${source}: attempts[${index}].failureCode must be one of ` ++ `${FIXTURE_LEGACY_LEASE_FAILURE_CODES.join(', ')} on a ${responseKind} row`+ )+ }+ row.failureCode = failureCode+ } else if ('failureCode' in value) {+ throw new Error(+ `${source}: attempts[${index}].failureCode is only valid on a legacy-lease failure row`+ )+ }++ // downloadSha256 and downloadBytes are present on every legacy-lease answer row only.+ if (responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer) {+ const downloadSha256 = value.downloadSha256+ const downloadBytes = value.downloadBytes+ if (typeof downloadSha256 !== 'string' || !SHA256_HEX.test(downloadSha256)) {+ throw new Error(+ `${source}: attempts[${index}].downloadSha256 must be a sha256 hex digest on a ` ++ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`+ )+ }+ if (!isByteCount(downloadBytes)) {+ throw new Error(+ `${source}: attempts[${index}].downloadBytes must be a non-negative integer on a ` ++ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`+ )+ }+ row.downloadSha256 = downloadSha256+ row.downloadBytes = downloadBytes+ } else if ('downloadSha256' in value || 'downloadBytes' in value) {+ throw new Error(+ `${source}: attempts[${index}].downloadSha256 and downloadBytes are only valid on a ` ++ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`+ )+ }+// Present only on document rows; an image row must not grow the field.if ('documentSha256' in value) {const documentSha256 = value.documentSha256@@ -623,7 +719,10 @@ export function appendedAttempts(current.reason !== row.reason ||current.byteRange?.offset !== row.byteRange?.offset ||current.byteRange?.length !== row.byteRange?.length ||- current.truncated !== row.truncated+ current.truncated !== row.truncated ||+ current.failureCode !== row.failureCode ||+ current.downloadSha256 !== row.downloadSha256 ||+ current.downloadBytes !== row.downloadBytes) {throw new Error(`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`diff --git a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.tsnew file mode 100644index 000000000..24f3df659--- /dev/null+++ b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts@@ -0,0 +1,892 @@+// desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts+//+// E2E_GUARDIAN_IPC_FLOW: this module drives no browser transition. It owns the+// cluster side of the legacy processing-lease restart lane (issue #1022): the+// workload stop/start scenarios, the seed of one legacy lease on the Host PVC,+// and read-only readers for the three signals the spec asserts after each+// scenario. The Desktop journey itself lives in the spec.+//+// Every kubectl call names the validated branch profile as its context and is+// bounded by a timeout. Arguments travel as a vector, never through a shell.+// Waits poll a live condition with a deadline; none of them is a fixed sleep.+import { execFileSync } from 'node:child_process'+import { randomUUID } from 'node:crypto'++const PROFILE_PATTERN = /^clerum-.+-[a-f0-9]{8}$/+const RUN_ID_PATTERN = /^image-capabilities-[a-f0-9]{12}$/+const SHA256_HEX = /^[a-f0-9]{64}$/++export const LEGACY_LEASE_LANE_MODES = ['fixed', 'vacuity'] as const+export type LegacyLeaseLaneMode = (typeof LEGACY_LEASE_LANE_MODES)[number]++/** The Host whose PVC carries the GFS download store (the fixture lane's Host). */+export const HOST = {+ namespace: 'mcp-host',+ deployment: 'chatllm',+ container: 'mcp-host',+ selector: 'app=chatllm',+} as const+export const HCC = {+ namespace: 'control-plane',+ deployment: 'host-context-controller',+ selector: 'app=host-context-controller',+} as const+export const WRC = {+ namespace: 'control-plane',+ deployment: 'workflow-recipes',+ selector: 'app=workflow-recipes',+} as const+export const GFS_WORKLOADS = [+ {+ namespace: 'gfs',+ deployment: 'gfsc-writer',+ selector: 'app=gfs-controller,clerum.io/gfsc-role=writer',+ },+ {+ namespace: 'gfs',+ deployment: 'gfsc-reader',+ selector: 'app=gfs-controller,clerum.io/gfsc-role=reader',+ },+] as const++/** The store ledger on the Host PVC, mounted at /workspace. */+export const STORE_LEDGER_PATH = '/workspace/.gfs-download-store/ledger-v1.json'+export const LEGACY_DISCARD_COUNTER = 'clerum_gfs_legacy_processing_leases_discarded_total'+const STORE_LOG_COMPONENT = 'GfsDownloadStore'+const DISCARD_MSG_PREFIX = 'GFS download store discarded '+const UNKNOWN_MSG_PREFIX = 'GFS download store could not confirm the discard of '++export const SEED_LABEL_KEY = 'evenfire.ai/legacy-lease-seed'++export interface LegacyLeaseLaneEnv {+ profile: string+ mode: LegacyLeaseLaneMode+ runId: string+}++/** Reads and validates the runner-owned bindings of this lane. */+export function requireLegacyLeaseLaneEnv(+ env: NodeJS.ProcessEnv = process.env+): LegacyLeaseLaneEnv {+ const problems: string[] = []+ const profile = (env.MINIKUBE_PROFILE ?? '').trim()+ if (!PROFILE_PATTERN.test(profile))+ problems.push(`MINIKUBE_PROFILE is not a branch profile ("${profile}")`)+ if ((env.CONTROL_API_REAL_PG_CONTEXT ?? '').trim() !== profile)+ problems.push('CONTROL_API_REAL_PG_CONTEXT must equal MINIKUBE_PROFILE')+ if ((env.E2E_K8S_CONTEXT ?? '').trim() !== profile)+ problems.push('E2E_K8S_CONTEXT must equal MINIKUBE_PROFILE (the GFS fixture helpers read it)')+ const mode = (env.LEGACY_LEASE_LANE_MODE ?? '').trim()+ if (!(LEGACY_LEASE_LANE_MODES as readonly string[]).includes(mode))+ problems.push(`LEGACY_LEASE_LANE_MODE must be fixed or vacuity ("${mode}")`)+ const runId = (env.LEGACY_LEASE_SEED_LABEL_VALUE ?? '').trim()+ if (!RUN_ID_PATTERN.test(runId) || runId !== (env.IMAGE_CAPABILITIES_RUN_ID ?? '').trim())+ problems.push('LEGACY_LEASE_SEED_LABEL_VALUE must equal IMAGE_CAPABILITIES_RUN_ID')+ if (problems.length > 0) {+ throw new Error(+ 'legacy-lease-restart lane is not configured:\n' ++ problems.map(problem => ` - ${problem}`).join('\n') ++ '\nRun it through `make minikube-run-legacy-lease-restart` (or the -vacuity target).'+ )+ }+ return { profile, mode: mode as LegacyLeaseLaneMode, runId }+}++export type Kubectl = (args: string[], options?: { timeoutMs?: number; input?: string }) => string++export function kubectlFor(profile: string): Kubectl {+ if (!PROFILE_PATTERN.test(profile)) throw new Error(`refusing kubectl for "${profile}"`)+ return (args, options = {}) => {+ const timeoutMs = options.timeoutMs ?? 30_000+ try {+ return execFileSync(+ 'kubectl',+ ['--context', profile, `--request-timeout=${Math.ceil(timeoutMs / 1000)}s`, ...args],+ {+ encoding: 'utf8',+ timeout: timeoutMs + 5_000,+ input: options.input,+ maxBuffer: 16 * 1024 * 1024,+ stdio: ['pipe', 'pipe', 'pipe'],+ }+ )+ } catch (error) {+ const stderr =+ typeof (error as { stderr?: unknown }).stderr === 'string'+ ? (error as { stderr: string }).stderr.trim()+ : ''+ throw new Error(`kubectl ${args.join(' ')} failed: ${(error as Error).message}\n${stderr}`)+ }+ }+}++/** Polls `probe` until it returns a value, or fails with the last observation. */+export async function waitFor<T>(+ label: string,+ timeoutMs: number,+ probe: () => { done: true; value: T } | { done: false; observed: string }+): Promise<T> {+ const deadline = Date.now() + timeoutMs+ let observed = 'not yet observed'+ for (;;) {+ const result = probe()+ if (result.done) return result.value+ observed = result.observed+ if (Date.now() >= deadline)+ throw new Error(`${label}: not reached within ${timeoutMs} ms (${observed})`)+ await new Promise(resolve => setTimeout(resolve, 500))+ }+}++export interface PodState {+ name: string+ uid: string+ phase: string+ terminating: boolean+ ready: boolean+ /** A container of this pod has started running at least once. */+ started: boolean+ image: string+}++export function readPods(kubectl: Kubectl, namespace: string, selector: string): PodState[] {+ const list = JSON.parse(+ kubectl(['-n', namespace, 'get', 'pod', '-l', selector, '-o', 'json'])+ ) as {+ items: Array<{+ metadata: { name: string; uid: string; deletionTimestamp?: string }+ spec: { containers: Array<{ image: string }> }+ status: {+ phase?: string+ conditions?: Array<{ type: string; status: string }>+ containerStatuses?: Array<{+ started?: boolean+ state?: { running?: unknown; terminated?: unknown }+ lastState?: { running?: unknown; terminated?: unknown }+ }>+ }+ }>+ }+ return list.items.map(item => ({+ name: item.metadata.name,+ uid: item.metadata.uid,+ phase: item.status.phase ?? 'Unknown',+ terminating: Boolean(item.metadata.deletionTimestamp),+ ready: (item.status.conditions ?? []).some(c => c.type === 'Ready' && c.status === 'True'),+ started: (item.status.containerStatuses ?? []).some(+ c =>+ c.started === true ||+ Boolean(c.state?.running) ||+ Boolean(c.state?.terminated) ||+ Boolean(c.lastState?.running) ||+ Boolean(c.lastState?.terminated)+ ),+ image: item.spec.containers[0]?.image ?? '',+ }))+}++function describePods(pods: PodState[]): string {+ return pods.length === 0+ ? 'no pods'+ : pods+ .map(+ p =>+ `${p.name}(${p.uid.slice(0, 8)} ${p.phase}${p.terminating ? ' terminating' : ''}${p.ready ? ' ready' : ''})`+ )+ .join(', ')+}++export function deploymentReplicas(kubectl: Kubectl, namespace: string, name: string): number {+ const replicas = JSON.parse(kubectl(['-n', namespace, 'get', 'deployment', name, '-o', 'json']))+ .spec.replicas+ if (!Number.isSafeInteger(replicas)) throw new Error(`${namespace}/${name} has no spec.replicas`)+ return replicas as number+}++export function scale(kubectl: Kubectl, namespace: string, name: string, replicas: number): void {+ kubectl(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])+}++export function rolloutStatus(kubectl: Kubectl, namespace: string, name: string): void {+ kubectl(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=180s'], {+ timeoutMs: 190_000,+ })+}++export function rolloutRestart(kubectl: Kubectl, namespace: string, name: string): void {+ kubectl(['-n', namespace, 'rollout', 'restart', `deployment/${name}`])+}++export async function waitNoPods(+ kubectl: Kubectl,+ namespace: string,+ selector: string,+ label: string+): Promise<void> {+ await waitFor(`${label} has no pods`, 120_000, () => {+ const pods = readPods(kubectl, namespace, selector)+ return pods.length === 0+ ? { done: true, value: undefined }+ : { done: false, observed: describePods(pods) }+ })+}++/** Exactly one Ready, non-terminating pod whose UID is not in `previous`. */+export async function waitSingleNewReadyPod(+ kubectl: Kubectl,+ namespace: string,+ selector: string,+ previous: ReadonlySet<string>,+ label: string,+ timeoutMs = 300_000+): Promise<PodState> {+ return waitFor(`${label} has one new Ready pod`, timeoutMs, () => {+ const pods = readPods(kubectl, namespace, selector)+ const [only] = pods+ if (pods.length === 1 && only && only.ready && !only.terminating && !previous.has(only.uid))+ return { done: true, value: only }+ return { done: false, observed: describePods(pods) }+ })+}++export interface PvcIdentity {+ name: string+ uid: string+}++/** The PVC the Host deployment mounts as its workspace, by claim name and UID. */+export function hostPvcIdentity(kubectl: Kubectl): PvcIdentity {+ const deployment = JSON.parse(+ kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])+ ) as {+ spec: {+ template: {+ spec: { volumes?: Array<{ name: string; persistentVolumeClaim?: { claimName: string } }> }+ }+ }+ }+ const claims = (deployment.spec.template.spec.volumes ?? [])+ .filter(volume => volume.name === 'workspace' && volume.persistentVolumeClaim)+ .map(volume => volume.persistentVolumeClaim!.claimName)+ if (claims.length !== 1) throw new Error(`Host deployment mounts ${claims.length} workspace PVCs`)+ const name = claims[0]!+ const uid = kubectl([+ '-n',+ HOST.namespace,+ 'get',+ 'pvc',+ name,+ '-o',+ 'jsonpath={.metadata.uid}',+ ]).trim()+ if (!uid) throw new Error(`PVC ${name} has no UID`)+ return { name, uid }+}++// ---------------------------------------------------------------------------+// Seeding one legacy lease on the stopped Host's PVC.+// ---------------------------------------------------------------------------++export interface LegacyLeaseRecord {+ leaseId: string+ callerIdentity: string+ recordIds: string[]+ acquiredAt: string+ expiresAt: string+}++/**+ * A valid, non-expired lease owned by a foreign caller, with no writer session+ * (written by an earlier boot) and no records: exactly the shape a pre-#1019+ * shell left behind when its Host died mid-command.+ */+export function legacyLease(now = Date.now()): LegacyLeaseRecord {+ const leaseId = randomUUID()+ return {+ leaseId,+ callerIdentity: `legacy-lease-e2e-foreign-${leaseId.slice(0, 8)}`,+ recordIds: [],+ acquiredAt: new Date(now - 60_000).toISOString(),+ expiresAt: new Date(now + 60 * 60_000).toISOString(),+ }+}++/**+ * Runs inside the seed pod as the Host's own uid/gid. Read-modify-write with+ * the store's own publication sequence (temp file 0600, fsync, rename, fsync of+ * the directory), then a re-read that must show the lease and the unchanged+ * record set. It refuses a ledger that already carries leases.+ */+const SEED_SCRIPT = `+const fs = require('node:fs')+const path = require('node:path')+const { randomUUID } = require('node:crypto')+const ledgerPath = ${JSON.stringify(STORE_LEDGER_PATH)}+const lease = JSON.parse(process.env.LEGACY_LEASE)+const before = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))+if (before.schemaVersion !== 1) throw new Error('ledger schemaVersion is not 1')+if (before.processingLeases !== undefined) throw new Error('ledger already carries processingLeases')+const recordIds = Object.keys(before.records || {}).sort()+const next = { ...before, processingLeases: { [lease.leaseId]: lease } }+const temporary = ledgerPath + '.tmp-' + randomUUID()+const fd = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)+try { fs.writeSync(fd, JSON.stringify(next)); fs.fsyncSync(fd) } finally { fs.closeSync(fd) }+fs.renameSync(temporary, ledgerPath)+const dir = fs.openSync(path.dirname(ledgerPath), 'r')+try { fs.fsyncSync(dir) } finally { fs.closeSync(dir) }+const after = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))+const afterIds = Object.keys(after.records || {}).sort()+const seeded = after.processingLeases && after.processingLeases[lease.leaseId]+if (!seeded || JSON.stringify(seeded) !== JSON.stringify(lease)) throw new Error('seeded lease not read back')+if (JSON.stringify(afterIds) !== JSON.stringify(recordIds)) throw new Error('record set changed')+const info = fs.statSync(ledgerPath)+process.stdout.write('LEGACY_LEASE_SEED ' + JSON.stringify({ leaseId: lease.leaseId, records: afterIds.length, mode: (info.mode & 0o777).toString(8), uid: info.uid, gid: info.gid }) + '\\n')+`++export interface SeedSummary {+ leaseId: string+ records: number+ mode: string+ uid: number+ gid: number+}++export function parseSeedSummary(logs: string, leaseId: string): SeedSummary {+ const lines = logs.split('\n').filter(line => line.startsWith('LEGACY_LEASE_SEED '))+ if (lines.length !== 1) throw new Error(`seed pod printed ${lines.length} summaries`)+ const summary = JSON.parse(lines[0]!.slice('LEGACY_LEASE_SEED '.length)) as SeedSummary+ if (summary.leaseId !== leaseId) throw new Error('seed summary names another lease')+ if (summary.mode !== '600' || summary.uid !== 1001 || summary.gid !== 1001)+ throw new Error(`seeded ledger has mode ${summary.mode} owner ${summary.uid}:${summary.gid}`)+ return summary+}++/**+ * Seeds the lease while HCC and the Host are at zero and the Host pod is gone.+ * The pod runs the Host's image as uid/gid 1001 (the Host's own identity) and+ * mounts the same PVC. It is deleted before this returns, on every path.+ */+export async function seedLegacyLease(+ kubectl: Kubectl,+ input: { runId: string; scenario: string; pvc: PvcIdentity; lease: LegacyLeaseRecord }+): Promise<SeedSummary> {+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 0)+ throw new Error('refusing to seed: the Host is not scaled to zero')+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 0)+ throw new Error('refusing to seed: HCC is not scaled to zero')+ const hostPods = readPods(kubectl, HOST.namespace, HOST.selector)+ if (hostPods.length !== 0)+ throw new Error(`refusing to seed: Host pods remain (${describePods(hostPods)})`)+ const image = JSON.parse(+ kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])+ ).spec.template.spec.containers.find((c: { name: string }) => c.name === HOST.container)?.image+ if (typeof image !== 'string' || !image) throw new Error('Host container image not found')+ const name = `legacy-lease-seed-${input.runId.slice(-12)}-${input.scenario.toLowerCase()}`+ const pod = {+ apiVersion: 'v1',+ kind: 'Pod',+ metadata: {+ name,+ namespace: HOST.namespace,+ labels: { [SEED_LABEL_KEY]: input.runId },+ },+ spec: {+ restartPolicy: 'Never',+ automountServiceAccountToken: false,+ securityContext: {+ runAsUser: 1001,+ runAsGroup: 1001,+ fsGroup: 1001,+ runAsNonRoot: true,+ seccompProfile: { type: 'RuntimeDefault' },+ },+ containers: [+ {+ name: 'seed',+ image,+ imagePullPolicy: 'IfNotPresent',+ command: ['node', '-e', SEED_SCRIPT],+ env: [{ name: 'LEGACY_LEASE', value: JSON.stringify(input.lease) }],+ securityContext: {+ allowPrivilegeEscalation: false,+ capabilities: { drop: ['ALL'] },+ },+ resources: {+ limits: { cpu: '200m', memory: '128Mi' },+ requests: { cpu: '50m', memory: '64Mi' },+ },+ volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],+ },+ ],+ volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: input.pvc.name } }],+ },+ }+ kubectl(['-n', HOST.namespace, 'create', '-f', '-'], { input: JSON.stringify(pod) })+ try {+ const phase = await waitFor(`seed pod ${name} finishes`, 120_000, () => {+ const current = kubectl([+ '-n',+ HOST.namespace,+ 'get',+ 'pod',+ name,+ '-o',+ 'jsonpath={.status.phase}',+ ]).trim()+ return current === 'Succeeded' || current === 'Failed'+ ? { done: true, value: current }+ : { done: false, observed: current || 'Pending' }+ })+ const logs = kubectl(['-n', HOST.namespace, 'logs', name, '-c', 'seed'])+ if (phase !== 'Succeeded') throw new Error(`seed pod ${name} ${phase}:\n${logs.slice(-2000)}`)+ return parseSeedSummary(logs, input.lease.leaseId)+ } finally {+ kubectl(+ [+ '-n',+ HOST.namespace,+ 'delete',+ 'pod',+ name,+ '--ignore-not-found',+ '--wait=true',+ '--timeout=90s',+ ],+ {+ timeoutMs: 100_000,+ }+ )+ }+}++// ---------------------------------------------------------------------------+// Scenarios.+// ---------------------------------------------------------------------------++export const SCENARIOS = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'] as const+export type Scenario = (typeof SCENARIOS)[number]++export interface ScenarioResult {+ scenario: Scenario+ pvc: PvcIdentity+ previousHostUid: string+ host: PodState+ seed: SeedSummary+ lease: LegacyLeaseRecord+ /** Workloads the scenario replaced, with their pod UIDs before and after. */+ replaced: Array<{ workload: string; before: string[]; after: string }>+}++function singleReadyPod(+ kubectl: Kubectl,+ namespace: string,+ selector: string,+ label: string+): PodState {+ const pods = readPods(kubectl, namespace, selector).filter(p => !p.terminating)+ const [only] = pods+ if (pods.length !== 1 || !only?.ready)+ throw new Error(`${label} is not one Ready pod (${describePods(pods)})`)+ return only+}++/**+ * Stops the Host. `crash` kills the running pod without grace and scales the+ * deployment to zero at once, so the replacement the ReplicaSet creates in+ * between must never start a container: if one does, the crash window was+ * lost and the scenario fails instead of silently becoming a graceful stop.+ */+async function stopHost(+ kubectl: Kubectl,+ mode: 'crash' | 'graceful',+ running: PodState+): Promise<void> {+ if (mode === 'graceful') {+ scale(kubectl, HOST.namespace, HOST.deployment, 0)+ await waitNoPods(kubectl, HOST.namespace, HOST.selector, 'Host')+ return+ }+ kubectl([+ '-n',+ HOST.namespace,+ 'delete',+ 'pod',+ running.name,+ '--force',+ '--grace-period=0',+ '--wait=false',+ ])+ scale(kubectl, HOST.namespace, HOST.deployment, 0)+ const startedReplacements = new Set<string>()+ await waitFor('Host has no pods after the crash', 120_000, () => {+ const pods = readPods(kubectl, HOST.namespace, HOST.selector)+ for (const pod of pods)+ if (pod.uid !== running.uid && pod.started) startedReplacements.add(pod.name)+ return pods.length === 0+ ? { done: true, value: undefined }+ : { done: false, observed: describePods(pods) }+ })+ if (startedReplacements.size > 0)+ throw new Error(+ `CRASH_WINDOW_LOST: a replacement Host pod started before the scale to zero (${[...startedReplacements].join(', ')})`+ )+}++async function restartWorkload(+ kubectl: Kubectl,+ workload: { namespace: string; deployment: string; selector: string }+): Promise<{ workload: string; before: string[]; after: string }> {+ const before = readPods(kubectl, workload.namespace, workload.selector).map(p => p.uid)+ if (before.length === 0) throw new Error(`${workload.deployment} has no pod to restart`)+ rolloutRestart(kubectl, workload.namespace, workload.deployment)+ rolloutStatus(kubectl, workload.namespace, workload.deployment)+ const pod = await waitSingleNewReadyPod(+ kubectl,+ workload.namespace,+ workload.selector,+ new Set(before),+ workload.deployment+ )+ return { workload: workload.deployment, before, after: pod.uid }+}++/**+ * Runs one scenario end to end: HCC to zero, Host stopped, lease seeded, the+ * scenario's own restarts, then HCC and the Host back. Returns the new Host pod.+ * Every step leaves HCC and the Host at their pre-scenario replicas on success;+ * on failure, `ensureWorkloadsUp` (and the runner's restoration) bring them back.+ */+export async function runScenario(+ kubectl: Kubectl,+ input: { scenario: Scenario; runId: string }+): Promise<ScenarioResult> {+ const { scenario } = input+ const replaced: ScenarioResult['replaced'] = []+ const pvcBefore = hostPvcIdentity(kubectl)+ const running = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')+ const hccBefore = singleReadyPod(kubectl, HCC.namespace, HCC.selector, 'HCC')+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)+ throw new Error('Host deployment is not at one replica before the scenario')+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)+ throw new Error('HCC deployment is not at one replica before the scenario')++ // HCC first: it owns the Host deployment and would scale it back up.+ scale(kubectl, HCC.namespace, HCC.deployment, 0)+ await waitNoPods(kubectl, HCC.namespace, HCC.selector, 'HCC')+ await stopHost(kubectl, scenario === 'S-crash' ? 'crash' : 'graceful', running)++ const lease = legacyLease()+ const seed = await seedLegacyLease(kubectl, {+ runId: input.runId,+ scenario,+ pvc: pvcBefore,+ lease,+ })++ const startHostThenHcc = async () => {+ scale(kubectl, HOST.namespace, HOST.deployment, 1)+ scale(kubectl, HCC.namespace, HCC.deployment, 1)+ }+ switch (scenario) {+ case 'S-crash':+ await startHostThenHcc()+ break+ case 'S-graceful':+ // A new template while stopped: the next pod is a fresh rollout, the way+ // a graceful `rollout restart` replaces the Host.+ rolloutRestart(kubectl, HOST.namespace, HOST.deployment)+ await startHostThenHcc()+ break+ case 'S-hcc':+ await startHostThenHcc()+ break+ case 'S-gfs':+ for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))+ await startHostThenHcc()+ break+ case 'S-update':+ // Upgrade order: GFS, WRC, HCC, then the Host.+ for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))+ replaced.push(await restartWorkload(kubectl, WRC))+ rolloutRestart(kubectl, HOST.namespace, HOST.deployment)+ scale(kubectl, HCC.namespace, HCC.deployment, 1)+ rolloutStatus(kubectl, HCC.namespace, HCC.deployment)+ scale(kubectl, HOST.namespace, HOST.deployment, 1)+ break+ }++ const hccPod = await waitSingleNewReadyPod(+ kubectl,+ HCC.namespace,+ HCC.selector,+ new Set([hccBefore.uid]),+ 'HCC'+ )+ replaced.push({ workload: HCC.deployment, before: [hccBefore.uid], after: hccPod.uid })+ const host = await waitSingleNewReadyPod(+ kubectl,+ HOST.namespace,+ HOST.selector,+ new Set([running.uid]),+ 'Host'+ )+ rolloutStatus(kubectl, HOST.namespace, HOST.deployment)+ // HCC's first reconcile must not roll the Host again: the pod that booted on+ // the seeded ledger is the one whose signals are read.+ const settled = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')+ if (settled.uid !== host.uid)+ throw new Error(`HCC replaced the Host pod after it booted (${host.uid} -> ${settled.uid})`)+ const pvcAfter = hostPvcIdentity(kubectl)+ if (pvcAfter.name !== pvcBefore.name || pvcAfter.uid !== pvcBefore.uid)+ throw new Error(+ `Host PVC changed across ${scenario}: ${JSON.stringify(pvcBefore)} -> ${JSON.stringify(pvcAfter)}`+ )+ return { scenario, pvc: pvcAfter, previousHostUid: running.uid, host, seed, lease, replaced }+}++/** Brings HCC and the Host back to one replica after a failed scenario. */+export function ensureWorkloadsUp(kubectl: Kubectl): void {+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)+ scale(kubectl, HOST.namespace, HOST.deployment, 1)+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)+ scale(kubectl, HCC.namespace, HCC.deployment, 1)+ rolloutStatus(kubectl, HCC.namespace, HCC.deployment)+ rolloutStatus(kubectl, HOST.namespace, HOST.deployment)+}++// ---------------------------------------------------------------------------+// Signal readers (read-only) and pure verdicts.+// ---------------------------------------------------------------------------++export function readStoreLedger(kubectl: Kubectl, podName: string): Record<string, unknown> {+ const raw = kubectl([+ '-n',+ HOST.namespace,+ 'exec',+ podName,+ '-c',+ HOST.container,+ '--',+ 'cat',+ STORE_LEDGER_PATH,+ ])+ const parsed = JSON.parse(raw) as unknown+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))+ throw new Error('store ledger is not a JSON object')+ return parsed as Record<string, unknown>+}++export function readHostLogLines(kubectl: Kubectl, podName: string): string[] {+ return kubectl(['-n', HOST.namespace, 'logs', podName, '-c', HOST.container], {+ timeoutMs: 60_000,+ })+ .split('\n')+ .filter(line => line.trim().length > 0)+}++export function readHostMetrics(kubectl: Kubectl, podName: string): string {+ return kubectl([+ '-n',+ HOST.namespace,+ 'exec',+ podName,+ '-c',+ HOST.container,+ '--',+ 'node',+ '-e',+ "fetch('http://127.0.0.1:8080/metrics').then(r=>{if(!r.ok)throw new Error('metrics '+r.status);return r.text()}).then(t=>process.stdout.write(t))",+ ])+}++/**+ * The value of an unlabelled metric, or null when the exposition has no such+ * sample. A metric with labels, or more than one sample, is refused.+ */+export function parseMetricValue(text: string, name: string): number | null {+ const samples = text+ .split('\n')+ .filter(+ line => !line.startsWith('#') && (line.startsWith(`${name} `) || line.startsWith(`${name}{`))+ )+ if (samples.length === 0) return null+ if (samples.length !== 1) throw new Error(`${name} has ${samples.length} samples`)+ const match = new RegExp(`^${name}(?:\\{\\})? ([0-9.eE+-]+)$`).exec(samples[0]!.trim())+ if (!match) throw new Error(`${name} sample is not an unlabelled number: ${samples[0]}`)+ const value = Number(match[1])+ if (!Number.isFinite(value)) throw new Error(`${name} sample is not finite`)+ return value+}++interface StoreLogLine {+ component?: unknown+ msg?: unknown+ discarded?: unknown+ legacyProcessingLeases?: unknown+ outcome?: unknown+}++function storeLogLines(logLines: string[]): StoreLogLine[] {+ const parsed: StoreLogLine[] = []+ for (const line of logLines) {+ if (!line.startsWith('{')) continue+ let value: unknown+ try {+ value = JSON.parse(line)+ } catch {+ continue+ }+ if (+ typeof value === 'object' &&+ value !== null &&+ (value as StoreLogLine).component === STORE_LOG_COMPONENT+ )+ parsed.push(value as StoreLogLine)+ }+ return parsed+}++export type RecoveryOutcome = 'discarded' | 'unknown-outcome'++/**+ * Signal 2 on the fixed Host. The ledger must have no `processingLeases`, and+ * exactly one of the two outcomes must be visible: one confirmed discard of+ * one lease with the counter at 1, or one unknown-outcome warning with the+ * counter at 0. The counter is read from a pod that just booted, so its value+ * is the delta. A missing counter is a failure: it is the liveness witness+ * that the metric registry of the fixed build is the one that answered.+ */+export function legacyLeaseRecoveryVerdict(input: {+ ledger: Record<string, unknown>+ logLines: string[]+ counter: number | null+}): RecoveryOutcome {+ if (Object.hasOwn(input.ledger, 'processingLeases'))+ throw new Error('the ledger still carries processingLeases after the restart')+ if (input.counter === null)+ throw new Error(`${LEGACY_DISCARD_COUNTER} is not exposed by the Host`)+ const lines = storeLogLines(input.logLines)+ const discards = lines.filter(+ line =>+ typeof line.msg === 'string' &&+ line.msg.startsWith(DISCARD_MSG_PREFIX) &&+ typeof line.discarded === 'number'+ )+ const unknown = lines.filter(+ line =>+ line.outcome === 'unknown' &&+ typeof line.msg === 'string' &&+ line.msg.startsWith(UNKNOWN_MSG_PREFIX) &&+ typeof line.legacyProcessingLeases === 'number'+ )+ if (+ discards.length === 1 &&+ unknown.length === 0 &&+ discards[0]!.discarded === 1 &&+ input.counter === 1+ )+ return 'discarded'+ if (+ unknown.length === 1 &&+ discards.length === 0 &&+ unknown[0]!.legacyProcessingLeases === 1 &&+ input.counter === 0+ )+ return 'unknown-outcome'+ throw new Error(+ `no recovery outcome: ${discards.length} discard line(s) ` ++ `(${discards.map(line => String(line.discarded)).join(',') || '-'}), ` ++ `${unknown.length} unknown-outcome line(s), counter ${input.counter}`+ )+}++/**+ * The vacuity witness on the pre-fix Host: the seeded lease is still in the+ * ledger, nothing was discarded, and the counter does not exist. Together with+ * the journey failing at the shell with download_busy, this shows the lane can+ * tell the fixed build from the broken one.+ */+export function vacuityStoreVerdict(input: {+ ledger: Record<string, unknown>+ logLines: string[]+ counter: number | null+ leaseId: string+}): 'lease-retained' {+ const leases = input.ledger.processingLeases as Record<string, unknown> | undefined+ if (!leases || typeof leases !== 'object' || !Object.hasOwn(leases, input.leaseId))+ throw new Error('the pre-fix Host no longer carries the seeded lease')+ const discards = storeLogLines(input.logLines).filter(+ line => typeof line.msg === 'string' && line.msg.startsWith(DISCARD_MSG_PREFIX)+ )+ if (discards.length !== 0) throw new Error('the pre-fix Host logged a legacy-lease discard')+ if (input.counter !== null) throw new Error(`the pre-fix Host exposes ${LEGACY_DISCARD_COUNTER}`)+ return 'lease-retained'+}++export interface StoreRecord {+ id: string+ hostPath: string+ sizeBytes: number+ sha256?: string+ state: string+}++export function storeRecords(ledger: Record<string, unknown>): StoreRecord[] {+ const records = ledger.records+ if (typeof records !== 'object' || records === null || Array.isArray(records))+ throw new Error('store ledger has no records object')+ return Object.values(records as Record<string, StoreRecord>)+}++/** The one completed record this journey added, matching the GFS source bytes. */+export function downloadedRecord(+ before: Record<string, unknown>,+ after: Record<string, unknown>,+ source: { sha256: string; bytes: number }+): StoreRecord {+ if (!SHA256_HEX.test(source.sha256)) throw new Error('source sha256 is not a digest')+ const known = new Set(storeRecords(before).map(record => record.id))+ const added = storeRecords(after).filter(record => !known.has(record.id))+ const matching = added.filter(+ record =>+ record.state === 'completed' &&+ record.sha256 === source.sha256 &&+ record.sizeBytes === source.bytes+ )+ if (added.length !== 1 || matching.length !== 1)+ throw new Error(+ `expected one new completed record for ${source.sha256}/${source.bytes} bytes; ` ++ `added ${JSON.stringify(added.map(r => ({ state: r.state, sha256: r.sha256, sizeBytes: r.sizeBytes })))}`+ )+ return matching[0]!+}++/** sha256 and size of a file on the Host PVC, computed in the Host container. */+export function hostFileDigest(+ kubectl: Kubectl,+ podName: string,+ hostPath: string+): { sha256: string; bytes: number } {+ if (path_is_unsafe(hostPath)) throw new Error(`refusing to read ${hostPath}`)+ const out = kubectl([+ '-n',+ HOST.namespace,+ 'exec',+ podName,+ '-c',+ HOST.container,+ '--',+ 'node',+ '-e',+ "const b=require('node:fs').readFileSync(process.argv[1]);process.stdout.write(require('node:crypto').createHash('sha256').update(b).digest('hex')+' '+b.length)",+ `/workspace/${hostPath}`,+ ]).trim()+ const [sha256, bytes] = out.split(' ')+ if (!sha256 || !SHA256_HEX.test(sha256) || !/^\d+$/.test(bytes ?? ''))+ throw new Error(`unexpected digest output: ${out}`)+ return { sha256, bytes: Number(bytes) }+}++function path_is_unsafe(hostPath: string): boolean {+ return hostPath.startsWith('/') || hostPath.split('/').some(part => part === '..' || part === '')+}diff --git a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.tsnew file mode 100644index 000000000..41fb82cd4--- /dev/null+++ b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts@@ -0,0 +1,59 @@+import { defineConfig } from '@playwright/test'+import path from 'node:path'+import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'+import { requireLegacyLeaseLaneEnv } from './helpers/legacyLeaseRestart'++// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns+// visible IPC transitions and reads the external provider ledger as its wire oracle.++/*+ * Desktop App legacy processing-lease restart lane (issue #1022).+ *+ * It reuses the image-capabilities fixture stack (derived Host image, provider+ * fixture, seeded catalog and login) and adds the cluster side: before each+ * scenario the Host and HCC are stopped, one legacy foreign processing lease is+ * seeded on the Host PVC, the scenario's restarts run, and the Desktop journey+ * (approved `shell_exec`, then `clerum__gfs_download`) must succeed on the Host+ * that booted on that ledger.+ *+ * - It loads NO env file. Every binding is exported by the runner+ * (`make minikube-run-legacy-lease-restart` or the -vacuity target) and a+ * missing or inconsistent one fails here, at config load.+ * - The runner selects the scenarios with --grep: the fixed lane runs the five+ * `legacy-lease-restart fixture:` tests, the vacuity lane runs the one+ * `legacy-lease-restart vacuity:` test against the pre-fix Host.+ */++const repoRoot = path.resolve(__dirname, '../../..')++if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {+ throw new Error(+ `NODE_ENV must be "test" (or unset) for the legacy-lease-restart fixture lane; ` ++ `received "${process.env.NODE_ENV}".`+ )+}+process.env.NODE_ENV = 'test'++// Fail at config load, with every missing binding listed at once.+requireImageCapabilitiesFixtureEnv(process.env)+requireLegacyLeaseLaneEnv(process.env)++const recorderRoot = process.env.QA_RECORDER_ROOT+ ? path.resolve(process.env.QA_RECORDER_ROOT)+ : path.join(repoRoot, '.local-notes', 'qa-recorder')++export default defineConfig({+ testDir: '.',+ testMatch: /qa-recorder-legacy-lease-restart\.spec\.ts/,+ // Per scenario: HCC and Host stop, seed pod, the scenario's own rollouts+ // (S-update replaces GFS, WRC, HCC and the Host), one Electron launch and a+ // two-tool journey with an approval.+ timeout: 660_000,+ expect: { timeout: 30_000 },+ retries: 0,+ workers: 1,+ fullyParallel: false,+ reporter: [['list']],+ outputDir: path.join(recorderRoot, 'runs', 'desktop-app-legacy-lease-restart'),+ preserveOutput: 'always',+})diff --git a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.tsnew file mode 100644index 000000000..e0e1a5dcb--- /dev/null+++ b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts@@ -0,0 +1,440 @@+import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'+import { createHash, randomBytes } from 'node:crypto'+import {+ GFS_E2E_DRIVE,+ type GfsFileFixture,+ cleanupGfsFixture,+ getE2EUserId,+ gfsWriterPod,+ kubectlOut,+ ridOf,+ seedGfsFileFixture,+ seedGfsGrant,+ uniqueGfsFixtureName,+} from '../../../tests/e2e/gfsUiFixtures'+import { discoverManagedGfsAgent } from './helpers/gfsAgentDiscovery'+import {+ FIXTURE_RESPONSE_KIND,+ IMAGE_CAPABILITY_FIXTURE_MODELS,+ type ImageCapabilityEvidenceSnapshot,+ appendedAttempts,+ readImageCapabilityEvidence,+ requireImageCapabilitiesFixtureEnv,+} from './helpers/imageCapabilityEvidence'+import {+ HOST,+ type Kubectl,+ LEGACY_DISCARD_COUNTER,+ type LegacyLeaseLaneMode,+ SCENARIOS,+ type Scenario,+ type ScenarioResult,+ downloadedRecord,+ ensureWorkloadsUp,+ hostFileDigest,+ kubectlFor,+ legacyLeaseRecoveryVerdict,+ parseMetricValue,+ readHostLogLines,+ readHostMetrics,+ readPods,+ readStoreLedger,+ requireLegacyLeaseLaneEnv,+ runScenario,+ vacuityStoreVerdict,+} from './helpers/legacyLeaseRestart'+import {+ EXTERNAL_REST_API_BASE_URL,+ RPC_PROXY_BASE_URL,+ assertAllowedTarget,+ desktopCredentials,+ finalizeRecording,+ launchDesktopApp,+ login,+ screenshotAndLog,+} from './qa-recorder-helpers'++/*+ * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the agent switch, the model popover and+ * the approval card all travel over Electron IPC, so the renderer has no HTTP+ * request to await for any transition. The oracles are the visible chat (the+ * fixture's answer line and the approval card), the provider-boundary ledger of+ * the derived in-cluster fixture, and read-only reads of the Host's own store+ * ledger, logs, metrics and workspace file.+ *+ * Issue #1022: a processing lease written by a Host build before #1019 has no+ * writer session. A Host that boots on such a ledger must discard it at+ * initialize (log `discarded 1`, counter 1) or report that it could not confirm+ * the discard; either way the ledger it serves carries no `processingLeases`,+ * and a shell command plus a GFS download work. The pre-fix build treats the+ * lease as an inherited executor, so its shell refuses with `download_busy`.+ *+ * Each scenario stops HCC and the Host, seeds one foreign lease on the Host PVC+ * from a pod running as the Host's uid/gid, performs its own restarts, and+ * brings HCC and the Host back. Then the Desktop journey runs: the user sends a+ * journey line, approves `shell_exec` by clicking Approve, and the fixture asks+ * for `clerum__gfs_download` of a seeded GFS file. The answer on screen carries+ * the sha256 and byte count of the Host's receipt, which must equal the GFS+ * source, as must the file the Host wrote to its workspace.+ *+ * Setup shortcuts (named, outside the behaviour under test): the GFS file and+ * its grants are seeded with SQL and the writer blob with kubectl cp, and the+ * legacy lease is written by the seed pod. Only the external provider peer is+ * simulated; nothing is mocked inside the cluster or the app.+ */++const SCENARIO_TITLES: Record<Scenario, string> = {+ 'S-crash':+ 'S-crash: a Host killed without grace boots on a legacy lease and serves shell then download',+ 'S-graceful':+ 'S-graceful: a Host stopped and rolled out boots on a legacy lease and serves shell then download',+ 'S-hcc':+ 'S-hcc: an HCC restart with the Host boots on a legacy lease and serves shell then download',+ 'S-gfs':+ 'S-gfs: each GFS pod restarts while the Host boots on a legacy lease, then shell and download work',+ 'S-update':+ 'S-update: GFS, WRC, HCC then Host restart in upgrade order on a legacy lease, then shell and download work',+}++const JOURNEY_RESPONSE_TIMEOUT_MS = 240_000+const ANY_TERMINAL_ANSWER = /LEGACY-LEASE-FIXTURE-(?:OK|SHELL-FAILED|DOWNLOAD-FAILED)/++interface LaneContext {+ mode: LegacyLeaseLaneMode+ runId: string+ kubectl: Kubectl+ env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>+ file: GfsFileFixture+ source: { sha256: string; bytes: number }+}++let lane: LaneContext | undefined++function requireLane(expected: LegacyLeaseLaneMode): LaneContext {+ if (!lane) throw new Error('legacy-lease-restart lane context was not prepared')+ // A mode mismatch is a runner defect: it fails, it never skips.+ if (lane.mode !== expected)+ throw new Error(+ `this test belongs to the ${expected} lane, but LEGACY_LEASE_LANE_MODE is ${lane.mode}`+ )+ return lane+}++test.describe.configure({ mode: 'serial' })++test.beforeAll(async () => {+ const env = requireImageCapabilitiesFixtureEnv()+ const laneEnv = requireLegacyLeaseLaneEnv()+ await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)+ await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)+ const kubectl = kubectlFor(laneEnv.profile)++ // The GFS subject of the Host this lane drives: exactly the chatllm Host.+ const agent = discoverManagedGfsAgent()+ if (agent.namespace !== HOST.namespace || agent.name !== HOST.deployment)+ throw new Error(`GFS agent discovery returned ${agent.subjectId}, not the lane's Host`)+ const ownerUserId = getE2EUserId(desktopCredentials().email)++ const file = seedGfsFileFixture(uniqueGfsFixtureName('e2e-gfs-legacy-lease'))+ try {+ seedGfsGrant({+ resourceId: file.fileResourceId,+ subjectType: 'user',+ subjectId: ownerUserId,+ permissions: ['read'],+ inherit: true,+ grantedBy: 'e2e:legacy-lease-restart',+ })+ seedGfsGrant({+ resourceId: file.fileResourceId,+ subjectType: 'host',+ subjectId: agent.subjectId,+ permissions: ['read'],+ inherit: true,+ grantedBy: 'e2e:legacy-lease-restart',+ })+ // The source the journey must reproduce: the bytes the fixture seeded, and+ // the same digest read back from the writer's blob.+ const content = Buffer.from(`E2E GFS file fixture: ${file.name}\n`, 'utf8')+ const sha256 = createHash('sha256').update(content).digest('hex')+ const blob = kubectlOut([+ '-n',+ 'gfs',+ 'exec',+ gfsWriterPod(),+ '--',+ 'sha256sum',+ `/data/gfs/${ridOf(file.fileResourceId)}`,+ ]).trim()+ if (blob.split(/\s+/)[0] !== sha256)+ throw new Error(`writer blob digest ${blob} differs from the seeded ${sha256}`)+ lane = {+ mode: laneEnv.mode,+ runId: laneEnv.runId,+ kubectl,+ env,+ file,+ source: { sha256, bytes: content.length },+ }+ } catch (error) {+ cleanupGfsFixture(file.name)+ throw error+ }+})++test.afterAll(async () => {+ if (lane) cleanupGfsFixture(lane.file.name)+})++/** Reach the authenticated shell through the real sign-in form. */+async function signIn(page: Page) {+ await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })+ const emailInput = page.locator('#email-input')+ const settingsMenu = page.getByTestId('nav-settings-menu')+ await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })+ if (await settingsMenu.isVisible()) {+ if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') await settingsMenu.click()+ await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')+ await page.getByTestId('logout-btn').click()+ }+ await expect(emailInput).toBeVisible({ timeout: 20_000 })+ await expect(page.getByTestId('nav-chat')).toHaveCount(0)+ await login(page, desktopCredentials())+ await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })+}++async function openFixtureChat(page: Page, hostRef: string) {+ const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')+ await page.getByTestId('nav-chat').click()+ const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })+ await expect(switchAgent).toBeVisible({ timeout: 30_000 })+ await switchAgent.click()+ const item = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })+ await expect(item).toHaveCount(1)+ await item.click()+ await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))++ await page.getByTestId('nav-new-chat').click()+ const composer = page.getByRole('textbox', { name: 'Agent message composer' })+ await expect(composer).toBeVisible({ timeout: 30_000 })+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })++ const model = IMAGE_CAPABILITY_FIXTURE_MODELS.supported+ const chip = page.getByTestId('selected-chat-model')+ await expect(chip).toBeVisible({ timeout: 30_000 })+ await chip.click()+ await expect(page.getByRole('menu', { name: 'Select model' })).toBeVisible({ timeout: 20_000 })+ const row = page.getByTestId(`model-option-${model}`)+ await expect(row).toHaveCount(1)+ await row.click()+ await expect(chip).toHaveAttribute('data-model-id', model)+ return composer+}++interface JourneyOutcome {+ answer: string+ approvals: string[]+ before: ImageCapabilityEvidenceSnapshot+ after: ImageCapabilityEvidenceSnapshot+}++/**+ * Sends the journey line and approves every tool approval the Host raises by+ * clicking Approve, until the fixture's terminal answer is on screen. Returns+ * the answer text and the input preview of each approval clicked.+ */+async function runJourney(page: Page, ctx: LaneContext, marker: string): Promise<JourneyOutcome> {+ const composer = await openFixtureChat(page, ctx.env.hostRef)+ const line = `LEGACY_LEASE_JOURNEY marker=${marker} drive=${GFS_E2E_DRIVE} resourceId=${ridOf(ctx.file.fileResourceId)}`+ const before = readImageCapabilityEvidence(ctx.env)+ expect(before.runId).toBe(ctx.env.runId)+ await composer.fill(line)+ const send = page.getByTestId('send-button')+ await expect(send).toBeEnabled({ timeout: 20_000 })+ await send.click()++ const approve = page.getByTestId('approval-approve-btn')+ const preview = page.getByTestId('approval-input-preview')+ const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')+ const terminal = response.filter({ hasText: ANY_TERMINAL_ANSWER })+ const approvals: string[] = []+ // The journey has at most two approvable tools (shell, then download).+ for (let round = 0; round < 3; round += 1) {+ await expect(approve.or(terminal)).toBeVisible({ timeout: JOURNEY_RESPONSE_TIMEOUT_MS })+ if ((await terminal.count()) > 0) break+ if (round === 2) throw new Error('more approvals than the journey has tools')+ await expect(preview).toBeVisible({ timeout: 20_000 })+ const shown = await preview.innerText()+ approvals.push(shown)+ await approve.click()+ // The clicked card leaves, or is replaced by the next tool's approval.+ await expect+ .poll(+ async () =>+ (await approve.count()) === 0 ||+ ((await preview.count()) > 0 && (await preview.innerText()) !== shown),+ { timeout: 60_000 }+ )+ .toBe(true)+ }+ await expect(terminal).toHaveCount(1)+ const answer = (await terminal.innerText()).trim()+ const after = readImageCapabilityEvidence(ctx.env)+ return { answer, approvals, before, after }+}++/** The shell approval the user clicked is the fixture's command for this marker. */+function expectShellApproved(approvals: string[], marker: string) {+ expect(approvals.length).toBeGreaterThanOrEqual(1)+ expect(approvals[0]).toContain('printf')+ expect(approvals[0]).toContain(marker)+}++async function withDesktop(+ testInfo: TestInfo,+ label: string,+ body: (page: Page) => Promise<void>+): Promise<void> {+ let app: ElectronApplication | undefined+ let recordedPage: Page | undefined+ try {+ const launched = await launchDesktopApp(+ testInfo,+ `issue1022-${label}-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`+ )+ app = launched.app+ recordedPage = launched.page+ await signIn(launched.page)+ await body(launched.page)+ await screenshotAndLog(launched.page, testInfo, `desktop-legacy-lease-${label}`)+ } catch (error) {+ if (recordedPage && !recordedPage.isClosed()) {+ await recordedPage+ .screenshot({ path: testInfo.outputPath('failure.png') })+ .catch(() => undefined)+ }+ throw error+ } finally {+ await finalizeRecording(app, recordedPage)+ }+}++/** The Host pod that booted on the seeded ledger is still the only Ready one. */+function expectHostUnchanged(ctx: LaneContext, result: ScenarioResult) {+ const pods = readPods(ctx.kubectl, HOST.namespace, HOST.selector).filter(p => !p.terminating)+ expect(pods.map(p => p.uid)).toEqual([result.host.uid])+ expect(pods[0]?.ready).toBe(true)+ expect(result.host.uid).not.toBe(result.previousHostUid)+}++function attachScenario(testInfo: TestInfo, data: unknown) {+ return testInfo.attach('scenario.json', {+ body: JSON.stringify(data, null, 2),+ contentType: 'application/json',+ })+}++for (const scenario of SCENARIOS) {+ test(`legacy-lease-restart fixture: ${SCENARIO_TITLES[scenario]}`, async ({}, testInfo) => {+ const ctx = requireLane('fixed')+ try {+ const result = await runScenario(ctx.kubectl, { scenario, runId: ctx.runId })+ expect(result.seed.leaseId).toBe(result.lease.leaseId)++ // Signal 2: the store the new Host serves has no legacy lease, and it+ // reports one of the two recovery outcomes.+ const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)+ const metrics = readHostMetrics(ctx.kubectl, result.host.name)+ const outcome = legacyLeaseRecoveryVerdict({+ ledger: ledgerAtBoot,+ logLines: readHostLogLines(ctx.kubectl, result.host.name),+ counter: parseMetricValue(metrics, LEGACY_DISCARD_COUNTER),+ })+ await attachScenario(testInfo, { ...result, outcome })++ // Signal 3: the Desktop journey on that Host.+ const marker = `legacy-lease-${randomBytes(8).toString('hex')}`+ await withDesktop(testInfo, scenario.toLowerCase(), async page => {+ const journey = await runJourney(page, ctx, marker)+ expectShellApproved(journey.approvals, marker)+ expect(journey.answer).toContain(+ `LEGACY-LEASE-FIXTURE-OK marker=${marker} sha256=${ctx.source.sha256} bytes=${ctx.source.bytes}`+ )+ const rows = appendedAttempts(journey.before, journey.after)+ const kinds = rows.map(row => row.responseKind)+ expect(+ kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)+ ).toHaveLength(1)+ expect(+ kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)+ ).toHaveLength(1)+ const answers = rows.filter(+ row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer+ )+ expect(answers).toHaveLength(1)+ expect(answers[0]?.downloadSha256).toBe(ctx.source.sha256)+ expect(answers[0]?.downloadBytes).toBe(ctx.source.bytes)+ const delta = (key: keyof ImageCapabilityEvidenceSnapshot['counters']) =>+ journey.after.counters[key] - journey.before.counters[key]+ expect(delta('legacyLeaseShellRequests')).toBe(1)+ expect(delta('legacyLeaseDownloadRequests')).toBe(1)+ expect(delta('legacyLeaseAnswers')).toBe(1)+ expect(delta('legacyLeaseFailures')).toBe(0)+ })++ // The store recorded exactly this download, and the workspace file holds+ // the GFS source bytes.+ const ledgerAfter = readStoreLedger(ctx.kubectl, result.host.name)+ const record = downloadedRecord(ledgerAtBoot, ledgerAfter, ctx.source)+ expect(hostFileDigest(ctx.kubectl, result.host.name, record.hostPath)).toEqual(ctx.source)+ expect(Object.hasOwn(ledgerAfter, 'processingLeases')).toBe(false)+ expectHostUnchanged(ctx, result)+ } finally {+ ensureWorkloadsUp(ctx.kubectl)+ }+ })+}++test('legacy-lease-restart vacuity: S-crash on base 74e0d81d9 fails at the shell with download_busy', async ({}, testInfo) => {+ const ctx = requireLane('vacuity')+ try {+ const result = await runScenario(ctx.kubectl, { scenario: 'S-crash', runId: ctx.runId })+ const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)+ const verdict = vacuityStoreVerdict({+ ledger: ledgerAtBoot,+ logLines: readHostLogLines(ctx.kubectl, result.host.name),+ counter: parseMetricValue(+ readHostMetrics(ctx.kubectl, result.host.name),+ LEGACY_DISCARD_COUNTER+ ),+ leaseId: result.lease.leaseId,+ })+ await attachScenario(testInfo, { ...result, verdict })++ const marker = `legacy-lease-${randomBytes(8).toString('hex')}`+ await withDesktop(testInfo, 'vacuity', async page => {+ const journey = await runJourney(page, ctx, marker)+ expectShellApproved(journey.approvals, marker)+ expect(journey.answer).toContain('LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy')+ const rows = appendedAttempts(journey.before, journey.after)+ const shellFailed = rows.filter(+ row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed+ )+ // Positive witness first: the shell step ran and failed with the store code.+ expect(shellFailed).toHaveLength(1)+ expect(shellFailed[0]?.failureCode).toBe('download_busy')+ expect(+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)+ ).toHaveLength(1)+ expect(+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)+ ).toHaveLength(0)+ expect(+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer)+ ).toHaveLength(0)+ })+ expectHostUnchanged(ctx, result)+ } finally {+ ensureWorkloadsUp(ctx.kubectl)+ }+})diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.tsindex 7912bc0d6..31c621059 100644--- a/desktop-app/test/imageCapabilityEvidence.test.ts+++ b/desktop-app/test/imageCapabilityEvidence.test.ts@@ -44,6 +44,24 @@ const rows = {truncated: true,}),textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),+ shellRequested: (): Row => ({+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'legacy-lease-shell-requested',+ }),+ legacyAnswer: (): Row => ({+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'legacy-lease-answer',+ downloadSha256: DIGEST,+ downloadBytes: 42,+ }),+ shellFailed: (): Row => ({+ model: 'glm-5.3-flash',+ imageSha256: null,+ responseKind: 'legacy-lease-shell-failed',+ failureCode: 'download_busy',+ }),}function ledger(attempts: Row[]): string {@@ -63,6 +81,10 @@ function ledger(attempts: Row[]): string {documentReadRequests: 0,documentAnswers: 0,documentFailures: 0,+ legacyLeaseShellRequests: 0,+ legacyLeaseDownloadRequests: 0,+ legacyLeaseAnswers: 0,+ legacyLeaseFailures: 0,},attempts,})@@ -120,6 +142,47 @@ describe('imageCapabilityEvidence ledger rows', () => {})})+describe('legacy-lease ledger rows', () => {+ it('keeps the failure code and the download receipt on the rows that carry them', () => {+ const snapshot = parse([rows.shellRequested(), rows.legacyAnswer(), rows.shellFailed()])+ expect(snapshot.attempts).toEqual([+ rows.shellRequested(),+ rows.legacyAnswer(),+ rows.shellFailed(),+ ])+ })++ it('requires a closed-set failure code on failure rows and nowhere else', () => {+ // Witness: the well-formed failure row parses with its code.+ expect(parse([rows.shellFailed()]).attempts[0]?.failureCode).toBe('download_busy')+ const { failureCode: _code, ...withoutCode } = rows.shellFailed()+ for (const [label, row] of [+ ['missing failureCode', withoutCode],+ ['free-text failureCode', { ...rows.shellFailed(), failureCode: 'disk on fire' }],+ ['failureCode on an answer row', { ...rows.legacyAnswer(), failureCode: 'download_busy' }],+ ['failureCode on a text row', { ...rows.textOnly(), failureCode: 'download_busy' }],+ ] as const) {+ expect(() => parse([row]), label).toThrow(/failureCode/)+ }+ })++ it('requires the download receipt on every answer row and nowhere else', () => {+ // Witness: the well-formed answer row parses with its receipt.+ expect(parse([rows.legacyAnswer()]).attempts[0]?.downloadBytes).toBe(42)+ const { downloadSha256: _sha, ...withoutSha } = rows.legacyAnswer()+ const { downloadBytes: _bytes, ...withoutBytes } = rows.legacyAnswer()+ for (const [label, row] of [+ ['missing downloadSha256', withoutSha],+ ['short downloadSha256', { ...rows.legacyAnswer(), downloadSha256: 'abc' }],+ ['missing downloadBytes', withoutBytes],+ ['negative downloadBytes', { ...rows.legacyAnswer(), downloadBytes: -1 }],+ ['receipt on a shell row', { ...rows.shellRequested(), downloadSha256: DIGEST }],+ ] as const) {+ expect(() => parse([row]), label).toThrow(/downloadSha256|downloadBytes/)+ }+ })+})+describe('appendedAttempts', () => {it('refuses a row whose reason or delivered page changed between reads', () => {const before = parse([rows.rejected(), rows.answer()])@@ -136,6 +199,21 @@ describe('appendedAttempts', () => {expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)}})++ it('refuses a legacy-lease row whose receipt or failure code changed between reads', () => {+ const before = parse([rows.legacyAnswer(), rows.shellFailed()])+ // Witness: an unchanged prefix yields the appended row.+ expect(+ appendedAttempts(before, parse([rows.legacyAnswer(), rows.shellFailed(), rows.textOnly()]))+ ).toEqual([rows.textOnly()])+ for (const [label, changed] of [+ ['sha', [{ ...rows.legacyAnswer(), downloadSha256: 'c'.repeat(64) }, rows.shellFailed()]],+ ['bytes', [{ ...rows.legacyAnswer(), downloadBytes: 43 }, rows.shellFailed()]],+ ['code', [rows.legacyAnswer(), { ...rows.shellFailed(), failureCode: 'writer_locked' }]],+ ] as const) {+ expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)+ }+ })})describe('rejection reason parity', () => {@@ -153,4 +231,17 @@ describe('rejection reason parity', () => {expect((declared as readonly string[]).length).toBeGreaterThan(0)expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')})++ it('declares the same legacy-lease failure codes as the fixture, independently', async () => {+ const providerModule = (await import(PROVIDER_MODULE_URL)) as {+ LEGACY_LEASE_FAILURE_CODES: readonly string[]+ }+ const declared = (evidenceModule as Record<string, unknown>).FIXTURE_LEGACY_LEASE_FAILURE_CODES+ expect(Array.isArray(declared)).toBe(true)+ expect(declared).not.toBe(providerModule.LEGACY_LEASE_FAILURE_CODES)+ expect([...(declared as readonly string[])].sort()).toEqual(+ [...providerModule.LEGACY_LEASE_FAILURE_CODES].sort()+ )+ expect(declared as readonly string[]).toContain('download_busy')+ })})diff --git a/desktop-app/test/legacyLeaseRestart.test.ts b/desktop-app/test/legacyLeaseRestart.test.tsnew file mode 100644index 000000000..9c9d358e1--- /dev/null+++ b/desktop-app/test/legacyLeaseRestart.test.ts@@ -0,0 +1,238 @@+import { describe, expect, it } from 'vitest'+import {+ LEGACY_DISCARD_COUNTER,+ downloadedRecord,+ legacyLease,+ legacyLeaseRecoveryVerdict,+ parseMetricValue,+ parseSeedSummary,+ requireLegacyLeaseLaneEnv,+ vacuityStoreVerdict,+} from './e2e-playwright/helpers/legacyLeaseRestart'++const PROFILE = 'clerum-fix-issue-1022-processing-lease-lifecycle-2e0feeb9'+const RUN_ID = 'image-capabilities-0123456789ab'+const SHA = 'a'.repeat(64)++const discardLine = JSON.stringify({+ level: 40,+ component: 'GfsDownloadStore',+ discarded: 1,+ msg: 'GFS download store discarded 1 legacy processing lease(s) at initialize',+})+const unknownLine = JSON.stringify({+ level: 40,+ component: 'GfsDownloadStore',+ legacyProcessingLeases: 1,+ outcome: 'unknown',+ msg: 'GFS download store could not confirm the discard of 1 legacy processing lease(s)',+})+const unrelatedLine = JSON.stringify({ level: 30, component: 'Server', msg: 'listening' })++describe('requireLegacyLeaseLaneEnv', () => {+ const valid = {+ MINIKUBE_PROFILE: PROFILE,+ CONTROL_API_REAL_PG_CONTEXT: PROFILE,+ E2E_K8S_CONTEXT: PROFILE,+ LEGACY_LEASE_LANE_MODE: 'fixed',+ LEGACY_LEASE_SEED_LABEL_VALUE: RUN_ID,+ IMAGE_CAPABILITIES_RUN_ID: RUN_ID,+ }++ it('accepts the runner bindings', () => {+ expect(requireLegacyLeaseLaneEnv(valid)).toEqual({+ profile: PROFILE,+ mode: 'fixed',+ runId: RUN_ID,+ })+ })++ it('lists every inconsistent binding at once', () => {+ let message = ''+ try {+ requireLegacyLeaseLaneEnv({+ ...valid,+ E2E_K8S_CONTEXT: 'minikube',+ LEGACY_LEASE_LANE_MODE: 'skip',+ LEGACY_LEASE_SEED_LABEL_VALUE: 'image-capabilities-ffffffffffff',+ })+ } catch (error) {+ message = (error as Error).message+ }+ expect(message).toContain('E2E_K8S_CONTEXT')+ expect(message).toContain('LEGACY_LEASE_LANE_MODE')+ expect(message).toContain('LEGACY_LEASE_SEED_LABEL_VALUE')+ })+})++describe('legacyLease', () => {+ it('is a valid, unexpired, foreign lease with no records and no writer session', () => {+ const now = Date.parse('2026-10-08T10:00:00.000Z')+ const lease = legacyLease(now)+ expect(lease.leaseId).toMatch(/^[0-9a-f-]{36}$/)+ expect(lease.recordIds).toEqual([])+ expect(lease.callerIdentity).not.toBe('')+ expect(Date.parse(lease.expiresAt)).toBeGreaterThan(now)+ expect(Date.parse(lease.expiresAt)).toBeGreaterThan(Date.parse(lease.acquiredAt))+ expect(Object.hasOwn(lease, 'writerSessionId')).toBe(false)+ })+})++describe('parseSeedSummary', () => {+ const summary = (fields: Record<string, unknown>) =>+ `LEGACY_LEASE_SEED ${JSON.stringify({ leaseId: 'l1', records: 0, mode: '600', uid: 1001, gid: 1001, ...fields })}\n`++ it('reads the one summary of the seeded lease', () => {+ expect(parseSeedSummary(summary({}), 'l1').records).toBe(0)+ })++ it('refuses a ledger left with another mode or owner', () => {+ expect(() => parseSeedSummary(summary({ mode: '644' }), 'l1')).toThrow(/mode 644/)+ expect(() => parseSeedSummary(summary({ uid: 0 }), 'l1')).toThrow(/owner 0:1001/)+ })++ it('refuses no summary, two summaries, or another lease', () => {+ expect(() => parseSeedSummary('', 'l1')).toThrow(/0 summaries/)+ expect(() => parseSeedSummary(summary({}) + summary({}), 'l1')).toThrow(/2 summaries/)+ expect(() => parseSeedSummary(summary({}), 'l2')).toThrow(/another lease/)+ })+})++describe('parseMetricValue', () => {+ const exposition = [+ `# HELP ${LEGACY_DISCARD_COUNTER} Legacy processing leases discarded.`,+ `# TYPE ${LEGACY_DISCARD_COUNTER} counter`,+ `${LEGACY_DISCARD_COUNTER} 1`,+ 'process_cpu_seconds_total 3.2',+ ].join('\n')++ it('reads an unlabelled sample', () => {+ expect(parseMetricValue(exposition, LEGACY_DISCARD_COUNTER)).toBe(1)+ })++ it('returns null when the metric is absent', () => {+ expect(parseMetricValue('process_cpu_seconds_total 3.2\n', LEGACY_DISCARD_COUNTER)).toBeNull()+ })++ it('refuses duplicate or labelled samples', () => {+ expect(() =>+ parseMetricValue(`${exposition}\n${LEGACY_DISCARD_COUNTER} 2`, LEGACY_DISCARD_COUNTER)+ ).toThrow(/2 samples/)+ expect(() =>+ parseMetricValue(`${LEGACY_DISCARD_COUNTER}{a="b"} 1`, LEGACY_DISCARD_COUNTER)+ ).toThrow(/not an unlabelled number/)+ })+})++describe('legacyLeaseRecoveryVerdict', () => {+ const ledger = { schemaVersion: 1, records: {} }++ it('accepts one confirmed discard with the counter at 1', () => {+ expect(+ legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine, discardLine], counter: 1 })+ ).toBe('discarded')+ })++ it('accepts one unknown-outcome warning with the counter at 0', () => {+ expect(legacyLeaseRecoveryVerdict({ ledger, logLines: [unknownLine], counter: 0 })).toBe(+ 'unknown-outcome'+ )+ })++ it('refuses a ledger that still carries processingLeases', () => {+ expect(() =>+ legacyLeaseRecoveryVerdict({+ ledger: { ...ledger, processingLeases: {} },+ logLines: [discardLine],+ counter: 1,+ })+ ).toThrow(/still carries processingLeases/)+ })++ it('refuses when no outcome was logged (the Host never ran the discard)', () => {+ expect(() =>+ legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine], counter: 0 })+ ).toThrow(/0 discard line\(s\).*0 unknown-outcome/)+ })++ it('refuses a discard whose counter disagrees, and a missing counter', () => {+ expect(() =>+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: 0 })+ ).toThrow(/counter 0/)+ expect(() =>+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: null })+ ).toThrow(/not exposed/)+ })++ it('refuses both outcomes at once, and a discard of more than one lease', () => {+ expect(() =>+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine, unknownLine], counter: 1 })+ ).toThrow(/no recovery outcome/)+ const two = discardLine.replace('"discarded":1', '"discarded":2')+ expect(() => legacyLeaseRecoveryVerdict({ ledger, logLines: [two], counter: 2 })).toThrow(+ /no recovery outcome/+ )+ })+})++describe('vacuityStoreVerdict', () => {+ const ledger = { schemaVersion: 1, records: {}, processingLeases: { l1: {} } }++ it('accepts the pre-fix Host keeping the lease without a counter', () => {+ expect(+ vacuityStoreVerdict({ ledger, logLines: [unrelatedLine], counter: null, leaseId: 'l1' })+ ).toBe('lease-retained')+ })++ it('refuses a discarded lease, a discard line, or an exposed counter', () => {+ expect(() =>+ vacuityStoreVerdict({+ ledger: { schemaVersion: 1, records: {} },+ logLines: [],+ counter: null,+ leaseId: 'l1',+ })+ ).toThrow(/no longer carries/)+ expect(() =>+ vacuityStoreVerdict({ ledger, logLines: [discardLine], counter: null, leaseId: 'l1' })+ ).toThrow(/logged a legacy-lease discard/)+ expect(() => vacuityStoreVerdict({ ledger, logLines: [], counter: 0, leaseId: 'l1' })).toThrow(+ /exposes/+ )+ })+})++describe('downloadedRecord', () => {+ const record = (id: string, fields: Record<string, unknown>) => ({+ id,+ hostPath: `downloads/${id}`,+ sizeBytes: 10,+ sha256: SHA,+ state: 'completed',+ ...fields,+ })+ const before = { schemaVersion: 1, records: { old: record('old', {}) } }++ it('returns the one new completed record with the source digest and size', () => {+ const after = { schemaVersion: 1, records: { ...before.records, fresh: record('fresh', {}) } }+ expect(downloadedRecord(before, after, { sha256: SHA, bytes: 10 }).id).toBe('fresh')+ })++ it('refuses no new record, a different digest, or an unfinished transfer', () => {+ expect(() => downloadedRecord(before, before, { sha256: SHA, bytes: 10 })).toThrow(/added \[\]/)+ const other = {+ schemaVersion: 1,+ records: { fresh: record('fresh', { sha256: 'b'.repeat(64) }) },+ }+ expect(() => downloadedRecord(before, other, { sha256: SHA, bytes: 10 })).toThrow(+ /expected one/+ )+ const partial = {+ schemaVersion: 1,+ records: { fresh: record('fresh', { state: 'transferring' }) },+ }+ expect(() => downloadedRecord(before, partial, { sha256: SHA, bytes: 10 })).toThrow(+ /expected one/+ )+ })+})diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.mdindex 97103dfb3..1c7b4d75a 100644--- a/docs/testing/minikube-t2-runbook.md+++ b/docs/testing/minikube-t2-runbook.md@@ -237,6 +237,105 @@ real-subscription interoperability or authorize real account use. The fourpublic targets and the guarded build body have hermetic boundary coverage in`scripts/tests/test-minikube-mutation-boundary.sh`.+### Legacy processing-lease restart lane++Issue #1022. This lane proves that a Host booting on a GFS download store+ledger that still carries a processing lease written before #1019 (no writer+session) discards that lease and keeps serving shell commands and GFS downloads.+It is development-only, uses the image-capabilities provider fixture, and has+the same requirements as that lane: Node 24, a clean tree, the branch-owned+`MINIKUBE_PROFILE` with a matching `CONTROL_API_REAL_PG_CONTEXT`, the mutation+lease, and the single-Host (`mcp-host/chatllm`) profile.++```sh+make minikube-build-image-capabilities-fixture+make minikube-t2+make minikube-run-legacy-lease-restart+```++The Playwright config is+`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts`.+The lane runs five serial scenarios. Each one scales HCC and then the Host+to 0. It waits until the Host pod is gone, and then seeds one legacy lease. The+lease is valid and not expired, belongs to a foreign caller, has+`recordIds: []` and has no `writerSessionId`. The seed pod is labelled+`evenfire.ai/legacy-lease-seed=<run id>`. It runs the Host image as uid/gid+1001 and mounts the Host PVC. It does a read-modify-write using the store's own+publication sequence: a temp file with mode 0600, then fsync, rename, and an+fsync of the directory. It refuses a ledger that already has leases. The pod+reads the ledger back and is deleted before the scenario continues.++| Scenario | How the Host is stopped | What runs after the seed |+| --- | --- | --- |+| `S-crash` | `delete pod --force --grace-period=0`, then an immediate scale to 0 | The Host and HCC return. The scenario fails with `CRASH_WINDOW_LOST` if a replacement pod started a container in between. |+| `S-graceful` | Scale to 0 (SIGTERM, full grace) | `rollout restart` of the Host, then the Host and HCC return |+| `S-hcc` | Scale to 0 | HCC and the Host return, so HCC is a new pod |+| `S-gfs` | Scale to 0 | Each GFS pod restarts in turn (`gfsc-writer`, then `gfsc-reader`), then the Host and HCC return |+| `S-update` | Scale to 0 | Upgrade order: GFS, then WRC, then HCC, then the Host |++After each scenario, the lane checks the following:++- **Pods.** The Host has a new pod UID and is Ready. HCC does not replace it+ after it boots. The Host PVC has the same name and UID before and after.+- **Store.** The served ledger has no `processingLeases`. The lane then needs+ one of two outcomes:+ - a `GfsDownloadStore` warning that 1 legacy lease was discarded, with+ `clerum_gfs_legacy_processing_leases_discarded_total` at 1;+ - one warning that the outcome is unknown, with that counter at 0.+- **Journey.** The Desktop user sends the journey line and approves+ `shell_exec` by clicking Approve. The fixture then requests+ `clerum__gfs_download` for a seeded GFS file.+ - The answer on screen and the fixture ledger must both carry the GFS+ source's sha256 and byte count.+ - The Host store must record exactly one new completed download with that+ digest.+ - The file in the Host workspace must hash to the same value.+ - The journey makes no direct RPC calls and uses no storage shortcuts or+ in-cluster mocks.++**Vacuity.** Run the vacuity lane once, and run it before the fixed lane. The+pre-fix Host quarantines every ledger record when it sees an inherited executor.+Those records stay on the PVC and keep counting against the caller's quota. The+fixed Host then discards the seeded lease when the runner restores it.++```sh+make minikube-build-legacy-lease-vacuity-fixture+make minikube-run-legacy-lease-restart-vacuity+```++The build archives base `74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e` and checks+that its store has the inherited-executor guard and no legacy-lease discard. It+builds `clerum/mcp-host:legacy-lease-vacuity` from that archive and puts this+HEAD's fixture layer on top as+`clerum/image-capabilities-mcp-host:legacy-lease-vacuity`. It then writes+`.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json` in the+canonical checkout. The runner checks the profile, the base revision, the+fixture-layer revision and the image identities against that manifest before it+patches anything.++The vacuity lane runs one `S-crash` scenario. It passes only when all of these+hold:++- the seeded lease is still in the ledger;+- no discard line was logged and the counter does not exist;+- the journey stops at the shell with+ `LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy`, with no download+ request.++**Restoration.** The runner journals the original HCC and Host replica counts+before Playwright starts. Zero is never a restored state. Restoration does the+following:++- deletes any seed pod of the run;+- scales the Host and then HCC back;+- waits for GFS, WRC, HCC and the Host to become Ready;+- restores the production Host image;+- waits until the Host's ledger has no `processingLeases`;+- fails when a live `e2e-gfs-legacy-lease-*` GFS fixture remains.++`make minikube-restore-image-capabilities` resumes an interrupted run the same+way.+### Orphaned lock recoveryIf `PROFILE_BUSY` reports that the lock has no valid owner PID, or that a stalediff --git a/scripts/e2e/build-legacy-lease-vacuity-image.sh b/scripts/e2e/build-legacy-lease-vacuity-image.shnew file mode 100755index 000000000..54ceac335--- /dev/null+++ b/scripts/e2e/build-legacy-lease-vacuity-image.sh@@ -0,0 +1,143 @@+#!/usr/bin/env bash+# Build the two images of the legacy processing-lease vacuity lane (issue #1022)+# inside the owned minikube profile, under the inherited T2 mutation lease:+#+# clerum/mcp-host:legacy-lease-vacuity+# mcp-host built from the pre-fix base revision (74e0d81d9, before #1019),+# whose store treats a legacy lease as an inherited executor;+# clerum/image-capabilities-mcp-host:legacy-lease-vacuity+# the provider-fixture layer of THIS HEAD on top of that Host.+#+# It then writes the manifest `make minikube-run-legacy-lease-restart-vacuity`+# proves before patching anything:+# <canonical>/.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json+#+# Nothing is published. Run it through+# `make minikube-build-legacy-lease-vacuity-fixture`.+set -euo pipefail++root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"+profile="${MINIKUBE_PROFILE:?MINIKUBE_PROFILE is required}"+base_revision=74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e+base_image=clerum/mcp-host:legacy-lease-vacuity+fixture_image=clerum/image-capabilities-mcp-host:legacy-lease-vacuity+deadline="${root}/scripts/minikube/run-with-deadline.mjs"++bash "${root}/scripts/minikube/require-t2-mutation-lock.sh"++[[ "${profile}" =~ ^clerum-.+-[a-f0-9]{8}$ ]] || {+ printf 'LEGACY_LEASE_VACUITY: %s is not a branch profile\n' "${profile}" >&2+ exit 1+}+[[ -z "$(git -C "${root}" status --porcelain)" ]] || {+ printf 'LEGACY_LEASE_VACUITY: the worktree must be clean (the fixture layer is built from HEAD)\n' >&2+ exit 1+}+head_revision="$(git -C "${root}" rev-parse --verify HEAD)"+[[ "$(git -C "${root}" cat-file -t "${base_revision}")" == commit ]] || {+ printf 'LEGACY_LEASE_VACUITY: base revision %s is not in this repository\n' "${base_revision}" >&2+ exit 1+}+common_dir="$(git -C "${root}" rev-parse --path-format=absolute --git-common-dir)"+canonical="$(cd -- "$(dirname -- "${common_dir}")" && pwd -P)"+[[ "$(basename -- "${canonical}")" == evenfire ]] || {+ printf 'LEGACY_LEASE_VACUITY: canonical Evenfire checkout required\n' >&2+ exit 1+}++run_dir="${canonical}/.local-notes/infra/runs/legacy-lease-vacuity/${profile}"+source_dir="${run_dir}/base-${base_revision:0:12}"++# Docker runs through the repository's isolated CLI environment (an empty task+# config, no ambient credentials), pointed at the profile's own daemon with+# `minikube docker-env`, exactly as scripts/minikube/build-images.sh does.+# shellcheck source=scripts/minikube/docker-cli-env.sh+source "${root}/scripts/minikube/docker-cli-env.sh"+cleanup() {+ local status=$? cleanup_status=0+ trap - EXIT INT TERM+ docker_cli_env_cleanup || cleanup_status=$?+ rm -rf -- "${source_dir}"+ if [[ "${status}" -eq 0 && "${cleanup_status}" -ne 0 ]]; then status="${cleanup_status}"; fi+ exit "${status}"+}+trap cleanup EXIT+trap 'exit 130' INT+trap 'exit 143' TERM++mkdir -p "${run_dir}"+rm -rf -- "${source_dir}"+mkdir -p "${source_dir}"+git -C "${root}" archive --format=tar "${base_revision}" | tar -x -C "${source_dir}"++# The archive must be the pre-fix store: it keeps the inherited-executor guard+# and has no legacy-lease discard. Either check failing means a wrong base.+store="${source_dir}/mcp-host/src/internalTools/gfsDownloadStore.ts"+grep -q 'hasInheritedExecutors' "${store}" || {+ printf 'LEGACY_LEASE_VACUITY: base store has no inherited-executor guard\n' >&2+ exit 1+}+if grep -q 'discardLegacyProcessingLeases' "${store}"; then+ printf 'LEGACY_LEASE_VACUITY: base store already discards legacy leases\n' >&2+ exit 1+fi++docker_cli_env_prepare+docker_env_output="$(node "${deadline}" --timeout-seconds 60 --kill-grace-seconds 5 --label minikube-docker-env -- \+ minikube -p "${profile}" docker-env --shell bash)"+[[ -n "${docker_env_output}" ]] || {+ printf 'LEGACY_LEASE_VACUITY: minikube returned an empty Docker environment\n' >&2+ exit 1+}+eval "${docker_env_output}"+unset DOCKER_API_VERSION++inspect() {+ docker_cli_run_public "inspect-$1" 60 docker image inspect "$1" --format "$2" | tr -d '\r\n'+}++# The base Host image from the archived pre-fix tree. Its build context is the+# archive root, which carries that revision's own root .dockerignore.+docker_cli_run_public legacy-lease-vacuity-host-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \+ docker build --tag "${base_image}" --file "${source_dir}/mcp-host/Dockerfile" "${source_dir}"+base_id="$(inspect "${base_image}" '{{.Id}}')"++# The provider-fixture layer of THIS HEAD on top of that Host.+docker_cli_run_public legacy-lease-vacuity-fixture-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \+ docker build --tag "${fixture_image}" \+ --build-arg "MCP_HOST_IMAGE=${base_image}" \+ --file "${root}/tests/e2e/fixtures/image-capabilities/Dockerfile" "${root}"+fixture_id="$(inspect "${fixture_image}" '{{.Id}}')"++for id in "${base_id}" "${fixture_id}"; do+ [[ "${id}" =~ ^sha256:[a-f0-9]{64}$ ]] || {+ printf 'LEGACY_LEASE_VACUITY: unexpected image id "%s"\n' "${id}" >&2+ exit 1+ }+done+# The fixture image must extend exactly the base just built.+parent_layers="$(inspect "${base_image}" '{{json .RootFS.Layers}}')"+fixture_layers="$(inspect "${fixture_image}" '{{json .RootFS.Layers}}')"+[[ "${fixture_layers}" == "${parent_layers%]},"* ]] || {+ printf 'LEGACY_LEASE_VACUITY: the fixture layer does not extend the pre-fix Host image\n' >&2+ exit 1+}++manifest="${run_dir}/manifest.json"+temporary="$(mktemp "${run_dir}/manifest.json.XXXXXX")"+node -e '+const [file, profile, baseRevision, head, baseImage, baseId, fixtureImage, fixtureId] = process.argv.slice(1)+const manifest = {+ kind: "legacy-lease-vacuity",+ profile,+ baseRevision,+ fixtureLayerRevision: head,+ images: { [baseImage]: baseId, [fixtureImage]: fixtureId },+ derivedFrom: { [fixtureImage]: { ref: baseImage, id: baseId } },+ builtAt: new Date().toISOString(),+}+require("node:fs").writeFileSync(file, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 })+' "${temporary}" "${profile}" "${base_revision}" "${head_revision}" \+ "${base_image}" "${base_id}" "${fixture_image}" "${fixture_id}"+mv -f -- "${temporary}" "${manifest}"+printf 'LEGACY_LEASE_VACUITY_BUILT manifest=%s base=%s fixture=%s\n' "${manifest}" "${base_id}" "${fixture_id}"diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjsindex 91594cc2f..3544bc021 100644--- a/scripts/e2e/image-capabilities-fixture.mjs+++ b/scripts/e2e/image-capabilities-fixture.mjs@@ -42,6 +42,69 @@ export const playwrightLanes = {label: 'document-upload-playwright',grep: 'document-upload fixture: ',},+ // Issue #1022: legacy processing-lease restart scenarios on the fixed Host.+ 'legacy-lease-restart': {+ config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',+ label: 'legacy-lease-restart-playwright',+ grep: 'legacy-lease-restart fixture: ',+ // S-crash, S-graceful, S-hcc, S-gfs and S-update: a shorter list is a lost scenario.+ expectedSpecs: 5,+ },+ // The same journey on a Host built from the pre-fix base revision: it must+ // fail at the shell step with download_busy, or the lane proves nothing.+ 'legacy-lease-restart-vacuity': {+ config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',+ label: 'legacy-lease-restart-vacuity-playwright',+ grep: 'legacy-lease-restart vacuity: ',+ expectedSpecs: 1,+ },+}++/**+ * The pre-fix revision the vacuity lane runs (#1022 base, before #1019), and+ * the two images `scripts/e2e/build-legacy-lease-vacuity-image.sh` builds from+ * it. They never share a tag with the HEAD images.+ */+export const legacyLeaseVacuity = Object.freeze({+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',+ baseImage: 'clerum/mcp-host:legacy-lease-vacuity',+ fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',+ manifestKind: 'legacy-lease-vacuity',+})++export function legacyLeaseVacuityManifestPath(canonical, profile) {+ return path.join(+ canonical,+ '.local-notes/infra/runs/legacy-lease-vacuity',+ profile,+ 'manifest.json'+ )+}++/** Label of the seed pods the legacy-lease spec creates; restoration deletes them. */+export const legacyLeaseSeedLabel = 'evenfire.ai/legacy-lease-seed'++/**+ * Runtime of each lane: the Host image HCC is pointed at, the Playwright+ * deadline, and whether the lane restarts workloads (legacy-lease lanes scale+ * HCC and the Host to zero and restart GFS and WRC, so restoration must bring+ * them back).+ */+const laneRuntimes = {+ image: { fixtureImage, timeoutSeconds: 900, legacyLease: null },+ 'document-upload': { fixtureImage, timeoutSeconds: 900, legacyLease: null },+ // Five scenarios, each with workload stop/start cycles and one Desktop launch.+ 'legacy-lease-restart': { fixtureImage, timeoutSeconds: 3600, legacyLease: 'fixed' },+ 'legacy-lease-restart-vacuity': {+ fixtureImage: legacyLeaseVacuity.fixtureImage,+ timeoutSeconds: 900,+ legacyLease: 'vacuity',+ },+}++export function laneRuntime(lane) {+ if (!Object.hasOwn(laneRuntimes, lane)) throw new Error(`Unknown lane ${JSON.stringify(lane)}`)+ return laneRuntimes[lane]}export function playwrightLaneConfig(env) {@@ -73,7 +136,7 @@ const isCount = value => Number.isSafeInteger(value) && value >= 0* test and no unexpected, skipped or flaky outcome.*/export function playwrightVerdict(report, { lane, configFile }) {- const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })+ const { grep, expectedSpecs } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })const reported = report?.config?.configFileif (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))throw new Error('Playwright report config mismatch')@@ -102,6 +165,10 @@ export function playwrightVerdict(report, { lane, configFile }) {)if (titles.some(title => !title.startsWith(grep)))throw new Error(`Playwright report has specs outside the ${lane} lane`)+ if (expectedSpecs !== undefined && titles.length !== expectedSpecs)+ throw new Error(+ `Playwright report lists ${titles.length} ${lane} specs, expected ${expectedSpecs}`+ )return 'PASS'}@@ -137,6 +204,53 @@ export function proveImages(manifest, observed, head, profile) {return { head, profile, images: observed }}+/**+ * Proves the vacuity images: the Host layer was built from the pre-fix base+ * revision, the fixture layer from this HEAD, both for this profile, and the+ * fixture layer derives from the exact base image present in the node.+ */+export function proveVacuityImages(manifest, observed, head, profile) {+ const { baseRevision, baseImage: vacuityBase, fixtureImage: vacuityFixture } = legacyLeaseVacuity+ if (+ !/^[a-f0-9]{40}$/.test(head) ||+ manifest?.kind !== legacyLeaseVacuity.manifestKind ||+ manifest.profile !== profile+ )+ throw new Error('Vacuity image ownership mismatch')+ if (manifest.baseRevision !== baseRevision)+ throw new Error('Vacuity Host image was not built from the pre-fix base revision')+ if (manifest.fixtureLayerRevision !== head)+ throw new Error('Vacuity fixture layer was not built from this HEAD')+ for (const ref of [vacuityBase, vacuityFixture]) {+ const id = manifest.images?.[ref]+ if (!/^sha256:[a-f0-9]{64}$/.test(id ?? '') || id !== observed[ref])+ throw new Error('Vacuity image identity mismatch')+ }+ const binding = manifest.derivedFrom?.[vacuityFixture]+ if (binding?.ref !== vacuityBase || binding.id !== observed[vacuityBase])+ throw new Error('Vacuity fixture derived-base mismatch')+ return { head, profile, baseRevision, images: observed }+}++/**+ * The legacy-lease lanes scale HCC and the Host. Each must be back at its+ * recorded replica count; zero is never a restored state.+ */+export function legacyLeaseReplicaPlan(recorded, live) {+ const plan = []+ // The Host first: HCC sets the Host replicas itself, and a Host already at+ // its count makes HCC's first reconcile a no-op.+ for (const key of ['host', 'hcc']) {+ const want = recorded?.[key]+ if (!Number.isSafeInteger(want) || want < 1)+ throw new Error(`Recorded ${key} replicas must be a positive integer`)+ if (!Number.isSafeInteger(live?.[key]) || live[key] < 0)+ throw new Error(`Live ${key} replicas are unreadable`)+ if (live[key] !== want) plan.push({ key, replicas: want })+ }+ return plan+}+export function modelInputs(runId) {if (!/^image-capabilities-[a-f0-9]{12}$/.test(runId)) throw new Error('Invalid fixture run')return [@@ -256,6 +370,7 @@ async function main() {label: laneLabel,grep: laneGrep,} = selectFixtureAction(process.argv[2], process.env)+ const runtime = action === 'run' ? laneRuntime(lane) : nullconst head = command('git', ['rev-parse', 'HEAD']).trim()const branch = command('git', ['branch', '--show-current']).trim()if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')@@ -339,6 +454,8 @@ async function main() {profile,head,root,+ fixtureImage: runtime.fixtureImage,+ legacyLease: runtime.legacyLease,rows: [],changes: {},restored: false,@@ -350,6 +467,10 @@ async function main() {(action === 'run' && state.head !== head))throw new Error('Run state identity mismatch')+ // Restore reads the lane's Host image from the persisted state, never from the+ // environment, so a stale lane variable cannot point restoration elsewhere.+ if (typeof state.fixtureImage !== 'string' || !state.fixtureImage)+ throw new Error('Run state has no fixture image; restore it with the runner that created it')const save = () => {const destination = path.join(evidence, 'state.json')if (fs.existsSync(destination)) {@@ -424,8 +545,86 @@ async function main() {if (!response.ok) throw new Error(`Fixture admin operation failed (${response.status})`)return response.status === 204 ? null : response.json()}+ const deploymentReplicas = (namespace, name) => get(namespace, 'deployment', name).spec.replicas+ const restoreLegacyLeaseWorkloads = () => {+ // Seed pods mount the Host PVC; none may outlive the run.+ kc([+ '-n',+ 'mcp-host',+ 'delete',+ 'pod',+ '-l',+ `${legacyLeaseSeedLabel}=${state.runId}`,+ '--ignore-not-found',+ '--wait=true',+ '--timeout=90s',+ ])+ const remaining = kc([+ '-n',+ 'mcp-host',+ 'get',+ 'pod',+ '-l',+ legacyLeaseSeedLabel,+ '-o',+ 'jsonpath={.items[*].metadata.name}',+ ]).trim()+ if (remaining) throw new Error('Legacy-lease seed pods remain in mcp-host')+ const liveReplicas = () => ({+ host: deploymentReplicas('mcp-host', 'chatllm'),+ hcc: deploymentReplicas('control-plane', 'host-context-controller'),+ })+ for (const { key, replicas } of legacyLeaseReplicaPlan(+ state.legacyLeaseReplicas,+ liveReplicas()+ )) {+ const [namespace, name] =+ key === 'host' ? ['mcp-host', 'chatllm'] : ['control-plane', 'host-context-controller']+ kc(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])+ }+ for (const [namespace, name] of [+ ['gfs', 'gfsc-writer'],+ ['gfs', 'gfsc-reader'],+ ['control-plane', 'workflow-recipes'],+ ['control-plane', 'host-context-controller'],+ ['mcp-host', 'chatllm'],+ ])+ kc(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=150s'])+ if (legacyLeaseReplicaPlan(state.legacyLeaseReplicas, liveReplicas()).length)+ throw new Error('HCC or the Host is not back at its recorded replicas')+ }+ // The Host discards a legacy lease at initialize, which runs at boot and can+ // wait out writer contention, so the ledger is polled with a bound.+ const waitLedgerWithoutLegacyLeases = async () => {+ const until = Date.now() + 240_000+ let last = 'unread'+ while (Date.now() < until) {+ try {+ const ledger = JSON.parse(+ kc([+ '-n',+ 'mcp-host',+ 'exec',+ 'deployment/chatllm',+ '-c',+ 'mcp-host',+ '--',+ 'cat',+ '/workspace/.gfs-download-store/ledger-v1.json',+ ])+ )+ if (!Object.hasOwn(ledger, 'processingLeases')) return+ last = 'processingLeases present'+ } catch (error) {+ last = error.message+ }+ await new Promise(resolve => setTimeout(resolve, 2_000))+ }+ throw new Error(`Restored Host ledger still carries legacy processing leases (${last})`)+ }const restoreRuntime = async () => {if (state.restored) return+ if (state.legacyLease && state.legacyLeaseReplicas) restoreLegacyLeaseWorkloads()// Restore the model before removing its fixture-only catalog rows.if (state.changes.host) {const current = get('mcp-host', 'host', 'chatllm')@@ -500,7 +699,7 @@ async function main() {path: state.hcc.path.replace(/value$/, 'name'),value: 'CONTEXT_MAPPER_HOST_IMAGE',},- { op: 'test', path: state.hcc.path, value: fixtureImage },+ { op: 'test', path: state.hcc.path, value: state.fixtureImage },{ op: 'replace', path: state.hcc.path, value: state.hcc.image },unmark,])@@ -582,6 +781,9 @@ async function main() {'process.stdout.write(String(!process.env.EVENFIRE_IMAGE_CAPABILITIES_FIXTURE && !require("node:fs").existsSync("/tmp/image-capabilities-evidence.json")))',]).trim()if (predicate !== 'true') throw new Error('Fixture state remains in Host')+ // The vacuity lane leaves its seeded lease on the PVC (the pre-fix Host+ // cannot discard it); the restored fixed Host must discard it at boot.+ if (state.legacyLease) await waitLedgerWithoutLegacyLeases()state.restored = truesave()}@@ -596,11 +798,20 @@ async function main() {}let failuretry {+ const vacuity = runtime.legacyLease === 'vacuity'const manifest = JSON.parse(- fs.readFileSync(path.join(root, 'deploy/minikube/.image-manifest.json'), 'utf8')+ fs.readFileSync(+ vacuity+ ? legacyLeaseVacuityManifestPath(canonical, profile)+ : path.join(root, 'deploy/minikube/.image-manifest.json'),+ 'utf8'+ ))+ const proofRefs = vacuity+ ? [legacyLeaseVacuity.baseImage, legacyLeaseVacuity.fixtureImage]+ : [baseImage, fixtureImage]const observed = Object.fromEntries(- [baseImage, fixtureImage].map(ref => [+ proofRefs.map(ref => [ref,command('minikube', ['-p',@@ -616,7 +827,9 @@ async function main() {]).trim(),]))- state.imageProof = proveImages(manifest, observed, head, profile)+ state.imageProof = vacuity+ ? proveVacuityImages(manifest, observed, head, profile)+ : proveImages(manifest, observed, head, profile)save()const host = get('mcp-host', 'host', 'chatllm')const hosts = JSON.parse(kc(['-n', 'mcp-host', 'get', 'hosts', '-o', 'json']))@@ -772,7 +985,7 @@ async function main() {value: 'CONTEXT_MAPPER_HOST_IMAGE',},{ op: 'test', path: state.hcc.path, value: state.hcc.image },- { op: 'replace', path: state.hcc.path, value: fixtureImage },+ { op: 'replace', path: state.hcc.path, value: state.fixtureImage },])kc(['-n',@@ -782,7 +995,7 @@ async function main() {'deployment/host-context-controller','--timeout=120s',])- await waitHostImage(fixtureImage)+ await waitHostImage(state.fixtureImage)const wire = JSON.parse(kc(['-n',@@ -811,6 +1024,26 @@ async function main() {QA_RECORDER_IMAGE_MODEL_UNSUPPORTED: 'glm-5.3',QA_RECORDER_IMAGE_MODEL_UNKNOWN: models[2].model,QA_RECORDER_ROOT: evidence,+ ...(runtime.legacyLease+ ? {+ // The GFS fixture helpers name the cluster through this variable.+ E2E_K8S_CONTEXT: profile,+ LEGACY_LEASE_LANE_MODE: runtime.legacyLease,+ LEGACY_LEASE_SEED_LABEL_VALUE: state.runId,+ }+ : {}),+ }+ if (runtime.legacyLease) {+ // The spec scales these to zero; restoration needs the values it found.+ const replicas = {+ host: deploymentReplicas('mcp-host', 'chatllm'),+ hcc: deploymentReplicas('control-plane', 'host-context-controller'),+ }+ // Validates the recorded counts: a workload already at zero is refused+ // here, before the spec runs, instead of becoming the restore target.+ legacyLeaseReplicaPlan(replicas, replicas)+ state.legacyLeaseReplicas = replicas+ save()}command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)@@ -826,7 +1059,7 @@ async function main() {['scripts/minikube/run-with-deadline.mjs','--timeout-seconds',- '900',+ String(runtime.timeoutSeconds),'--heartbeat-seconds','20','--kill-grace-seconds',@@ -847,7 +1080,7 @@ async function main() {...environment,PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),},- timeout: 930_000,+ timeout: (runtime.timeoutSeconds + 30) * 1000,})report(output)@@ -875,7 +1108,9 @@ async function main() {try {const current = get('mcp-host', 'deployment', 'chatllm')if (- current.spec.template.spec.containers.some(container => container.image === fixtureImage)+ current.spec.template.spec.containers.some(+ container => container.image === state.fixtureImage+ )) {const logs = kc(['-n', 'mcp-host', 'logs', 'deployment/chatllm', '--tail=60'])state.startupDiagnostic = [@@ -902,6 +1137,32 @@ async function main() {)}uninstallSignalRestore()+ if (!failure && state.legacyLease) {+ // The spec removes its GFS fixtures; a live row left behind fails the run.+ try {+ const live = kc([+ '-n',+ 'control-plane',+ 'exec',+ 'deployment/control-postgres',+ '--',+ 'psql',+ '-v',+ 'ON_ERROR_STOP=1',+ '-U',+ 'postgres',+ '-d',+ 'profiles',+ '-t',+ '-A',+ '-c',+ "SELECT count(*) FROM gfs_resources WHERE drive = 'main' AND deleted_at IS NULL AND path_cache LIKE '/e2e-gfs-legacy-lease-%';",+ ]).trim()+ if (live !== '0') failure = new Error(`Legacy-lease GFS fixture rows remain (${live})`)+ } catch (error) {+ failure = new Error(`Legacy-lease GFS fixture check failed: ${error.message}`)+ }+ }if (failure) throw failureprocess.stdout.write(fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjsindex 13ae290e9..0e81c46f5 100644--- a/scripts/e2e/image-capabilities-fixture.test.mjs+++ b/scripts/e2e/image-capabilities-fixture.test.mjs@@ -8,10 +8,15 @@ import {fixtureImage,fixturePassLine,installSignalRestore,+ laneRuntime,+ legacyLeaseReplicaPlan,+ legacyLeaseVacuity,+ legacyLeaseVacuityManifestPath,modelInputs,playwrightLaneConfig,playwrightVerdict,proveImages,+ proveVacuityImages,requireOwnedResource,runAnnotation,sanitizeFixtureReport,@@ -352,3 +357,190 @@ test('the playwright log redacts the admin password, the session cookie and bearassert.ok(!report.includes('opaque-bearer-value'))assert.match(report, /3 passed \(41\.2s\)/)})++// ---------------------------------------------------------------------------+// Issue #1022: legacy processing-lease restart lanes.+// ---------------------------------------------------------------------------++test('the legacy-lease lanes share one config and select disjoint journeys', () => {+ const fixed = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart' })+ const vacuity = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart-vacuity' })+ assert.equal(+ fixed.config,+ 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts'+ )+ assert.equal(vacuity.config, fixed.config)+ assert.equal(fixed.grep, 'legacy-lease-restart fixture: ')+ assert.equal(vacuity.grep, 'legacy-lease-restart vacuity: ')+ assert.equal(vacuity.grep.startsWith(fixed.grep), false)+ assert.equal(fixed.grep.startsWith(vacuity.grep), false)+ const options = {+ lane: 'legacy-lease-restart',+ configFile: laneConfigFile('legacy-lease-restart'),+ }+ const scenarioTitles = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'].map(+ scenario => `legacy-lease-restart fixture: ${scenario}`+ )+ // Witness: the lane's own five scenario titles are PASS.+ assert.equal(+ playwrightVerdict(+ playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles }),+ options+ ),+ 'PASS'+ )+ // A clean report that lost scenarios is not the lane.+ assert.throws(+ () =>+ playwrightVerdict(+ playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles.slice(0, 2) }),+ options+ ),+ /lists 2 legacy-lease-restart specs, expected 5/+ )+ // The vacuity lane is exactly its one S-crash journey.+ assert.equal(+ playwrightVerdict(+ playwrightReport({+ lane: 'legacy-lease-restart-vacuity',+ titles: ['legacy-lease-restart vacuity: S-crash'],+ }),+ {+ lane: 'legacy-lease-restart-vacuity',+ configFile: laneConfigFile('legacy-lease-restart-vacuity'),+ }+ ),+ 'PASS'+ )+ // A vacuity title in the fixed lane's report is outside that lane.+ assert.throws(+ () =>+ playwrightVerdict(+ playwrightReport({+ lane: 'legacy-lease-restart',+ titles: ['legacy-lease-restart vacuity: S-crash'],+ }),+ options+ ),+ /outside the legacy-lease-restart lane/+ )+})++test('each lane runtime names its Host image and deadline; existing lanes are unchanged', () => {+ for (const lane of ['image', 'document-upload']) {+ assert.deepEqual(laneRuntime(lane), { fixtureImage, timeoutSeconds: 900, legacyLease: null })+ }+ assert.deepEqual(laneRuntime('legacy-lease-restart'), {+ fixtureImage,+ timeoutSeconds: 3600,+ legacyLease: 'fixed',+ })+ assert.deepEqual(laneRuntime('legacy-lease-restart-vacuity'), {+ fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',+ timeoutSeconds: 900,+ legacyLease: 'vacuity',+ })+ // The vacuity images never reuse a HEAD tag.+ assert.notEqual(legacyLeaseVacuity.fixtureImage, fixtureImage)+ assert.notEqual(legacyLeaseVacuity.baseImage, baseImage)+ for (const lane of ['typo', '__proto__', '']) {+ assert.throws(() => laneRuntime(lane), /Unknown lane/, lane)+ }+})++test('the vacuity manifest lives under the canonical run root, per profile', () => {+ assert.equal(+ legacyLeaseVacuityManifestPath('/repo/evenfire', profile),+ `/repo/evenfire/.local-notes/infra/runs/legacy-lease-vacuity/${profile}/manifest.json`+ )+})++const vacuityIds = {+ [legacyLeaseVacuity.baseImage]: `sha256:${'e'.repeat(64)}`,+ [legacyLeaseVacuity.fixtureImage]: `sha256:${'f'.repeat(64)}`,+}+function vacuityManifest() {+ return {+ kind: 'legacy-lease-vacuity',+ profile,+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',+ fixtureLayerRevision: head,+ images: { ...vacuityIds },+ derivedFrom: {+ [legacyLeaseVacuity.fixtureImage]: {+ ref: legacyLeaseVacuity.baseImage,+ id: vacuityIds[legacyLeaseVacuity.baseImage],+ },+ },+ }+}++test('vacuity images must come from the pre-fix base and this HEAD, for this profile', () => {+ // Witness: the well-formed manifest is proven.+ assert.deepEqual(proveVacuityImages(vacuityManifest(), vacuityIds, head, profile), {+ head,+ profile,+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',+ images: vacuityIds,+ })+ for (const [label, mutate, pattern] of [+ ['other kind', value => (value.kind = 'image-manifest'), /ownership/],+ ['other profile', value => (value.profile = 'clerum-other-12345678'), /ownership/],+ ['built from HEAD', value => (value.baseRevision = head), /pre-fix base/],+ ['stale fixture layer', value => (value.fixtureLayerRevision = 'd'.repeat(40)), /this HEAD/],+ [+ 'other base id',+ value => (value.images[legacyLeaseVacuity.baseImage] = `sha256:${'1'.repeat(64)}`),+ /identity/,+ ],+ [+ 'fixture from another base',+ value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].id = `sha256:${'2'.repeat(64)}`),+ /derived-base/,+ ],+ [+ 'fixture from the HEAD Host',+ value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].ref = baseImage),+ /derived-base/,+ ],+ ]) {+ const changed = vacuityManifest()+ mutate(changed)+ assert.throws(() => proveVacuityImages(changed, vacuityIds, head, profile), pattern, label)+ }+ // The node holds a different image than the manifest recorded.+ assert.throws(+ () =>+ proveVacuityImages(+ vacuityManifest(),+ { ...vacuityIds, [legacyLeaseVacuity.fixtureImage]: `sha256:${'3'.repeat(64)}` },+ head,+ profile+ ),+ /identity/+ )+ // A HEAD image manifest is never accepted as a vacuity proof.+ assert.throws(() => proveVacuityImages(manifest(), ids, head, profile), /ownership/)+})++test('restoration scales the Host before HCC and never accepts zero as restored', () => {+ const recorded = { host: 1, hcc: 1 }+ // Witness: both at zero yields a plan, Host first.+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 0, hcc: 0 }), [+ { key: 'host', replicas: 1 },+ { key: 'hcc', replicas: 1 },+ ])+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 0 }), [+ { key: 'hcc', replicas: 1 },+ ])+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 1 }), [])+ for (const [label, bad] of [+ ['zero host', { host: 0, hcc: 1 }],+ ['zero hcc', { host: 1, hcc: 0 }],+ ['missing', {}],+ ['fractional', { host: 1.5, hcc: 1 }],+ ]) {+ assert.throws(() => legacyLeaseReplicaPlan(bad, { host: 1, hcc: 1 }), /Recorded/, label)+ }+ assert.throws(() => legacyLeaseReplicaPlan(recorded, { host: undefined, hcc: 1 }), /unreadable/)+})diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.mdindex b3a207519..624c1216b 100644--- a/tests/e2e/fixtures/image-capabilities/README.md+++ b/tests/e2e/fixtures/image-capabilities/README.md@@ -55,6 +55,39 @@ file chip, sends, and requires the digest of the attached bytes in the answer,one `document-read-requested` row, one `document-answer` row carrying thatdigest, no image row and no `documentFailures`.+## Legacy processing-lease restart journey (issue #1022)++`make minikube-run-legacy-lease-restart` is the same runner with+`IMAGE_CAPABILITIES_LANE=legacy-lease-restart`; the config is+`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts` and+the cluster side lives in `helpers/legacyLeaseRestart.ts`. The user message+carries one line:++```+LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>+```++The fixture then scripts the turns from the message list alone: it asks for+`shell_exec` with a command whose output is `<marker>-shell-ok` (a token absent+from the command), then for `clerum__gfs_download` of the named resource once+the shell result echoes the token, and answers+`LEGACY-LEASE-FIXTURE-OK marker=… sha256=… bytes=…` from the download receipt.+A shell or download result without what the next step needs is answered with+`LEGACY-LEASE-FIXTURE-SHELL-FAILED code=<code>` or+`LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=<code>`, where the code is a GFS+download store error code (for example `download_busy`) or `unrecognized`.+`download_busy` is what the pre-fix Host returns for a legacy lease. The ledger+rows are `legacy-lease-shell-requested`, `legacy-lease-download-requested`,+`legacy-lease-answer` (with `downloadSha256` and `downloadBytes`),+`legacy-lease-shell-failed` and `legacy-lease-download-failed` (with+`failureCode`). It never records command output.++The vacuity lane, `make minikube-run-legacy-lease-restart-vacuity`, runs the same+fixture layer on a Host built from the pre-fix base by+`make minikube-build-legacy-lease-vacuity-fixture`. The procedure and the+signals checked after each scenario are in+`docs/testing/minikube-t2-runbook.md` (Legacy processing-lease restart lane).+## Local executionUse Node 24 and Bash 5, a clean development branch based on current `origin/dev`,diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjsindex 85bf71b60..1ecd30441 100644--- a/tests/e2e/fixtures/image-capabilities/provider.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs@@ -160,6 +160,12 @@ export const FIXTURE_REJECTION_REASONS = Object.freeze(['document-tool-output-unreadable','document-page-range-malformed','document-stream-unsupported',+ 'legacy-lease-shell-not-offered',+ 'legacy-lease-download-not-offered',+ 'legacy-lease-tool-result-unexpected',+ 'legacy-lease-tool-result-count',+ 'legacy-lease-download-result-malformed',+ 'legacy-lease-stream-unsupported',])/** A refusal with a stable machine-readable reason code for the evidence tally. */@@ -512,6 +518,187 @@ function classifyDocumentTurn(body) {}}+/*+ * Legacy processing-lease restart journey (issue #1022, `make+ * minikube-run-legacy-lease-restart`). The user message carries one line+ *+ * LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>+ *+ * and the fixture scripts two tool calls from the message list alone:+ *+ * 1. no tool result yet: call `shell_exec` with a command whose output is+ * `<marker>-shell-ok`. That token does not appear in the command, so only+ * a command that really ran can echo it back;+ * 2. one shell result carrying that token: call `clerum__gfs_download` for the+ * named resource;+ * 3. a shell result and a download result: answer with the sha256 and byte+ * count the Host's download receipt reports.+ *+ * A shell or download result that does not carry what step 2 or 3 needs is+ * answered with a failure line naming a closed-set code (a store error code such+ * as `download_busy`, or `unrecognized`), so the journey fails on screen at the+ * step that broke. The ledger records only the step, the code, and for the+ * answer the receipt's digest and byte count, never the command output.+ */+const SHELL_TOOL = 'shell_exec'+const GFS_DOWNLOAD_TOOL = 'clerum__gfs_download'+const LEGACY_LEASE_JOURNEY_LINE =+ /LEGACY_LEASE_JOURNEY marker=(legacy-lease-[a-f0-9]{16}) drive=([a-z0-9][a-z0-9-]{0,62}) resourceId=([a-f0-9]{32})/+const LEGACY_LEASE_SHELL_TOKEN_SUFFIX = '-shell-ok'+export const LEGACY_LEASE_ANSWER_PREFIX = 'LEGACY-LEASE-FIXTURE-OK'+export const LEGACY_LEASE_SHELL_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-SHELL-FAILED'+export const LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED'++/**+ * Codes a failed shell or download step may report: the GFS download store's+ * error codes (`GfsDownloadStoreErrorCode`), plus `unrecognized` for a result+ * that names none of them. A closed set of fixed strings, never tool output.+ */+export const LEGACY_LEASE_FAILURE_CODES = Object.freeze([+ 'caller_mismatch',+ 'caller_quota_exceeded',+ 'corrupt_store_ledger',+ 'download_expired',+ 'download_missing',+ 'download_busy',+ 'host_quota_exceeded',+ 'publication_cancelled',+ 'storage_write_failed',+ 'unsupported_store_schema',+ 'workspace_unavailable',+ 'writer_locked',+ 'unrecognized',+])++/** The store error text (`GfsDownloadStoreError`): `GFS download store failed (<code>)`. */+const STORE_FAILURE = /GFS download store failed \(([a-z_]+)\)/++function toolOutputPattern(name) {+ return new RegExp(+ `^<tool_output name="${name}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`+ )+}+const SHELL_TOOL_OUTPUT = toolOutputPattern(SHELL_TOOL)+const GFS_DOWNLOAD_TOOL_OUTPUT = toolOutputPattern(GFS_DOWNLOAD_TOOL)++class FixtureLegacyLeaseError extends Error {+ constructor(code, message) {+ super(message)+ this.name = 'FixtureLegacyLeaseError'+ this.code = code+ }+}++function failureCode(content) {+ const code = STORE_FAILURE.exec(content)?.[1]+ return code && LEGACY_LEASE_FAILURE_CODES.includes(code) ? code : 'unrecognized'+}++function offeredTool(body, name) {+ const tools = Array.isArray(body.tools) ? body.tools : []+ return tools.some(tool => tool?.function?.name === name)+}++/**+ * Classifies a legacy-lease journey turn.+ *+ * - `null`: the last user message carries no journey line, or no tool is offered+ * (a title or summary request), so the request is not part of the journey.+ * - `{ stage: 'shell', marker }`: ask for the shell command.+ * - `{ stage: 'download', drive, resourceId }`: the shell echoed the token.+ * - `{ stage: 'shell-failed', code }`: the shell result lacks the token.+ * - `{ stage: 'answer', marker, sha256, sizeBytes }`: the download receipt.+ * - `{ stage: 'download-failed', code }`: the download result is an error.+ */+function classifyLegacyLeaseTurn(body) {+ if (!Array.isArray(body.tools) || body.tools.length === 0) return null+ const userIndex = lastUserIndex(body.messages)+ if (userIndex < 0) return null+ const user = body.messages[userIndex]+ if (typeof user.content !== 'string') return null+ const journey = LEGACY_LEASE_JOURNEY_LINE.exec(user.content)+ if (!journey) return null+ const [, marker, drive, resourceId] = journey++ const toolMessages = body.messages+ .slice(userIndex + 1)+ .filter(message => message?.role === 'tool')+ if (toolMessages.length > 2)+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-tool-result-count',+ `the journey makes two tool calls, received ${toolMessages.length} results`+ )+ const contents = toolMessages.map(message => message.content)+ if (contents.some(content => typeof content !== 'string'))+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-tool-result-unexpected',+ 'a tool result is not a string'+ )++ if (contents.length === 0) {+ if (!offeredTool(body, SHELL_TOOL))+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-shell-not-offered',+ 'the Host did not offer shell_exec for the journey'+ )+ return { stage: 'shell', marker }+ }++ const shell = SHELL_TOOL_OUTPUT.exec(contents[0])+ if (!shell)+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-tool-result-unexpected',+ 'the first tool result is not a shell_exec result'+ )+ const token = `${marker}${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}`+ const echoed = shell[1].split('\n').some(line => line.trim() === token)+ if (contents.length === 1) {+ if (!echoed) return { stage: 'shell-failed', code: failureCode(shell[1]) }+ if (!offeredTool(body, GFS_DOWNLOAD_TOOL))+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-download-not-offered',+ 'the Host did not offer clerum__gfs_download after the shell step'+ )+ return { stage: 'download', drive, resourceId }+ }+ if (!echoed)+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-tool-result-unexpected',+ 'a download result follows a shell result without the token'+ )++ const download = GFS_DOWNLOAD_TOOL_OUTPUT.exec(contents[1])+ if (!download)+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-tool-result-unexpected',+ 'the second tool result is not a clerum__gfs_download result'+ )+ let receipt+ try {+ receipt = JSON.parse(download[1])+ } catch {+ return { stage: 'download-failed', code: failureCode(download[1]) }+ }+ if (+ !receipt ||+ typeof receipt !== 'object' ||+ receipt.delivery !== 'workspace_file' ||+ typeof receipt.sha256 !== 'string' ||+ !SHA256_HEX.test(receipt.sha256) ||+ !isByteCount(receipt.sizeBytes)+ )+ throw new FixtureLegacyLeaseError(+ 'legacy-lease-download-result-malformed',+ 'the download result is not a workspace_file receipt with sha256 and sizeBytes'+ )+ return {+ stage: 'answer',+ marker,+ sha256: receipt.sha256.toLowerCase(),+ sizeBytes: receipt.sizeBytes,+ }+}+const isByteCount = value => Number.isSafeInteger(value) && value >= 0const INTERNAL_HOST_SUFFIXES = [@@ -758,6 +945,10 @@ export function createImageFixtureFetch(documentReadRequests: 0,documentAnswers: 0,documentFailures: 0,+ legacyLeaseShellRequests: 0,+ legacyLeaseDownloadRequests: 0,+ legacyLeaseAnswers: 0,+ legacyLeaseFailures: 0,},attempts: [],}@@ -911,6 +1102,74 @@ export function createImageFixtureFetch(})}++ let legacyTurn+ try {+ legacyTurn = classifyLegacyLeaseTurn(body)+ } catch (error) {+ if (!(error instanceof FixtureLegacyLeaseError)) throw error+ state.counters.legacyLeaseFailures += 1+ return refuse(model, error.code, { code: error.code })+ }+ if (legacyTurn) {+ if (stream) {+ // A tool call cannot be answered in this fixture's SSE, exactly as for+ // the document read.+ state.counters.legacyLeaseFailures += 1+ return refuse(model, 'legacy-lease-stream-unsupported', {+ code: 'legacy_lease_stream_unsupported',+ })+ }+ sequence += 1+ const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`+ const reply = (responseKind, content, fields = {}) => {+ recordCall(model, null, responseKind, fields)+ publish()+ return jsonResponse(completionBody(completionId, model, content))+ }+ const call = (responseKind, name, args) => {+ recordCall(model, null, responseKind)+ publish()+ return jsonResponse(toolCallCompletionBody(completionId, model, name, args))+ }+ switch (legacyTurn.stage) {+ case 'shell':+ state.counters.legacyLeaseShellRequests += 1+ return call('legacy-lease-shell-requested', SHELL_TOOL, {+ command: `printf '%s${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}\\n' ${legacyTurn.marker}`,+ })+ case 'download':+ state.counters.legacyLeaseDownloadRequests += 1+ return call('legacy-lease-download-requested', GFS_DOWNLOAD_TOOL, {+ drive: legacyTurn.drive,+ resourceId: legacyTurn.resourceId,+ })+ case 'shell-failed':+ state.counters.legacyLeaseFailures += 1+ return reply(+ 'legacy-lease-shell-failed',+ `${LEGACY_LEASE_SHELL_FAILED_PREFIX} code=${legacyTurn.code}`,+ { failureCode: legacyTurn.code }+ )+ case 'download-failed':+ state.counters.legacyLeaseFailures += 1+ return reply(+ 'legacy-lease-download-failed',+ `${LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX} code=${legacyTurn.code}`,+ { failureCode: legacyTurn.code }+ )+ case 'answer':+ state.counters.legacyLeaseAnswers += 1+ return reply(+ 'legacy-lease-answer',+ `${LEGACY_LEASE_ANSWER_PREFIX} marker=${legacyTurn.marker} ` ++ `sha256=${legacyTurn.sha256} bytes=${legacyTurn.sizeBytes}`,+ { downloadSha256: legacyTurn.sha256, downloadBytes: legacyTurn.sizeBytes }+ )+ default:+ throw new Error(`Unhandled legacy-lease stage ${String(legacyTurn.stage)}`)+ }+ }state.counters.textOnlyResponses += 1return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })}diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjsindex a527950d2..3beca6da5 100644--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs@@ -1091,6 +1091,12 @@ const EXPECTED_REJECTION_REASONS = ['image-png-malformed','image-png-unsupported-form','image-tile-grid-mismatch',+ 'legacy-lease-download-not-offered',+ 'legacy-lease-download-result-malformed',+ 'legacy-lease-shell-not-offered',+ 'legacy-lease-stream-unsupported',+ 'legacy-lease-tool-result-count',+ 'legacy-lease-tool-result-unexpected','provider-auth-mismatch','provider-body-invalid','provider-body-unsupported',@@ -1227,3 +1233,308 @@ test('every rejected row records its closed-set reason and no request content',assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)})++// ---------------------------------------------------------------------------+// Legacy processing-lease restart journey (issue #1022): the fixture scripts+// shell_exec, then clerum__gfs_download, then answers from the receipt.+// ---------------------------------------------------------------------------++const LEGACY_MARKER = 'legacy-lease-0123456789abcdef'+const LEGACY_RESOURCE = 'a'.repeat(32)+const LEGACY_SHA = 'b'.repeat(64)+const LEGACY_TOKEN = `${LEGACY_MARKER}-shell-ok`++const toolDefinition = name => ({+ type: 'function',+ function: { name, description: name, parameters: { type: 'object' } },+})+const LEGACY_TOOLS = [toolDefinition('shell_exec'), toolDefinition('clerum__gfs_download')]++function legacyUserMessage(marker = LEGACY_MARKER) {+ return {+ role: 'user',+ content:+ '<turn-context>\ndate: 2026-10-08\n</turn-context>\n\n' ++ `Run the journey. LEGACY_LEASE_JOURNEY marker=${marker} drive=main resourceId=${LEGACY_RESOURCE}`,+ }+}++function legacyMessages(results) {+ const messages = [legacyUserMessage()]+ for (const [index, [name, content]] of results.entries()) {+ messages.push({+ role: 'assistant',+ content: null,+ tool_calls: [{ id: `call-${index}`, type: 'function', function: { name, arguments: '{}' } }],+ })+ messages.push({ role: 'tool', tool_call_id: `call-${index}`, content })+ }+ return messages+}++const wrapped = (name, content) =>+ `<tool_output name="${name}" sanitized="false">\n${content}\n</tool_output>`+const shellResult = content => ['shell_exec', wrapped('shell_exec', content)]+const downloadResult = content => ['clerum__gfs_download', wrapped('clerum__gfs_download', content)]+const receipt = (overrides = {}) =>+ JSON.stringify({+ delivery: 'workspace_file',+ id: '11111111-2222-4333-8444-555555555555',+ path: '.gfs-downloads/input-1/report.pdf',+ sizeBytes: 42,+ sha256: LEGACY_SHA,+ ...overrides,+ })++test('legacy-lease step 1: asks for a shell command whose output token is not in the command', async () => {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS })+ )+ assert.equal(response.status, 200)+ const body = await response.json()+ assert.equal(body.choices[0].finish_reason, 'tool_calls')+ const [toolCall] = body.choices[0].message.tool_calls+ assert.equal(toolCall.function.name, 'shell_exec')+ const { command } = JSON.parse(toolCall.function.arguments)+ // The approval preview shows the marker, but only an executed command can+ // produce the joined token.+ assert.ok(command.includes(LEGACY_MARKER))+ assert.equal(command.includes(LEGACY_TOKEN), false)+ assert.equal(command, `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.legacyLeaseShellRequests, 1)+ assert.deepEqual(evidence.attempts, [+ { model: VISUAL_MODEL, imageSha256: null, responseKind: 'legacy-lease-shell-requested' },+ ])+})++test('legacy-lease step 2: a shell result with the token asks for the GFS download', async () => {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(`${LEGACY_TOKEN}\n`)]), {+ tools: LEGACY_TOOLS,+ })+ )+ const body = await response.json()+ const [toolCall] = body.choices[0].message.tool_calls+ assert.equal(toolCall.function.name, 'clerum__gfs_download')+ assert.deepEqual(JSON.parse(toolCall.function.arguments), {+ drive: 'main',+ resourceId: LEGACY_RESOURCE,+ })+ assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 1)+ assert.equal(h.getEvidence().attempts[0].responseKind, 'legacy-lease-download-requested')+})++test('legacy-lease step 2: a lease failure before command start is answered with its store code', async () => {+ const h = harness()+ const response = await call(+ h,+ chatBody(+ VISUAL_MODEL,+ legacyMessages([+ shellResult(+ 'Processing lease failed before command start: GFS download store failed (download_busy)'+ ),+ ]),+ { tools: LEGACY_TOOLS }+ )+ )+ const body = await response.json()+ assert.equal(body.choices[0].finish_reason, 'stop')+ assert.equal(+ body.choices[0].message.content,+ 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy'+ )+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.legacyLeaseFailures, 1)+ assert.equal(evidence.counters.legacyLeaseDownloadRequests, 0)+ assert.deepEqual(evidence.attempts, [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'legacy-lease-shell-failed',+ failureCode: 'download_busy',+ },+ ])+})++test('legacy-lease step 2: output without the token or a known code is unrecognized, never a download', async () => {+ for (const content of [+ // The command echoed back is not the token.+ `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`,+ 'GFS download store failed (not_a_store_code)',+ `${LEGACY_TOKEN}-extra`,+ ]) {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(content)]), { tools: LEGACY_TOOLS })+ )+ const body = await response.json()+ assert.equal(+ body.choices[0].message.content,+ 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=unrecognized',+ content+ )+ assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 0, content)+ }+})++test('legacy-lease step 3: the answer carries the receipt sha256 and byte count', async () => {+ const h = harness()+ const response = await call(+ h,+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt())]), {+ tools: LEGACY_TOOLS,+ })+ )+ const body = await response.json()+ assert.equal(+ body.choices[0].message.content,+ `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${LEGACY_SHA} bytes=42`+ )+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.legacyLeaseAnswers, 1)+ assert.deepEqual(evidence.attempts, [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'legacy-lease-answer',+ downloadSha256: LEGACY_SHA,+ downloadBytes: 42,+ },+ ])+})++test('legacy-lease step 3: a different receipt gives a different answer (not a constant)', async () => {+ const other = 'c'.repeat(64)+ const h = harness()+ const response = await call(+ h,+ chatBody(+ VISUAL_MODEL,+ legacyMessages([+ shellResult(LEGACY_TOKEN),+ downloadResult(receipt({ sha256: other, sizeBytes: 7 })),+ ]),+ { tools: LEGACY_TOOLS }+ )+ )+ const body = await response.json()+ assert.equal(+ body.choices[0].message.content,+ `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${other} bytes=7`+ )+})++test('legacy-lease step 3: a download error is answered with its store code', async () => {+ const h = harness()+ const response = await call(+ h,+ chatBody(+ VISUAL_MODEL,+ legacyMessages([+ shellResult(LEGACY_TOKEN),+ downloadResult('Error: GFS download store failed (workspace_unavailable)'),+ ]),+ { tools: LEGACY_TOOLS }+ )+ )+ const body = await response.json()+ assert.equal(+ body.choices[0].message.content,+ 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=workspace_unavailable'+ )+ assert.deepEqual(h.getEvidence().attempts, [+ {+ model: VISUAL_MODEL,+ imageSha256: null,+ responseKind: 'legacy-lease-download-failed',+ failureCode: 'workspace_unavailable',+ },+ ])+})++test('legacy-lease turns the fixture cannot use are refused with a closed-set reason', async () => {+ const cases = [+ [+ 'shell not offered',+ chatBody(VISUAL_MODEL, [legacyUserMessage()], {+ tools: [toolDefinition('clerum__gfs_download')],+ }),+ 'legacy-lease-shell-not-offered',+ ],+ [+ 'download not offered',+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN)]), {+ tools: [toolDefinition('shell_exec')],+ }),+ 'legacy-lease-download-not-offered',+ ],+ [+ 'first result from another tool',+ chatBody(VISUAL_MODEL, legacyMessages([downloadResult(receipt())]), { tools: LEGACY_TOOLS }),+ 'legacy-lease-tool-result-unexpected',+ ],+ [+ 'three results',+ chatBody(+ VISUAL_MODEL,+ legacyMessages([+ shellResult(LEGACY_TOKEN),+ downloadResult(receipt()),+ downloadResult(receipt()),+ ]),+ { tools: LEGACY_TOOLS }+ ),+ 'legacy-lease-tool-result-count',+ ],+ [+ 'receipt without sha256',+ chatBody(+ VISUAL_MODEL,+ legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt({ sha256: 'x' }))]),+ { tools: LEGACY_TOOLS }+ ),+ 'legacy-lease-download-result-malformed',+ ],+ [+ 'streamed turn',+ chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS, stream: true }),+ 'legacy-lease-stream-unsupported',+ ],+ ]+ for (const [label, body, reason] of cases) {+ const h = harness()+ const response = await call(h, body)+ assert.ok(response.status >= 400, label)+ const evidence = h.getEvidence()+ assert.equal(evidence.counters.legacyLeaseFailures, 1, label)+ assert.deepEqual(+ evidence.attempts,+ [{ model: VISUAL_MODEL, imageSha256: null, responseKind: 'rejected', reason }],+ label+ )+ }+})++test('legacy-lease: a journey line without offered tools is an ordinary text-only request', async () => {+ const h = harness()+ const response = await call(h, chatBody(VISUAL_MODEL, [legacyUserMessage()]))+ const body = await response.json()+ assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)+ // Witness: the request reached the fixture and was answered as text.+ assert.equal(h.getEvidence().counters.textOnlyResponses, 1)+ assert.equal(h.getEvidence().counters.legacyLeaseShellRequests, 0)+})++test('legacy-lease failure codes are a frozen closed set that includes download_busy', () => {+ const codes = providerModule.LEGACY_LEASE_FAILURE_CODES+ assert.equal(Object.isFrozen(codes), true)+ assert.ok(codes.includes('download_busy'))+ assert.ok(codes.includes('unrecognized'))+})diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.jsonindex 86c865e52..5153d8811 100644--- a/tests/e2e/fixtures/package-lock.json+++ b/tests/e2e/fixtures/package-lock.json@@ -1,12 +1,12 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.107",+ "version": "1.0.108","lockfileVersion": 3,"requires": true,"packages": {"": {"name": "@clerum/e2e-test-fixtures",- "version": "1.0.107",+ "version": "1.0.108","dependencies": {"@kubernetes/client-node": "^1.0.0"},diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.jsonindex 18d0e2843..a1819b160 100644--- a/tests/e2e/fixtures/package.json+++ b/tests/e2e/fixtures/package.json@@ -1,6 +1,6 @@{"name": "@clerum/e2e-test-fixtures",- "version": "1.0.107",+ "version": "1.0.108","description": "E2E test fixtures and utilities for Clerum testing","type": "module","main": "index.ts",