Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

test(e2e): add the minikube legacy processing lease restart lane

Alfredo Lopeza06a2adc4139
diff --git a/Makefile b/Makefile
index b74d68007..46dbb3fcc 100644
--- a/Makefile
+++ b/Makefile
@@ -321,6 +321,25 @@ minikube-run-document-upload: ## Run the visible Desktop document-upload journey
IMAGE_CAPABILITIES_LANE=document-upload \
bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
+.PHONY: minikube-run-legacy-lease-restart minikube-run-legacy-lease-restart-vacuity
+minikube-run-legacy-lease-restart: ## Run the legacy processing-lease restart scenarios (issue #1022) with the Desktop shell+GFS download journey and restore the Host
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ IMAGE_CAPABILITIES_LANE=legacy-lease-restart \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
+
+minikube-run-legacy-lease-restart-vacuity: ## Prove the legacy-lease lane fails on the pre-fix Host (base 74e0d81d9): S-crash must stop at the shell with download_busy
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ IMAGE_CAPABILITIES_LANE=legacy-lease-restart-vacuity \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
+
+.PHONY: minikube-build-legacy-lease-vacuity-fixture
+minikube-build-legacy-lease-vacuity-fixture: ## Build the pre-fix Host (base 74e0d81d9) and its provider-fixture layer for the legacy-lease vacuity lane under the owned profile lease
+ @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
+ T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
+ bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-legacy-lease-vacuity-image.sh
+
minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)
@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }
@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
index d70615959..23dec419f 100644
--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
@@ -57,6 +57,14 @@ export const FIXTURE_RESPONSE_KIND = {
// delivered.
documentReadRequested: 'document-read-requested',
documentAnswer: 'document-answer',
+ // Issue #1022: the legacy processing-lease restart journey. shell_exec first,
+ // then clerum__gfs_download, then an answer built from the download receipt;
+ // a failed step is answered with the store code it reported.
+ legacyLeaseShellRequested: 'legacy-lease-shell-requested',
+ legacyLeaseDownloadRequested: 'legacy-lease-download-requested',
+ legacyLeaseAnswer: 'legacy-lease-answer',
+ legacyLeaseShellFailed: 'legacy-lease-shell-failed',
+ legacyLeaseDownloadFailed: 'legacy-lease-download-failed',
} as const
/**
@@ -85,6 +93,33 @@ export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze([
'document-tool-output-unreadable',
'document-page-range-malformed',
'document-stream-unsupported',
+ 'legacy-lease-shell-not-offered',
+ 'legacy-lease-download-not-offered',
+ 'legacy-lease-tool-result-unexpected',
+ 'legacy-lease-tool-result-count',
+ 'legacy-lease-download-result-malformed',
+ 'legacy-lease-stream-unsupported',
+])
+
+/**
+ * Store failure codes a legacy-lease failure row may carry (the
+ * `GfsDownloadStoreError` codes plus `unrecognized`). Declared independently of
+ * the fixture; `test/imageCapabilityEvidence.test.ts` checks the lists agree.
+ */
+export const FIXTURE_LEGACY_LEASE_FAILURE_CODES: readonly string[] = Object.freeze([
+ 'caller_mismatch',
+ 'caller_quota_exceeded',
+ 'corrupt_store_ledger',
+ 'download_expired',
+ 'download_missing',
+ 'download_busy',
+ 'host_quota_exceeded',
+ 'publication_cancelled',
+ 'storage_write_failed',
+ 'unsupported_store_schema',
+ 'workspace_unavailable',
+ 'writer_locked',
+ 'unrecognized',
])
/** The fixture's text-only answer. It names no image and no color. */
@@ -312,6 +347,12 @@ export interface FixtureProviderAttempt {
truncated?: boolean
/** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */
reason?: string
+ /** One of {@link FIXTURE_LEGACY_LEASE_FAILURE_CODES}. Legacy-lease failure rows only. */
+ failureCode?: string
+ /** sha256 the download receipt reported. Legacy-lease answer row only. */
+ downloadSha256?: string
+ /** Byte count the download receipt reported. Legacy-lease answer row only. */
+ downloadBytes?: number
}
export interface FixtureEvidenceCounters {
@@ -326,6 +367,10 @@ export interface FixtureEvidenceCounters {
documentReadRequests: number
documentAnswers: number
documentFailures: number
+ legacyLeaseShellRequests: number
+ legacyLeaseDownloadRequests: number
+ legacyLeaseAnswers: number
+ legacyLeaseFailures: number
}
export interface ImageCapabilityEvidenceSnapshot {
@@ -352,6 +397,10 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = [
'documentReadRequests',
'documentAnswers',
'documentFailures',
+ 'legacyLeaseShellRequests',
+ 'legacyLeaseDownloadRequests',
+ 'legacyLeaseAnswers',
+ 'legacyLeaseFailures',
]
function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {
@@ -434,6 +483,53 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
)
}
+ // failureCode is present on every legacy-lease failure row and on no other row.
+ const legacyFailure =
+ responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed ||
+ responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadFailed
+ if (legacyFailure) {
+ const failureCode = value.failureCode
+ if (
+ typeof failureCode !== 'string' ||
+ !FIXTURE_LEGACY_LEASE_FAILURE_CODES.includes(failureCode)
+ ) {
+ throw new Error(
+ `${source}: attempts[${index}].failureCode must be one of ` +
+ `${FIXTURE_LEGACY_LEASE_FAILURE_CODES.join(', ')} on a ${responseKind} row`
+ )
+ }
+ row.failureCode = failureCode
+ } else if ('failureCode' in value) {
+ throw new Error(
+ `${source}: attempts[${index}].failureCode is only valid on a legacy-lease failure row`
+ )
+ }
+
+ // downloadSha256 and downloadBytes are present on every legacy-lease answer row only.
+ if (responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer) {
+ const downloadSha256 = value.downloadSha256
+ const downloadBytes = value.downloadBytes
+ if (typeof downloadSha256 !== 'string' || !SHA256_HEX.test(downloadSha256)) {
+ throw new Error(
+ `${source}: attempts[${index}].downloadSha256 must be a sha256 hex digest on a ` +
+ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
+ )
+ }
+ if (!isByteCount(downloadBytes)) {
+ throw new Error(
+ `${source}: attempts[${index}].downloadBytes must be a non-negative integer on a ` +
+ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
+ )
+ }
+ row.downloadSha256 = downloadSha256
+ row.downloadBytes = downloadBytes
+ } else if ('downloadSha256' in value || 'downloadBytes' in value) {
+ throw new Error(
+ `${source}: attempts[${index}].downloadSha256 and downloadBytes are only valid on a ` +
+ `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
+ )
+ }
+
// Present only on document rows; an image row must not grow the field.
if ('documentSha256' in value) {
const documentSha256 = value.documentSha256
@@ -623,7 +719,10 @@ export function appendedAttempts(
current.reason !== row.reason ||
current.byteRange?.offset !== row.byteRange?.offset ||
current.byteRange?.length !== row.byteRange?.length ||
- current.truncated !== row.truncated
+ current.truncated !== row.truncated ||
+ current.failureCode !== row.failureCode ||
+ current.downloadSha256 !== row.downloadSha256 ||
+ current.downloadBytes !== row.downloadBytes
) {
throw new Error(
`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`
diff --git a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
new file mode 100644
index 000000000..24f3df659
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
@@ -0,0 +1,892 @@
+// desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
+//
+// E2E_GUARDIAN_IPC_FLOW: this module drives no browser transition. It owns the
+// cluster side of the legacy processing-lease restart lane (issue #1022): the
+// workload stop/start scenarios, the seed of one legacy lease on the Host PVC,
+// and read-only readers for the three signals the spec asserts after each
+// scenario. The Desktop journey itself lives in the spec.
+//
+// Every kubectl call names the validated branch profile as its context and is
+// bounded by a timeout. Arguments travel as a vector, never through a shell.
+// Waits poll a live condition with a deadline; none of them is a fixed sleep.
+import { execFileSync } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
+
+const PROFILE_PATTERN = /^clerum-.+-[a-f0-9]{8}$/
+const RUN_ID_PATTERN = /^image-capabilities-[a-f0-9]{12}$/
+const SHA256_HEX = /^[a-f0-9]{64}$/
+
+export const LEGACY_LEASE_LANE_MODES = ['fixed', 'vacuity'] as const
+export type LegacyLeaseLaneMode = (typeof LEGACY_LEASE_LANE_MODES)[number]
+
+/** The Host whose PVC carries the GFS download store (the fixture lane's Host). */
+export const HOST = {
+ namespace: 'mcp-host',
+ deployment: 'chatllm',
+ container: 'mcp-host',
+ selector: 'app=chatllm',
+} as const
+export const HCC = {
+ namespace: 'control-plane',
+ deployment: 'host-context-controller',
+ selector: 'app=host-context-controller',
+} as const
+export const WRC = {
+ namespace: 'control-plane',
+ deployment: 'workflow-recipes',
+ selector: 'app=workflow-recipes',
+} as const
+export const GFS_WORKLOADS = [
+ {
+ namespace: 'gfs',
+ deployment: 'gfsc-writer',
+ selector: 'app=gfs-controller,clerum.io/gfsc-role=writer',
+ },
+ {
+ namespace: 'gfs',
+ deployment: 'gfsc-reader',
+ selector: 'app=gfs-controller,clerum.io/gfsc-role=reader',
+ },
+] as const
+
+/** The store ledger on the Host PVC, mounted at /workspace. */
+export const STORE_LEDGER_PATH = '/workspace/.gfs-download-store/ledger-v1.json'
+export const LEGACY_DISCARD_COUNTER = 'clerum_gfs_legacy_processing_leases_discarded_total'
+const STORE_LOG_COMPONENT = 'GfsDownloadStore'
+const DISCARD_MSG_PREFIX = 'GFS download store discarded '
+const UNKNOWN_MSG_PREFIX = 'GFS download store could not confirm the discard of '
+
+export const SEED_LABEL_KEY = 'evenfire.ai/legacy-lease-seed'
+
+export interface LegacyLeaseLaneEnv {
+ profile: string
+ mode: LegacyLeaseLaneMode
+ runId: string
+}
+
+/** Reads and validates the runner-owned bindings of this lane. */
+export function requireLegacyLeaseLaneEnv(
+ env: NodeJS.ProcessEnv = process.env
+): LegacyLeaseLaneEnv {
+ const problems: string[] = []
+ const profile = (env.MINIKUBE_PROFILE ?? '').trim()
+ if (!PROFILE_PATTERN.test(profile))
+ problems.push(`MINIKUBE_PROFILE is not a branch profile ("${profile}")`)
+ if ((env.CONTROL_API_REAL_PG_CONTEXT ?? '').trim() !== profile)
+ problems.push('CONTROL_API_REAL_PG_CONTEXT must equal MINIKUBE_PROFILE')
+ if ((env.E2E_K8S_CONTEXT ?? '').trim() !== profile)
+ problems.push('E2E_K8S_CONTEXT must equal MINIKUBE_PROFILE (the GFS fixture helpers read it)')
+ const mode = (env.LEGACY_LEASE_LANE_MODE ?? '').trim()
+ if (!(LEGACY_LEASE_LANE_MODES as readonly string[]).includes(mode))
+ problems.push(`LEGACY_LEASE_LANE_MODE must be fixed or vacuity ("${mode}")`)
+ const runId = (env.LEGACY_LEASE_SEED_LABEL_VALUE ?? '').trim()
+ if (!RUN_ID_PATTERN.test(runId) || runId !== (env.IMAGE_CAPABILITIES_RUN_ID ?? '').trim())
+ problems.push('LEGACY_LEASE_SEED_LABEL_VALUE must equal IMAGE_CAPABILITIES_RUN_ID')
+ if (problems.length > 0) {
+ throw new Error(
+ 'legacy-lease-restart lane is not configured:\n' +
+ problems.map(problem => ` - ${problem}`).join('\n') +
+ '\nRun it through `make minikube-run-legacy-lease-restart` (or the -vacuity target).'
+ )
+ }
+ return { profile, mode: mode as LegacyLeaseLaneMode, runId }
+}
+
+export type Kubectl = (args: string[], options?: { timeoutMs?: number; input?: string }) => string
+
+export function kubectlFor(profile: string): Kubectl {
+ if (!PROFILE_PATTERN.test(profile)) throw new Error(`refusing kubectl for "${profile}"`)
+ return (args, options = {}) => {
+ const timeoutMs = options.timeoutMs ?? 30_000
+ try {
+ return execFileSync(
+ 'kubectl',
+ ['--context', profile, `--request-timeout=${Math.ceil(timeoutMs / 1000)}s`, ...args],
+ {
+ encoding: 'utf8',
+ timeout: timeoutMs + 5_000,
+ input: options.input,
+ maxBuffer: 16 * 1024 * 1024,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ }
+ )
+ } catch (error) {
+ const stderr =
+ typeof (error as { stderr?: unknown }).stderr === 'string'
+ ? (error as { stderr: string }).stderr.trim()
+ : ''
+ throw new Error(`kubectl ${args.join(' ')} failed: ${(error as Error).message}\n${stderr}`)
+ }
+ }
+}
+
+/** Polls `probe` until it returns a value, or fails with the last observation. */
+export async function waitFor<T>(
+ label: string,
+ timeoutMs: number,
+ probe: () => { done: true; value: T } | { done: false; observed: string }
+): Promise<T> {
+ const deadline = Date.now() + timeoutMs
+ let observed = 'not yet observed'
+ for (;;) {
+ const result = probe()
+ if (result.done) return result.value
+ observed = result.observed
+ if (Date.now() >= deadline)
+ throw new Error(`${label}: not reached within ${timeoutMs} ms (${observed})`)
+ await new Promise(resolve => setTimeout(resolve, 500))
+ }
+}
+
+export interface PodState {
+ name: string
+ uid: string
+ phase: string
+ terminating: boolean
+ ready: boolean
+ /** A container of this pod has started running at least once. */
+ started: boolean
+ image: string
+}
+
+export function readPods(kubectl: Kubectl, namespace: string, selector: string): PodState[] {
+ const list = JSON.parse(
+ kubectl(['-n', namespace, 'get', 'pod', '-l', selector, '-o', 'json'])
+ ) as {
+ items: Array<{
+ metadata: { name: string; uid: string; deletionTimestamp?: string }
+ spec: { containers: Array<{ image: string }> }
+ status: {
+ phase?: string
+ conditions?: Array<{ type: string; status: string }>
+ containerStatuses?: Array<{
+ started?: boolean
+ state?: { running?: unknown; terminated?: unknown }
+ lastState?: { running?: unknown; terminated?: unknown }
+ }>
+ }
+ }>
+ }
+ return list.items.map(item => ({
+ name: item.metadata.name,
+ uid: item.metadata.uid,
+ phase: item.status.phase ?? 'Unknown',
+ terminating: Boolean(item.metadata.deletionTimestamp),
+ ready: (item.status.conditions ?? []).some(c => c.type === 'Ready' && c.status === 'True'),
+ started: (item.status.containerStatuses ?? []).some(
+ c =>
+ c.started === true ||
+ Boolean(c.state?.running) ||
+ Boolean(c.state?.terminated) ||
+ Boolean(c.lastState?.running) ||
+ Boolean(c.lastState?.terminated)
+ ),
+ image: item.spec.containers[0]?.image ?? '',
+ }))
+}
+
+function describePods(pods: PodState[]): string {
+ return pods.length === 0
+ ? 'no pods'
+ : pods
+ .map(
+ p =>
+ `${p.name}(${p.uid.slice(0, 8)} ${p.phase}${p.terminating ? ' terminating' : ''}${p.ready ? ' ready' : ''})`
+ )
+ .join(', ')
+}
+
+export function deploymentReplicas(kubectl: Kubectl, namespace: string, name: string): number {
+ const replicas = JSON.parse(kubectl(['-n', namespace, 'get', 'deployment', name, '-o', 'json']))
+ .spec.replicas
+ if (!Number.isSafeInteger(replicas)) throw new Error(`${namespace}/${name} has no spec.replicas`)
+ return replicas as number
+}
+
+export function scale(kubectl: Kubectl, namespace: string, name: string, replicas: number): void {
+ kubectl(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])
+}
+
+export function rolloutStatus(kubectl: Kubectl, namespace: string, name: string): void {
+ kubectl(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=180s'], {
+ timeoutMs: 190_000,
+ })
+}
+
+export function rolloutRestart(kubectl: Kubectl, namespace: string, name: string): void {
+ kubectl(['-n', namespace, 'rollout', 'restart', `deployment/${name}`])
+}
+
+export async function waitNoPods(
+ kubectl: Kubectl,
+ namespace: string,
+ selector: string,
+ label: string
+): Promise<void> {
+ await waitFor(`${label} has no pods`, 120_000, () => {
+ const pods = readPods(kubectl, namespace, selector)
+ return pods.length === 0
+ ? { done: true, value: undefined }
+ : { done: false, observed: describePods(pods) }
+ })
+}
+
+/** Exactly one Ready, non-terminating pod whose UID is not in `previous`. */
+export async function waitSingleNewReadyPod(
+ kubectl: Kubectl,
+ namespace: string,
+ selector: string,
+ previous: ReadonlySet<string>,
+ label: string,
+ timeoutMs = 300_000
+): Promise<PodState> {
+ return waitFor(`${label} has one new Ready pod`, timeoutMs, () => {
+ const pods = readPods(kubectl, namespace, selector)
+ const [only] = pods
+ if (pods.length === 1 && only && only.ready && !only.terminating && !previous.has(only.uid))
+ return { done: true, value: only }
+ return { done: false, observed: describePods(pods) }
+ })
+}
+
+export interface PvcIdentity {
+ name: string
+ uid: string
+}
+
+/** The PVC the Host deployment mounts as its workspace, by claim name and UID. */
+export function hostPvcIdentity(kubectl: Kubectl): PvcIdentity {
+ const deployment = JSON.parse(
+ kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])
+ ) as {
+ spec: {
+ template: {
+ spec: { volumes?: Array<{ name: string; persistentVolumeClaim?: { claimName: string } }> }
+ }
+ }
+ }
+ const claims = (deployment.spec.template.spec.volumes ?? [])
+ .filter(volume => volume.name === 'workspace' && volume.persistentVolumeClaim)
+ .map(volume => volume.persistentVolumeClaim!.claimName)
+ if (claims.length !== 1) throw new Error(`Host deployment mounts ${claims.length} workspace PVCs`)
+ const name = claims[0]!
+ const uid = kubectl([
+ '-n',
+ HOST.namespace,
+ 'get',
+ 'pvc',
+ name,
+ '-o',
+ 'jsonpath={.metadata.uid}',
+ ]).trim()
+ if (!uid) throw new Error(`PVC ${name} has no UID`)
+ return { name, uid }
+}
+
+// ---------------------------------------------------------------------------
+// Seeding one legacy lease on the stopped Host's PVC.
+// ---------------------------------------------------------------------------
+
+export interface LegacyLeaseRecord {
+ leaseId: string
+ callerIdentity: string
+ recordIds: string[]
+ acquiredAt: string
+ expiresAt: string
+}
+
+/**
+ * A valid, non-expired lease owned by a foreign caller, with no writer session
+ * (written by an earlier boot) and no records: exactly the shape a pre-#1019
+ * shell left behind when its Host died mid-command.
+ */
+export function legacyLease(now = Date.now()): LegacyLeaseRecord {
+ const leaseId = randomUUID()
+ return {
+ leaseId,
+ callerIdentity: `legacy-lease-e2e-foreign-${leaseId.slice(0, 8)}`,
+ recordIds: [],
+ acquiredAt: new Date(now - 60_000).toISOString(),
+ expiresAt: new Date(now + 60 * 60_000).toISOString(),
+ }
+}
+
+/**
+ * Runs inside the seed pod as the Host's own uid/gid. Read-modify-write with
+ * the store's own publication sequence (temp file 0600, fsync, rename, fsync of
+ * the directory), then a re-read that must show the lease and the unchanged
+ * record set. It refuses a ledger that already carries leases.
+ */
+const SEED_SCRIPT = `
+const fs = require('node:fs')
+const path = require('node:path')
+const { randomUUID } = require('node:crypto')
+const ledgerPath = ${JSON.stringify(STORE_LEDGER_PATH)}
+const lease = JSON.parse(process.env.LEGACY_LEASE)
+const before = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))
+if (before.schemaVersion !== 1) throw new Error('ledger schemaVersion is not 1')
+if (before.processingLeases !== undefined) throw new Error('ledger already carries processingLeases')
+const recordIds = Object.keys(before.records || {}).sort()
+const next = { ...before, processingLeases: { [lease.leaseId]: lease } }
+const temporary = ledgerPath + '.tmp-' + randomUUID()
+const fd = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)
+try { fs.writeSync(fd, JSON.stringify(next)); fs.fsyncSync(fd) } finally { fs.closeSync(fd) }
+fs.renameSync(temporary, ledgerPath)
+const dir = fs.openSync(path.dirname(ledgerPath), 'r')
+try { fs.fsyncSync(dir) } finally { fs.closeSync(dir) }
+const after = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))
+const afterIds = Object.keys(after.records || {}).sort()
+const seeded = after.processingLeases && after.processingLeases[lease.leaseId]
+if (!seeded || JSON.stringify(seeded) !== JSON.stringify(lease)) throw new Error('seeded lease not read back')
+if (JSON.stringify(afterIds) !== JSON.stringify(recordIds)) throw new Error('record set changed')
+const info = fs.statSync(ledgerPath)
+process.stdout.write('LEGACY_LEASE_SEED ' + JSON.stringify({ leaseId: lease.leaseId, records: afterIds.length, mode: (info.mode & 0o777).toString(8), uid: info.uid, gid: info.gid }) + '\\n')
+`
+
+export interface SeedSummary {
+ leaseId: string
+ records: number
+ mode: string
+ uid: number
+ gid: number
+}
+
+export function parseSeedSummary(logs: string, leaseId: string): SeedSummary {
+ const lines = logs.split('\n').filter(line => line.startsWith('LEGACY_LEASE_SEED '))
+ if (lines.length !== 1) throw new Error(`seed pod printed ${lines.length} summaries`)
+ const summary = JSON.parse(lines[0]!.slice('LEGACY_LEASE_SEED '.length)) as SeedSummary
+ if (summary.leaseId !== leaseId) throw new Error('seed summary names another lease')
+ if (summary.mode !== '600' || summary.uid !== 1001 || summary.gid !== 1001)
+ throw new Error(`seeded ledger has mode ${summary.mode} owner ${summary.uid}:${summary.gid}`)
+ return summary
+}
+
+/**
+ * Seeds the lease while HCC and the Host are at zero and the Host pod is gone.
+ * The pod runs the Host's image as uid/gid 1001 (the Host's own identity) and
+ * mounts the same PVC. It is deleted before this returns, on every path.
+ */
+export async function seedLegacyLease(
+ kubectl: Kubectl,
+ input: { runId: string; scenario: string; pvc: PvcIdentity; lease: LegacyLeaseRecord }
+): Promise<SeedSummary> {
+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 0)
+ throw new Error('refusing to seed: the Host is not scaled to zero')
+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 0)
+ throw new Error('refusing to seed: HCC is not scaled to zero')
+ const hostPods = readPods(kubectl, HOST.namespace, HOST.selector)
+ if (hostPods.length !== 0)
+ throw new Error(`refusing to seed: Host pods remain (${describePods(hostPods)})`)
+ const image = JSON.parse(
+ kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])
+ ).spec.template.spec.containers.find((c: { name: string }) => c.name === HOST.container)?.image
+ if (typeof image !== 'string' || !image) throw new Error('Host container image not found')
+ const name = `legacy-lease-seed-${input.runId.slice(-12)}-${input.scenario.toLowerCase()}`
+ const pod = {
+ apiVersion: 'v1',
+ kind: 'Pod',
+ metadata: {
+ name,
+ namespace: HOST.namespace,
+ labels: { [SEED_LABEL_KEY]: input.runId },
+ },
+ spec: {
+ restartPolicy: 'Never',
+ automountServiceAccountToken: false,
+ securityContext: {
+ runAsUser: 1001,
+ runAsGroup: 1001,
+ fsGroup: 1001,
+ runAsNonRoot: true,
+ seccompProfile: { type: 'RuntimeDefault' },
+ },
+ containers: [
+ {
+ name: 'seed',
+ image,
+ imagePullPolicy: 'IfNotPresent',
+ command: ['node', '-e', SEED_SCRIPT],
+ env: [{ name: 'LEGACY_LEASE', value: JSON.stringify(input.lease) }],
+ securityContext: {
+ allowPrivilegeEscalation: false,
+ capabilities: { drop: ['ALL'] },
+ },
+ resources: {
+ limits: { cpu: '200m', memory: '128Mi' },
+ requests: { cpu: '50m', memory: '64Mi' },
+ },
+ volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
+ },
+ ],
+ volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: input.pvc.name } }],
+ },
+ }
+ kubectl(['-n', HOST.namespace, 'create', '-f', '-'], { input: JSON.stringify(pod) })
+ try {
+ const phase = await waitFor(`seed pod ${name} finishes`, 120_000, () => {
+ const current = kubectl([
+ '-n',
+ HOST.namespace,
+ 'get',
+ 'pod',
+ name,
+ '-o',
+ 'jsonpath={.status.phase}',
+ ]).trim()
+ return current === 'Succeeded' || current === 'Failed'
+ ? { done: true, value: current }
+ : { done: false, observed: current || 'Pending' }
+ })
+ const logs = kubectl(['-n', HOST.namespace, 'logs', name, '-c', 'seed'])
+ if (phase !== 'Succeeded') throw new Error(`seed pod ${name} ${phase}:\n${logs.slice(-2000)}`)
+ return parseSeedSummary(logs, input.lease.leaseId)
+ } finally {
+ kubectl(
+ [
+ '-n',
+ HOST.namespace,
+ 'delete',
+ 'pod',
+ name,
+ '--ignore-not-found',
+ '--wait=true',
+ '--timeout=90s',
+ ],
+ {
+ timeoutMs: 100_000,
+ }
+ )
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Scenarios.
+// ---------------------------------------------------------------------------
+
+export const SCENARIOS = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'] as const
+export type Scenario = (typeof SCENARIOS)[number]
+
+export interface ScenarioResult {
+ scenario: Scenario
+ pvc: PvcIdentity
+ previousHostUid: string
+ host: PodState
+ seed: SeedSummary
+ lease: LegacyLeaseRecord
+ /** Workloads the scenario replaced, with their pod UIDs before and after. */
+ replaced: Array<{ workload: string; before: string[]; after: string }>
+}
+
+function singleReadyPod(
+ kubectl: Kubectl,
+ namespace: string,
+ selector: string,
+ label: string
+): PodState {
+ const pods = readPods(kubectl, namespace, selector).filter(p => !p.terminating)
+ const [only] = pods
+ if (pods.length !== 1 || !only?.ready)
+ throw new Error(`${label} is not one Ready pod (${describePods(pods)})`)
+ return only
+}
+
+/**
+ * Stops the Host. `crash` kills the running pod without grace and scales the
+ * deployment to zero at once, so the replacement the ReplicaSet creates in
+ * between must never start a container: if one does, the crash window was
+ * lost and the scenario fails instead of silently becoming a graceful stop.
+ */
+async function stopHost(
+ kubectl: Kubectl,
+ mode: 'crash' | 'graceful',
+ running: PodState
+): Promise<void> {
+ if (mode === 'graceful') {
+ scale(kubectl, HOST.namespace, HOST.deployment, 0)
+ await waitNoPods(kubectl, HOST.namespace, HOST.selector, 'Host')
+ return
+ }
+ kubectl([
+ '-n',
+ HOST.namespace,
+ 'delete',
+ 'pod',
+ running.name,
+ '--force',
+ '--grace-period=0',
+ '--wait=false',
+ ])
+ scale(kubectl, HOST.namespace, HOST.deployment, 0)
+ const startedReplacements = new Set<string>()
+ await waitFor('Host has no pods after the crash', 120_000, () => {
+ const pods = readPods(kubectl, HOST.namespace, HOST.selector)
+ for (const pod of pods)
+ if (pod.uid !== running.uid && pod.started) startedReplacements.add(pod.name)
+ return pods.length === 0
+ ? { done: true, value: undefined }
+ : { done: false, observed: describePods(pods) }
+ })
+ if (startedReplacements.size > 0)
+ throw new Error(
+ `CRASH_WINDOW_LOST: a replacement Host pod started before the scale to zero (${[...startedReplacements].join(', ')})`
+ )
+}
+
+async function restartWorkload(
+ kubectl: Kubectl,
+ workload: { namespace: string; deployment: string; selector: string }
+): Promise<{ workload: string; before: string[]; after: string }> {
+ const before = readPods(kubectl, workload.namespace, workload.selector).map(p => p.uid)
+ if (before.length === 0) throw new Error(`${workload.deployment} has no pod to restart`)
+ rolloutRestart(kubectl, workload.namespace, workload.deployment)
+ rolloutStatus(kubectl, workload.namespace, workload.deployment)
+ const pod = await waitSingleNewReadyPod(
+ kubectl,
+ workload.namespace,
+ workload.selector,
+ new Set(before),
+ workload.deployment
+ )
+ return { workload: workload.deployment, before, after: pod.uid }
+}
+
+/**
+ * Runs one scenario end to end: HCC to zero, Host stopped, lease seeded, the
+ * scenario's own restarts, then HCC and the Host back. Returns the new Host pod.
+ * Every step leaves HCC and the Host at their pre-scenario replicas on success;
+ * on failure, `ensureWorkloadsUp` (and the runner's restoration) bring them back.
+ */
+export async function runScenario(
+ kubectl: Kubectl,
+ input: { scenario: Scenario; runId: string }
+): Promise<ScenarioResult> {
+ const { scenario } = input
+ const replaced: ScenarioResult['replaced'] = []
+ const pvcBefore = hostPvcIdentity(kubectl)
+ const running = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')
+ const hccBefore = singleReadyPod(kubectl, HCC.namespace, HCC.selector, 'HCC')
+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)
+ throw new Error('Host deployment is not at one replica before the scenario')
+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)
+ throw new Error('HCC deployment is not at one replica before the scenario')
+
+ // HCC first: it owns the Host deployment and would scale it back up.
+ scale(kubectl, HCC.namespace, HCC.deployment, 0)
+ await waitNoPods(kubectl, HCC.namespace, HCC.selector, 'HCC')
+ await stopHost(kubectl, scenario === 'S-crash' ? 'crash' : 'graceful', running)
+
+ const lease = legacyLease()
+ const seed = await seedLegacyLease(kubectl, {
+ runId: input.runId,
+ scenario,
+ pvc: pvcBefore,
+ lease,
+ })
+
+ const startHostThenHcc = async () => {
+ scale(kubectl, HOST.namespace, HOST.deployment, 1)
+ scale(kubectl, HCC.namespace, HCC.deployment, 1)
+ }
+ switch (scenario) {
+ case 'S-crash':
+ await startHostThenHcc()
+ break
+ case 'S-graceful':
+ // A new template while stopped: the next pod is a fresh rollout, the way
+ // a graceful `rollout restart` replaces the Host.
+ rolloutRestart(kubectl, HOST.namespace, HOST.deployment)
+ await startHostThenHcc()
+ break
+ case 'S-hcc':
+ await startHostThenHcc()
+ break
+ case 'S-gfs':
+ for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))
+ await startHostThenHcc()
+ break
+ case 'S-update':
+ // Upgrade order: GFS, WRC, HCC, then the Host.
+ for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))
+ replaced.push(await restartWorkload(kubectl, WRC))
+ rolloutRestart(kubectl, HOST.namespace, HOST.deployment)
+ scale(kubectl, HCC.namespace, HCC.deployment, 1)
+ rolloutStatus(kubectl, HCC.namespace, HCC.deployment)
+ scale(kubectl, HOST.namespace, HOST.deployment, 1)
+ break
+ }
+
+ const hccPod = await waitSingleNewReadyPod(
+ kubectl,
+ HCC.namespace,
+ HCC.selector,
+ new Set([hccBefore.uid]),
+ 'HCC'
+ )
+ replaced.push({ workload: HCC.deployment, before: [hccBefore.uid], after: hccPod.uid })
+ const host = await waitSingleNewReadyPod(
+ kubectl,
+ HOST.namespace,
+ HOST.selector,
+ new Set([running.uid]),
+ 'Host'
+ )
+ rolloutStatus(kubectl, HOST.namespace, HOST.deployment)
+ // HCC's first reconcile must not roll the Host again: the pod that booted on
+ // the seeded ledger is the one whose signals are read.
+ const settled = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')
+ if (settled.uid !== host.uid)
+ throw new Error(`HCC replaced the Host pod after it booted (${host.uid} -> ${settled.uid})`)
+ const pvcAfter = hostPvcIdentity(kubectl)
+ if (pvcAfter.name !== pvcBefore.name || pvcAfter.uid !== pvcBefore.uid)
+ throw new Error(
+ `Host PVC changed across ${scenario}: ${JSON.stringify(pvcBefore)} -> ${JSON.stringify(pvcAfter)}`
+ )
+ return { scenario, pvc: pvcAfter, previousHostUid: running.uid, host, seed, lease, replaced }
+}
+
+/** Brings HCC and the Host back to one replica after a failed scenario. */
+export function ensureWorkloadsUp(kubectl: Kubectl): void {
+ if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)
+ scale(kubectl, HOST.namespace, HOST.deployment, 1)
+ if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)
+ scale(kubectl, HCC.namespace, HCC.deployment, 1)
+ rolloutStatus(kubectl, HCC.namespace, HCC.deployment)
+ rolloutStatus(kubectl, HOST.namespace, HOST.deployment)
+}
+
+// ---------------------------------------------------------------------------
+// Signal readers (read-only) and pure verdicts.
+// ---------------------------------------------------------------------------
+
+export function readStoreLedger(kubectl: Kubectl, podName: string): Record<string, unknown> {
+ const raw = kubectl([
+ '-n',
+ HOST.namespace,
+ 'exec',
+ podName,
+ '-c',
+ HOST.container,
+ '--',
+ 'cat',
+ STORE_LEDGER_PATH,
+ ])
+ const parsed = JSON.parse(raw) as unknown
+ if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
+ throw new Error('store ledger is not a JSON object')
+ return parsed as Record<string, unknown>
+}
+
+export function readHostLogLines(kubectl: Kubectl, podName: string): string[] {
+ return kubectl(['-n', HOST.namespace, 'logs', podName, '-c', HOST.container], {
+ timeoutMs: 60_000,
+ })
+ .split('\n')
+ .filter(line => line.trim().length > 0)
+}
+
+export function readHostMetrics(kubectl: Kubectl, podName: string): string {
+ return kubectl([
+ '-n',
+ HOST.namespace,
+ 'exec',
+ podName,
+ '-c',
+ HOST.container,
+ '--',
+ 'node',
+ '-e',
+ "fetch('http://127.0.0.1:8080/metrics').then(r=>{if(!r.ok)throw new Error('metrics '+r.status);return r.text()}).then(t=>process.stdout.write(t))",
+ ])
+}
+
+/**
+ * The value of an unlabelled metric, or null when the exposition has no such
+ * sample. A metric with labels, or more than one sample, is refused.
+ */
+export function parseMetricValue(text: string, name: string): number | null {
+ const samples = text
+ .split('\n')
+ .filter(
+ line => !line.startsWith('#') && (line.startsWith(`${name} `) || line.startsWith(`${name}{`))
+ )
+ if (samples.length === 0) return null
+ if (samples.length !== 1) throw new Error(`${name} has ${samples.length} samples`)
+ const match = new RegExp(`^${name}(?:\\{\\})? ([0-9.eE+-]+)$`).exec(samples[0]!.trim())
+ if (!match) throw new Error(`${name} sample is not an unlabelled number: ${samples[0]}`)
+ const value = Number(match[1])
+ if (!Number.isFinite(value)) throw new Error(`${name} sample is not finite`)
+ return value
+}
+
+interface StoreLogLine {
+ component?: unknown
+ msg?: unknown
+ discarded?: unknown
+ legacyProcessingLeases?: unknown
+ outcome?: unknown
+}
+
+function storeLogLines(logLines: string[]): StoreLogLine[] {
+ const parsed: StoreLogLine[] = []
+ for (const line of logLines) {
+ if (!line.startsWith('{')) continue
+ let value: unknown
+ try {
+ value = JSON.parse(line)
+ } catch {
+ continue
+ }
+ if (
+ typeof value === 'object' &&
+ value !== null &&
+ (value as StoreLogLine).component === STORE_LOG_COMPONENT
+ )
+ parsed.push(value as StoreLogLine)
+ }
+ return parsed
+}
+
+export type RecoveryOutcome = 'discarded' | 'unknown-outcome'
+
+/**
+ * Signal 2 on the fixed Host. The ledger must have no `processingLeases`, and
+ * exactly one of the two outcomes must be visible: one confirmed discard of
+ * one lease with the counter at 1, or one unknown-outcome warning with the
+ * counter at 0. The counter is read from a pod that just booted, so its value
+ * is the delta. A missing counter is a failure: it is the liveness witness
+ * that the metric registry of the fixed build is the one that answered.
+ */
+export function legacyLeaseRecoveryVerdict(input: {
+ ledger: Record<string, unknown>
+ logLines: string[]
+ counter: number | null
+}): RecoveryOutcome {
+ if (Object.hasOwn(input.ledger, 'processingLeases'))
+ throw new Error('the ledger still carries processingLeases after the restart')
+ if (input.counter === null)
+ throw new Error(`${LEGACY_DISCARD_COUNTER} is not exposed by the Host`)
+ const lines = storeLogLines(input.logLines)
+ const discards = lines.filter(
+ line =>
+ typeof line.msg === 'string' &&
+ line.msg.startsWith(DISCARD_MSG_PREFIX) &&
+ typeof line.discarded === 'number'
+ )
+ const unknown = lines.filter(
+ line =>
+ line.outcome === 'unknown' &&
+ typeof line.msg === 'string' &&
+ line.msg.startsWith(UNKNOWN_MSG_PREFIX) &&
+ typeof line.legacyProcessingLeases === 'number'
+ )
+ if (
+ discards.length === 1 &&
+ unknown.length === 0 &&
+ discards[0]!.discarded === 1 &&
+ input.counter === 1
+ )
+ return 'discarded'
+ if (
+ unknown.length === 1 &&
+ discards.length === 0 &&
+ unknown[0]!.legacyProcessingLeases === 1 &&
+ input.counter === 0
+ )
+ return 'unknown-outcome'
+ throw new Error(
+ `no recovery outcome: ${discards.length} discard line(s) ` +
+ `(${discards.map(line => String(line.discarded)).join(',') || '-'}), ` +
+ `${unknown.length} unknown-outcome line(s), counter ${input.counter}`
+ )
+}
+
+/**
+ * The vacuity witness on the pre-fix Host: the seeded lease is still in the
+ * ledger, nothing was discarded, and the counter does not exist. Together with
+ * the journey failing at the shell with download_busy, this shows the lane can
+ * tell the fixed build from the broken one.
+ */
+export function vacuityStoreVerdict(input: {
+ ledger: Record<string, unknown>
+ logLines: string[]
+ counter: number | null
+ leaseId: string
+}): 'lease-retained' {
+ const leases = input.ledger.processingLeases as Record<string, unknown> | undefined
+ if (!leases || typeof leases !== 'object' || !Object.hasOwn(leases, input.leaseId))
+ throw new Error('the pre-fix Host no longer carries the seeded lease')
+ const discards = storeLogLines(input.logLines).filter(
+ line => typeof line.msg === 'string' && line.msg.startsWith(DISCARD_MSG_PREFIX)
+ )
+ if (discards.length !== 0) throw new Error('the pre-fix Host logged a legacy-lease discard')
+ if (input.counter !== null) throw new Error(`the pre-fix Host exposes ${LEGACY_DISCARD_COUNTER}`)
+ return 'lease-retained'
+}
+
+export interface StoreRecord {
+ id: string
+ hostPath: string
+ sizeBytes: number
+ sha256?: string
+ state: string
+}
+
+export function storeRecords(ledger: Record<string, unknown>): StoreRecord[] {
+ const records = ledger.records
+ if (typeof records !== 'object' || records === null || Array.isArray(records))
+ throw new Error('store ledger has no records object')
+ return Object.values(records as Record<string, StoreRecord>)
+}
+
+/** The one completed record this journey added, matching the GFS source bytes. */
+export function downloadedRecord(
+ before: Record<string, unknown>,
+ after: Record<string, unknown>,
+ source: { sha256: string; bytes: number }
+): StoreRecord {
+ if (!SHA256_HEX.test(source.sha256)) throw new Error('source sha256 is not a digest')
+ const known = new Set(storeRecords(before).map(record => record.id))
+ const added = storeRecords(after).filter(record => !known.has(record.id))
+ const matching = added.filter(
+ record =>
+ record.state === 'completed' &&
+ record.sha256 === source.sha256 &&
+ record.sizeBytes === source.bytes
+ )
+ if (added.length !== 1 || matching.length !== 1)
+ throw new Error(
+ `expected one new completed record for ${source.sha256}/${source.bytes} bytes; ` +
+ `added ${JSON.stringify(added.map(r => ({ state: r.state, sha256: r.sha256, sizeBytes: r.sizeBytes })))}`
+ )
+ return matching[0]!
+}
+
+/** sha256 and size of a file on the Host PVC, computed in the Host container. */
+export function hostFileDigest(
+ kubectl: Kubectl,
+ podName: string,
+ hostPath: string
+): { sha256: string; bytes: number } {
+ if (path_is_unsafe(hostPath)) throw new Error(`refusing to read ${hostPath}`)
+ const out = kubectl([
+ '-n',
+ HOST.namespace,
+ 'exec',
+ podName,
+ '-c',
+ HOST.container,
+ '--',
+ 'node',
+ '-e',
+ "const b=require('node:fs').readFileSync(process.argv[1]);process.stdout.write(require('node:crypto').createHash('sha256').update(b).digest('hex')+' '+b.length)",
+ `/workspace/${hostPath}`,
+ ]).trim()
+ const [sha256, bytes] = out.split(' ')
+ if (!sha256 || !SHA256_HEX.test(sha256) || !/^\d+$/.test(bytes ?? ''))
+ throw new Error(`unexpected digest output: ${out}`)
+ return { sha256, bytes: Number(bytes) }
+}
+
+function path_is_unsafe(hostPath: string): boolean {
+ return hostPath.startsWith('/') || hostPath.split('/').some(part => part === '..' || part === '')
+}
diff --git a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts
new file mode 100644
index 000000000..41fb82cd4
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts
@@ -0,0 +1,59 @@
+import { defineConfig } from '@playwright/test'
+import path from 'node:path'
+import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'
+import { requireLegacyLeaseLaneEnv } from './helpers/legacyLeaseRestart'
+
+// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns
+// visible IPC transitions and reads the external provider ledger as its wire oracle.
+
+/*
+ * Desktop App legacy processing-lease restart lane (issue #1022).
+ *
+ * It reuses the image-capabilities fixture stack (derived Host image, provider
+ * fixture, seeded catalog and login) and adds the cluster side: before each
+ * scenario the Host and HCC are stopped, one legacy foreign processing lease is
+ * seeded on the Host PVC, the scenario's restarts run, and the Desktop journey
+ * (approved `shell_exec`, then `clerum__gfs_download`) must succeed on the Host
+ * that booted on that ledger.
+ *
+ * - It loads NO env file. Every binding is exported by the runner
+ * (`make minikube-run-legacy-lease-restart` or the -vacuity target) and a
+ * missing or inconsistent one fails here, at config load.
+ * - The runner selects the scenarios with --grep: the fixed lane runs the five
+ * `legacy-lease-restart fixture:` tests, the vacuity lane runs the one
+ * `legacy-lease-restart vacuity:` test against the pre-fix Host.
+ */
+
+const repoRoot = path.resolve(__dirname, '../../..')
+
+if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {
+ throw new Error(
+ `NODE_ENV must be "test" (or unset) for the legacy-lease-restart fixture lane; ` +
+ `received "${process.env.NODE_ENV}".`
+ )
+}
+process.env.NODE_ENV = 'test'
+
+// Fail at config load, with every missing binding listed at once.
+requireImageCapabilitiesFixtureEnv(process.env)
+requireLegacyLeaseLaneEnv(process.env)
+
+const recorderRoot = process.env.QA_RECORDER_ROOT
+ ? path.resolve(process.env.QA_RECORDER_ROOT)
+ : path.join(repoRoot, '.local-notes', 'qa-recorder')
+
+export default defineConfig({
+ testDir: '.',
+ testMatch: /qa-recorder-legacy-lease-restart\.spec\.ts/,
+ // Per scenario: HCC and Host stop, seed pod, the scenario's own rollouts
+ // (S-update replaces GFS, WRC, HCC and the Host), one Electron launch and a
+ // two-tool journey with an approval.
+ timeout: 660_000,
+ expect: { timeout: 30_000 },
+ retries: 0,
+ workers: 1,
+ fullyParallel: false,
+ reporter: [['list']],
+ outputDir: path.join(recorderRoot, 'runs', 'desktop-app-legacy-lease-restart'),
+ preserveOutput: 'always',
+})
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts
new file mode 100644
index 000000000..e0e1a5dcb
--- /dev/null
+++ b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts
@@ -0,0 +1,440 @@
+import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'
+import { createHash, randomBytes } from 'node:crypto'
+import {
+ GFS_E2E_DRIVE,
+ type GfsFileFixture,
+ cleanupGfsFixture,
+ getE2EUserId,
+ gfsWriterPod,
+ kubectlOut,
+ ridOf,
+ seedGfsFileFixture,
+ seedGfsGrant,
+ uniqueGfsFixtureName,
+} from '../../../tests/e2e/gfsUiFixtures'
+import { discoverManagedGfsAgent } from './helpers/gfsAgentDiscovery'
+import {
+ FIXTURE_RESPONSE_KIND,
+ IMAGE_CAPABILITY_FIXTURE_MODELS,
+ type ImageCapabilityEvidenceSnapshot,
+ appendedAttempts,
+ readImageCapabilityEvidence,
+ requireImageCapabilitiesFixtureEnv,
+} from './helpers/imageCapabilityEvidence'
+import {
+ HOST,
+ type Kubectl,
+ LEGACY_DISCARD_COUNTER,
+ type LegacyLeaseLaneMode,
+ SCENARIOS,
+ type Scenario,
+ type ScenarioResult,
+ downloadedRecord,
+ ensureWorkloadsUp,
+ hostFileDigest,
+ kubectlFor,
+ legacyLeaseRecoveryVerdict,
+ parseMetricValue,
+ readHostLogLines,
+ readHostMetrics,
+ readPods,
+ readStoreLedger,
+ requireLegacyLeaseLaneEnv,
+ runScenario,
+ vacuityStoreVerdict,
+} from './helpers/legacyLeaseRestart'
+import {
+ EXTERNAL_REST_API_BASE_URL,
+ RPC_PROXY_BASE_URL,
+ assertAllowedTarget,
+ desktopCredentials,
+ finalizeRecording,
+ launchDesktopApp,
+ login,
+ screenshotAndLog,
+} from './qa-recorder-helpers'
+
+/*
+ * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the agent switch, the model popover and
+ * the approval card all travel over Electron IPC, so the renderer has no HTTP
+ * request to await for any transition. The oracles are the visible chat (the
+ * fixture's answer line and the approval card), the provider-boundary ledger of
+ * the derived in-cluster fixture, and read-only reads of the Host's own store
+ * ledger, logs, metrics and workspace file.
+ *
+ * Issue #1022: a processing lease written by a Host build before #1019 has no
+ * writer session. A Host that boots on such a ledger must discard it at
+ * initialize (log `discarded 1`, counter 1) or report that it could not confirm
+ * the discard; either way the ledger it serves carries no `processingLeases`,
+ * and a shell command plus a GFS download work. The pre-fix build treats the
+ * lease as an inherited executor, so its shell refuses with `download_busy`.
+ *
+ * Each scenario stops HCC and the Host, seeds one foreign lease on the Host PVC
+ * from a pod running as the Host's uid/gid, performs its own restarts, and
+ * brings HCC and the Host back. Then the Desktop journey runs: the user sends a
+ * journey line, approves `shell_exec` by clicking Approve, and the fixture asks
+ * for `clerum__gfs_download` of a seeded GFS file. The answer on screen carries
+ * the sha256 and byte count of the Host's receipt, which must equal the GFS
+ * source, as must the file the Host wrote to its workspace.
+ *
+ * Setup shortcuts (named, outside the behaviour under test): the GFS file and
+ * its grants are seeded with SQL and the writer blob with kubectl cp, and the
+ * legacy lease is written by the seed pod. Only the external provider peer is
+ * simulated; nothing is mocked inside the cluster or the app.
+ */
+
+const SCENARIO_TITLES: Record<Scenario, string> = {
+ 'S-crash':
+ 'S-crash: a Host killed without grace boots on a legacy lease and serves shell then download',
+ 'S-graceful':
+ 'S-graceful: a Host stopped and rolled out boots on a legacy lease and serves shell then download',
+ 'S-hcc':
+ 'S-hcc: an HCC restart with the Host boots on a legacy lease and serves shell then download',
+ 'S-gfs':
+ 'S-gfs: each GFS pod restarts while the Host boots on a legacy lease, then shell and download work',
+ 'S-update':
+ 'S-update: GFS, WRC, HCC then Host restart in upgrade order on a legacy lease, then shell and download work',
+}
+
+const JOURNEY_RESPONSE_TIMEOUT_MS = 240_000
+const ANY_TERMINAL_ANSWER = /LEGACY-LEASE-FIXTURE-(?:OK|SHELL-FAILED|DOWNLOAD-FAILED)/
+
+interface LaneContext {
+ mode: LegacyLeaseLaneMode
+ runId: string
+ kubectl: Kubectl
+ env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>
+ file: GfsFileFixture
+ source: { sha256: string; bytes: number }
+}
+
+let lane: LaneContext | undefined
+
+function requireLane(expected: LegacyLeaseLaneMode): LaneContext {
+ if (!lane) throw new Error('legacy-lease-restart lane context was not prepared')
+ // A mode mismatch is a runner defect: it fails, it never skips.
+ if (lane.mode !== expected)
+ throw new Error(
+ `this test belongs to the ${expected} lane, but LEGACY_LEASE_LANE_MODE is ${lane.mode}`
+ )
+ return lane
+}
+
+test.describe.configure({ mode: 'serial' })
+
+test.beforeAll(async () => {
+ const env = requireImageCapabilitiesFixtureEnv()
+ const laneEnv = requireLegacyLeaseLaneEnv()
+ await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)
+ await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)
+ const kubectl = kubectlFor(laneEnv.profile)
+
+ // The GFS subject of the Host this lane drives: exactly the chatllm Host.
+ const agent = discoverManagedGfsAgent()
+ if (agent.namespace !== HOST.namespace || agent.name !== HOST.deployment)
+ throw new Error(`GFS agent discovery returned ${agent.subjectId}, not the lane's Host`)
+ const ownerUserId = getE2EUserId(desktopCredentials().email)
+
+ const file = seedGfsFileFixture(uniqueGfsFixtureName('e2e-gfs-legacy-lease'))
+ try {
+ seedGfsGrant({
+ resourceId: file.fileResourceId,
+ subjectType: 'user',
+ subjectId: ownerUserId,
+ permissions: ['read'],
+ inherit: true,
+ grantedBy: 'e2e:legacy-lease-restart',
+ })
+ seedGfsGrant({
+ resourceId: file.fileResourceId,
+ subjectType: 'host',
+ subjectId: agent.subjectId,
+ permissions: ['read'],
+ inherit: true,
+ grantedBy: 'e2e:legacy-lease-restart',
+ })
+ // The source the journey must reproduce: the bytes the fixture seeded, and
+ // the same digest read back from the writer's blob.
+ const content = Buffer.from(`E2E GFS file fixture: ${file.name}\n`, 'utf8')
+ const sha256 = createHash('sha256').update(content).digest('hex')
+ const blob = kubectlOut([
+ '-n',
+ 'gfs',
+ 'exec',
+ gfsWriterPod(),
+ '--',
+ 'sha256sum',
+ `/data/gfs/${ridOf(file.fileResourceId)}`,
+ ]).trim()
+ if (blob.split(/\s+/)[0] !== sha256)
+ throw new Error(`writer blob digest ${blob} differs from the seeded ${sha256}`)
+ lane = {
+ mode: laneEnv.mode,
+ runId: laneEnv.runId,
+ kubectl,
+ env,
+ file,
+ source: { sha256, bytes: content.length },
+ }
+ } catch (error) {
+ cleanupGfsFixture(file.name)
+ throw error
+ }
+})
+
+test.afterAll(async () => {
+ if (lane) cleanupGfsFixture(lane.file.name)
+})
+
+/** Reach the authenticated shell through the real sign-in form. */
+async function signIn(page: Page) {
+ await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })
+ const emailInput = page.locator('#email-input')
+ const settingsMenu = page.getByTestId('nav-settings-menu')
+ await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })
+ if (await settingsMenu.isVisible()) {
+ if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') await settingsMenu.click()
+ await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')
+ await page.getByTestId('logout-btn').click()
+ }
+ await expect(emailInput).toBeVisible({ timeout: 20_000 })
+ await expect(page.getByTestId('nav-chat')).toHaveCount(0)
+ await login(page, desktopCredentials())
+ await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })
+}
+
+async function openFixtureChat(page: Page, hostRef: string) {
+ const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ await page.getByTestId('nav-chat').click()
+ const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })
+ await expect(switchAgent).toBeVisible({ timeout: 30_000 })
+ await switchAgent.click()
+ const item = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })
+ await expect(item).toHaveCount(1)
+ await item.click()
+ await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))
+
+ await page.getByTestId('nav-new-chat').click()
+ const composer = page.getByRole('textbox', { name: 'Agent message composer' })
+ await expect(composer).toBeVisible({ timeout: 30_000 })
+ await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })
+
+ const model = IMAGE_CAPABILITY_FIXTURE_MODELS.supported
+ const chip = page.getByTestId('selected-chat-model')
+ await expect(chip).toBeVisible({ timeout: 30_000 })
+ await chip.click()
+ await expect(page.getByRole('menu', { name: 'Select model' })).toBeVisible({ timeout: 20_000 })
+ const row = page.getByTestId(`model-option-${model}`)
+ await expect(row).toHaveCount(1)
+ await row.click()
+ await expect(chip).toHaveAttribute('data-model-id', model)
+ return composer
+}
+
+interface JourneyOutcome {
+ answer: string
+ approvals: string[]
+ before: ImageCapabilityEvidenceSnapshot
+ after: ImageCapabilityEvidenceSnapshot
+}
+
+/**
+ * Sends the journey line and approves every tool approval the Host raises by
+ * clicking Approve, until the fixture's terminal answer is on screen. Returns
+ * the answer text and the input preview of each approval clicked.
+ */
+async function runJourney(page: Page, ctx: LaneContext, marker: string): Promise<JourneyOutcome> {
+ const composer = await openFixtureChat(page, ctx.env.hostRef)
+ const line = `LEGACY_LEASE_JOURNEY marker=${marker} drive=${GFS_E2E_DRIVE} resourceId=${ridOf(ctx.file.fileResourceId)}`
+ const before = readImageCapabilityEvidence(ctx.env)
+ expect(before.runId).toBe(ctx.env.runId)
+ await composer.fill(line)
+ const send = page.getByTestId('send-button')
+ await expect(send).toBeEnabled({ timeout: 20_000 })
+ await send.click()
+
+ const approve = page.getByTestId('approval-approve-btn')
+ const preview = page.getByTestId('approval-input-preview')
+ const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')
+ const terminal = response.filter({ hasText: ANY_TERMINAL_ANSWER })
+ const approvals: string[] = []
+ // The journey has at most two approvable tools (shell, then download).
+ for (let round = 0; round < 3; round += 1) {
+ await expect(approve.or(terminal)).toBeVisible({ timeout: JOURNEY_RESPONSE_TIMEOUT_MS })
+ if ((await terminal.count()) > 0) break
+ if (round === 2) throw new Error('more approvals than the journey has tools')
+ await expect(preview).toBeVisible({ timeout: 20_000 })
+ const shown = await preview.innerText()
+ approvals.push(shown)
+ await approve.click()
+ // The clicked card leaves, or is replaced by the next tool's approval.
+ await expect
+ .poll(
+ async () =>
+ (await approve.count()) === 0 ||
+ ((await preview.count()) > 0 && (await preview.innerText()) !== shown),
+ { timeout: 60_000 }
+ )
+ .toBe(true)
+ }
+ await expect(terminal).toHaveCount(1)
+ const answer = (await terminal.innerText()).trim()
+ const after = readImageCapabilityEvidence(ctx.env)
+ return { answer, approvals, before, after }
+}
+
+/** The shell approval the user clicked is the fixture's command for this marker. */
+function expectShellApproved(approvals: string[], marker: string) {
+ expect(approvals.length).toBeGreaterThanOrEqual(1)
+ expect(approvals[0]).toContain('printf')
+ expect(approvals[0]).toContain(marker)
+}
+
+async function withDesktop(
+ testInfo: TestInfo,
+ label: string,
+ body: (page: Page) => Promise<void>
+): Promise<void> {
+ let app: ElectronApplication | undefined
+ let recordedPage: Page | undefined
+ try {
+ const launched = await launchDesktopApp(
+ testInfo,
+ `issue1022-${label}-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`
+ )
+ app = launched.app
+ recordedPage = launched.page
+ await signIn(launched.page)
+ await body(launched.page)
+ await screenshotAndLog(launched.page, testInfo, `desktop-legacy-lease-${label}`)
+ } catch (error) {
+ if (recordedPage && !recordedPage.isClosed()) {
+ await recordedPage
+ .screenshot({ path: testInfo.outputPath('failure.png') })
+ .catch(() => undefined)
+ }
+ throw error
+ } finally {
+ await finalizeRecording(app, recordedPage)
+ }
+}
+
+/** The Host pod that booted on the seeded ledger is still the only Ready one. */
+function expectHostUnchanged(ctx: LaneContext, result: ScenarioResult) {
+ const pods = readPods(ctx.kubectl, HOST.namespace, HOST.selector).filter(p => !p.terminating)
+ expect(pods.map(p => p.uid)).toEqual([result.host.uid])
+ expect(pods[0]?.ready).toBe(true)
+ expect(result.host.uid).not.toBe(result.previousHostUid)
+}
+
+function attachScenario(testInfo: TestInfo, data: unknown) {
+ return testInfo.attach('scenario.json', {
+ body: JSON.stringify(data, null, 2),
+ contentType: 'application/json',
+ })
+}
+
+for (const scenario of SCENARIOS) {
+ test(`legacy-lease-restart fixture: ${SCENARIO_TITLES[scenario]}`, async ({}, testInfo) => {
+ const ctx = requireLane('fixed')
+ try {
+ const result = await runScenario(ctx.kubectl, { scenario, runId: ctx.runId })
+ expect(result.seed.leaseId).toBe(result.lease.leaseId)
+
+ // Signal 2: the store the new Host serves has no legacy lease, and it
+ // reports one of the two recovery outcomes.
+ const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)
+ const metrics = readHostMetrics(ctx.kubectl, result.host.name)
+ const outcome = legacyLeaseRecoveryVerdict({
+ ledger: ledgerAtBoot,
+ logLines: readHostLogLines(ctx.kubectl, result.host.name),
+ counter: parseMetricValue(metrics, LEGACY_DISCARD_COUNTER),
+ })
+ await attachScenario(testInfo, { ...result, outcome })
+
+ // Signal 3: the Desktop journey on that Host.
+ const marker = `legacy-lease-${randomBytes(8).toString('hex')}`
+ await withDesktop(testInfo, scenario.toLowerCase(), async page => {
+ const journey = await runJourney(page, ctx, marker)
+ expectShellApproved(journey.approvals, marker)
+ expect(journey.answer).toContain(
+ `LEGACY-LEASE-FIXTURE-OK marker=${marker} sha256=${ctx.source.sha256} bytes=${ctx.source.bytes}`
+ )
+ const rows = appendedAttempts(journey.before, journey.after)
+ const kinds = rows.map(row => row.responseKind)
+ expect(
+ kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)
+ ).toHaveLength(1)
+ expect(
+ kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)
+ ).toHaveLength(1)
+ const answers = rows.filter(
+ row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer
+ )
+ expect(answers).toHaveLength(1)
+ expect(answers[0]?.downloadSha256).toBe(ctx.source.sha256)
+ expect(answers[0]?.downloadBytes).toBe(ctx.source.bytes)
+ const delta = (key: keyof ImageCapabilityEvidenceSnapshot['counters']) =>
+ journey.after.counters[key] - journey.before.counters[key]
+ expect(delta('legacyLeaseShellRequests')).toBe(1)
+ expect(delta('legacyLeaseDownloadRequests')).toBe(1)
+ expect(delta('legacyLeaseAnswers')).toBe(1)
+ expect(delta('legacyLeaseFailures')).toBe(0)
+ })
+
+ // The store recorded exactly this download, and the workspace file holds
+ // the GFS source bytes.
+ const ledgerAfter = readStoreLedger(ctx.kubectl, result.host.name)
+ const record = downloadedRecord(ledgerAtBoot, ledgerAfter, ctx.source)
+ expect(hostFileDigest(ctx.kubectl, result.host.name, record.hostPath)).toEqual(ctx.source)
+ expect(Object.hasOwn(ledgerAfter, 'processingLeases')).toBe(false)
+ expectHostUnchanged(ctx, result)
+ } finally {
+ ensureWorkloadsUp(ctx.kubectl)
+ }
+ })
+}
+
+test('legacy-lease-restart vacuity: S-crash on base 74e0d81d9 fails at the shell with download_busy', async ({}, testInfo) => {
+ const ctx = requireLane('vacuity')
+ try {
+ const result = await runScenario(ctx.kubectl, { scenario: 'S-crash', runId: ctx.runId })
+ const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)
+ const verdict = vacuityStoreVerdict({
+ ledger: ledgerAtBoot,
+ logLines: readHostLogLines(ctx.kubectl, result.host.name),
+ counter: parseMetricValue(
+ readHostMetrics(ctx.kubectl, result.host.name),
+ LEGACY_DISCARD_COUNTER
+ ),
+ leaseId: result.lease.leaseId,
+ })
+ await attachScenario(testInfo, { ...result, verdict })
+
+ const marker = `legacy-lease-${randomBytes(8).toString('hex')}`
+ await withDesktop(testInfo, 'vacuity', async page => {
+ const journey = await runJourney(page, ctx, marker)
+ expectShellApproved(journey.approvals, marker)
+ expect(journey.answer).toContain('LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy')
+ const rows = appendedAttempts(journey.before, journey.after)
+ const shellFailed = rows.filter(
+ row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed
+ )
+ // Positive witness first: the shell step ran and failed with the store code.
+ expect(shellFailed).toHaveLength(1)
+ expect(shellFailed[0]?.failureCode).toBe('download_busy')
+ expect(
+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)
+ ).toHaveLength(1)
+ expect(
+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)
+ ).toHaveLength(0)
+ expect(
+ rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer)
+ ).toHaveLength(0)
+ })
+ expectHostUnchanged(ctx, result)
+ } finally {
+ ensureWorkloadsUp(ctx.kubectl)
+ }
+})
diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.ts
index 7912bc0d6..31c621059 100644
--- a/desktop-app/test/imageCapabilityEvidence.test.ts
+++ b/desktop-app/test/imageCapabilityEvidence.test.ts
@@ -44,6 +44,24 @@ const rows = {
truncated: true,
}),
textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),
+ shellRequested: (): Row => ({
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'legacy-lease-shell-requested',
+ }),
+ legacyAnswer: (): Row => ({
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'legacy-lease-answer',
+ downloadSha256: DIGEST,
+ downloadBytes: 42,
+ }),
+ shellFailed: (): Row => ({
+ model: 'glm-5.3-flash',
+ imageSha256: null,
+ responseKind: 'legacy-lease-shell-failed',
+ failureCode: 'download_busy',
+ }),
}
function ledger(attempts: Row[]): string {
@@ -63,6 +81,10 @@ function ledger(attempts: Row[]): string {
documentReadRequests: 0,
documentAnswers: 0,
documentFailures: 0,
+ legacyLeaseShellRequests: 0,
+ legacyLeaseDownloadRequests: 0,
+ legacyLeaseAnswers: 0,
+ legacyLeaseFailures: 0,
},
attempts,
})
@@ -120,6 +142,47 @@ describe('imageCapabilityEvidence ledger rows', () => {
})
})
+describe('legacy-lease ledger rows', () => {
+ it('keeps the failure code and the download receipt on the rows that carry them', () => {
+ const snapshot = parse([rows.shellRequested(), rows.legacyAnswer(), rows.shellFailed()])
+ expect(snapshot.attempts).toEqual([
+ rows.shellRequested(),
+ rows.legacyAnswer(),
+ rows.shellFailed(),
+ ])
+ })
+
+ it('requires a closed-set failure code on failure rows and nowhere else', () => {
+ // Witness: the well-formed failure row parses with its code.
+ expect(parse([rows.shellFailed()]).attempts[0]?.failureCode).toBe('download_busy')
+ const { failureCode: _code, ...withoutCode } = rows.shellFailed()
+ for (const [label, row] of [
+ ['missing failureCode', withoutCode],
+ ['free-text failureCode', { ...rows.shellFailed(), failureCode: 'disk on fire' }],
+ ['failureCode on an answer row', { ...rows.legacyAnswer(), failureCode: 'download_busy' }],
+ ['failureCode on a text row', { ...rows.textOnly(), failureCode: 'download_busy' }],
+ ] as const) {
+ expect(() => parse([row]), label).toThrow(/failureCode/)
+ }
+ })
+
+ it('requires the download receipt on every answer row and nowhere else', () => {
+ // Witness: the well-formed answer row parses with its receipt.
+ expect(parse([rows.legacyAnswer()]).attempts[0]?.downloadBytes).toBe(42)
+ const { downloadSha256: _sha, ...withoutSha } = rows.legacyAnswer()
+ const { downloadBytes: _bytes, ...withoutBytes } = rows.legacyAnswer()
+ for (const [label, row] of [
+ ['missing downloadSha256', withoutSha],
+ ['short downloadSha256', { ...rows.legacyAnswer(), downloadSha256: 'abc' }],
+ ['missing downloadBytes', withoutBytes],
+ ['negative downloadBytes', { ...rows.legacyAnswer(), downloadBytes: -1 }],
+ ['receipt on a shell row', { ...rows.shellRequested(), downloadSha256: DIGEST }],
+ ] as const) {
+ expect(() => parse([row]), label).toThrow(/downloadSha256|downloadBytes/)
+ }
+ })
+})
+
describe('appendedAttempts', () => {
it('refuses a row whose reason or delivered page changed between reads', () => {
const before = parse([rows.rejected(), rows.answer()])
@@ -136,6 +199,21 @@ describe('appendedAttempts', () => {
expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)
}
})
+
+ it('refuses a legacy-lease row whose receipt or failure code changed between reads', () => {
+ const before = parse([rows.legacyAnswer(), rows.shellFailed()])
+ // Witness: an unchanged prefix yields the appended row.
+ expect(
+ appendedAttempts(before, parse([rows.legacyAnswer(), rows.shellFailed(), rows.textOnly()]))
+ ).toEqual([rows.textOnly()])
+ for (const [label, changed] of [
+ ['sha', [{ ...rows.legacyAnswer(), downloadSha256: 'c'.repeat(64) }, rows.shellFailed()]],
+ ['bytes', [{ ...rows.legacyAnswer(), downloadBytes: 43 }, rows.shellFailed()]],
+ ['code', [rows.legacyAnswer(), { ...rows.shellFailed(), failureCode: 'writer_locked' }]],
+ ] as const) {
+ expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)
+ }
+ })
})
describe('rejection reason parity', () => {
@@ -153,4 +231,17 @@ describe('rejection reason parity', () => {
expect((declared as readonly string[]).length).toBeGreaterThan(0)
expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')
})
+
+ it('declares the same legacy-lease failure codes as the fixture, independently', async () => {
+ const providerModule = (await import(PROVIDER_MODULE_URL)) as {
+ LEGACY_LEASE_FAILURE_CODES: readonly string[]
+ }
+ const declared = (evidenceModule as Record<string, unknown>).FIXTURE_LEGACY_LEASE_FAILURE_CODES
+ expect(Array.isArray(declared)).toBe(true)
+ expect(declared).not.toBe(providerModule.LEGACY_LEASE_FAILURE_CODES)
+ expect([...(declared as readonly string[])].sort()).toEqual(
+ [...providerModule.LEGACY_LEASE_FAILURE_CODES].sort()
+ )
+ expect(declared as readonly string[]).toContain('download_busy')
+ })
})
diff --git a/desktop-app/test/legacyLeaseRestart.test.ts b/desktop-app/test/legacyLeaseRestart.test.ts
new file mode 100644
index 000000000..9c9d358e1
--- /dev/null
+++ b/desktop-app/test/legacyLeaseRestart.test.ts
@@ -0,0 +1,238 @@
+import { describe, expect, it } from 'vitest'
+import {
+ LEGACY_DISCARD_COUNTER,
+ downloadedRecord,
+ legacyLease,
+ legacyLeaseRecoveryVerdict,
+ parseMetricValue,
+ parseSeedSummary,
+ requireLegacyLeaseLaneEnv,
+ vacuityStoreVerdict,
+} from './e2e-playwright/helpers/legacyLeaseRestart'
+
+const PROFILE = 'clerum-fix-issue-1022-processing-lease-lifecycle-2e0feeb9'
+const RUN_ID = 'image-capabilities-0123456789ab'
+const SHA = 'a'.repeat(64)
+
+const discardLine = JSON.stringify({
+ level: 40,
+ component: 'GfsDownloadStore',
+ discarded: 1,
+ msg: 'GFS download store discarded 1 legacy processing lease(s) at initialize',
+})
+const unknownLine = JSON.stringify({
+ level: 40,
+ component: 'GfsDownloadStore',
+ legacyProcessingLeases: 1,
+ outcome: 'unknown',
+ msg: 'GFS download store could not confirm the discard of 1 legacy processing lease(s)',
+})
+const unrelatedLine = JSON.stringify({ level: 30, component: 'Server', msg: 'listening' })
+
+describe('requireLegacyLeaseLaneEnv', () => {
+ const valid = {
+ MINIKUBE_PROFILE: PROFILE,
+ CONTROL_API_REAL_PG_CONTEXT: PROFILE,
+ E2E_K8S_CONTEXT: PROFILE,
+ LEGACY_LEASE_LANE_MODE: 'fixed',
+ LEGACY_LEASE_SEED_LABEL_VALUE: RUN_ID,
+ IMAGE_CAPABILITIES_RUN_ID: RUN_ID,
+ }
+
+ it('accepts the runner bindings', () => {
+ expect(requireLegacyLeaseLaneEnv(valid)).toEqual({
+ profile: PROFILE,
+ mode: 'fixed',
+ runId: RUN_ID,
+ })
+ })
+
+ it('lists every inconsistent binding at once', () => {
+ let message = ''
+ try {
+ requireLegacyLeaseLaneEnv({
+ ...valid,
+ E2E_K8S_CONTEXT: 'minikube',
+ LEGACY_LEASE_LANE_MODE: 'skip',
+ LEGACY_LEASE_SEED_LABEL_VALUE: 'image-capabilities-ffffffffffff',
+ })
+ } catch (error) {
+ message = (error as Error).message
+ }
+ expect(message).toContain('E2E_K8S_CONTEXT')
+ expect(message).toContain('LEGACY_LEASE_LANE_MODE')
+ expect(message).toContain('LEGACY_LEASE_SEED_LABEL_VALUE')
+ })
+})
+
+describe('legacyLease', () => {
+ it('is a valid, unexpired, foreign lease with no records and no writer session', () => {
+ const now = Date.parse('2026-10-08T10:00:00.000Z')
+ const lease = legacyLease(now)
+ expect(lease.leaseId).toMatch(/^[0-9a-f-]{36}$/)
+ expect(lease.recordIds).toEqual([])
+ expect(lease.callerIdentity).not.toBe('')
+ expect(Date.parse(lease.expiresAt)).toBeGreaterThan(now)
+ expect(Date.parse(lease.expiresAt)).toBeGreaterThan(Date.parse(lease.acquiredAt))
+ expect(Object.hasOwn(lease, 'writerSessionId')).toBe(false)
+ })
+})
+
+describe('parseSeedSummary', () => {
+ const summary = (fields: Record<string, unknown>) =>
+ `LEGACY_LEASE_SEED ${JSON.stringify({ leaseId: 'l1', records: 0, mode: '600', uid: 1001, gid: 1001, ...fields })}\n`
+
+ it('reads the one summary of the seeded lease', () => {
+ expect(parseSeedSummary(summary({}), 'l1').records).toBe(0)
+ })
+
+ it('refuses a ledger left with another mode or owner', () => {
+ expect(() => parseSeedSummary(summary({ mode: '644' }), 'l1')).toThrow(/mode 644/)
+ expect(() => parseSeedSummary(summary({ uid: 0 }), 'l1')).toThrow(/owner 0:1001/)
+ })
+
+ it('refuses no summary, two summaries, or another lease', () => {
+ expect(() => parseSeedSummary('', 'l1')).toThrow(/0 summaries/)
+ expect(() => parseSeedSummary(summary({}) + summary({}), 'l1')).toThrow(/2 summaries/)
+ expect(() => parseSeedSummary(summary({}), 'l2')).toThrow(/another lease/)
+ })
+})
+
+describe('parseMetricValue', () => {
+ const exposition = [
+ `# HELP ${LEGACY_DISCARD_COUNTER} Legacy processing leases discarded.`,
+ `# TYPE ${LEGACY_DISCARD_COUNTER} counter`,
+ `${LEGACY_DISCARD_COUNTER} 1`,
+ 'process_cpu_seconds_total 3.2',
+ ].join('\n')
+
+ it('reads an unlabelled sample', () => {
+ expect(parseMetricValue(exposition, LEGACY_DISCARD_COUNTER)).toBe(1)
+ })
+
+ it('returns null when the metric is absent', () => {
+ expect(parseMetricValue('process_cpu_seconds_total 3.2\n', LEGACY_DISCARD_COUNTER)).toBeNull()
+ })
+
+ it('refuses duplicate or labelled samples', () => {
+ expect(() =>
+ parseMetricValue(`${exposition}\n${LEGACY_DISCARD_COUNTER} 2`, LEGACY_DISCARD_COUNTER)
+ ).toThrow(/2 samples/)
+ expect(() =>
+ parseMetricValue(`${LEGACY_DISCARD_COUNTER}{a="b"} 1`, LEGACY_DISCARD_COUNTER)
+ ).toThrow(/not an unlabelled number/)
+ })
+})
+
+describe('legacyLeaseRecoveryVerdict', () => {
+ const ledger = { schemaVersion: 1, records: {} }
+
+ it('accepts one confirmed discard with the counter at 1', () => {
+ expect(
+ legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine, discardLine], counter: 1 })
+ ).toBe('discarded')
+ })
+
+ it('accepts one unknown-outcome warning with the counter at 0', () => {
+ expect(legacyLeaseRecoveryVerdict({ ledger, logLines: [unknownLine], counter: 0 })).toBe(
+ 'unknown-outcome'
+ )
+ })
+
+ it('refuses a ledger that still carries processingLeases', () => {
+ expect(() =>
+ legacyLeaseRecoveryVerdict({
+ ledger: { ...ledger, processingLeases: {} },
+ logLines: [discardLine],
+ counter: 1,
+ })
+ ).toThrow(/still carries processingLeases/)
+ })
+
+ it('refuses when no outcome was logged (the Host never ran the discard)', () => {
+ expect(() =>
+ legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine], counter: 0 })
+ ).toThrow(/0 discard line\(s\).*0 unknown-outcome/)
+ })
+
+ it('refuses a discard whose counter disagrees, and a missing counter', () => {
+ expect(() =>
+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: 0 })
+ ).toThrow(/counter 0/)
+ expect(() =>
+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: null })
+ ).toThrow(/not exposed/)
+ })
+
+ it('refuses both outcomes at once, and a discard of more than one lease', () => {
+ expect(() =>
+ legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine, unknownLine], counter: 1 })
+ ).toThrow(/no recovery outcome/)
+ const two = discardLine.replace('"discarded":1', '"discarded":2')
+ expect(() => legacyLeaseRecoveryVerdict({ ledger, logLines: [two], counter: 2 })).toThrow(
+ /no recovery outcome/
+ )
+ })
+})
+
+describe('vacuityStoreVerdict', () => {
+ const ledger = { schemaVersion: 1, records: {}, processingLeases: { l1: {} } }
+
+ it('accepts the pre-fix Host keeping the lease without a counter', () => {
+ expect(
+ vacuityStoreVerdict({ ledger, logLines: [unrelatedLine], counter: null, leaseId: 'l1' })
+ ).toBe('lease-retained')
+ })
+
+ it('refuses a discarded lease, a discard line, or an exposed counter', () => {
+ expect(() =>
+ vacuityStoreVerdict({
+ ledger: { schemaVersion: 1, records: {} },
+ logLines: [],
+ counter: null,
+ leaseId: 'l1',
+ })
+ ).toThrow(/no longer carries/)
+ expect(() =>
+ vacuityStoreVerdict({ ledger, logLines: [discardLine], counter: null, leaseId: 'l1' })
+ ).toThrow(/logged a legacy-lease discard/)
+ expect(() => vacuityStoreVerdict({ ledger, logLines: [], counter: 0, leaseId: 'l1' })).toThrow(
+ /exposes/
+ )
+ })
+})
+
+describe('downloadedRecord', () => {
+ const record = (id: string, fields: Record<string, unknown>) => ({
+ id,
+ hostPath: `downloads/${id}`,
+ sizeBytes: 10,
+ sha256: SHA,
+ state: 'completed',
+ ...fields,
+ })
+ const before = { schemaVersion: 1, records: { old: record('old', {}) } }
+
+ it('returns the one new completed record with the source digest and size', () => {
+ const after = { schemaVersion: 1, records: { ...before.records, fresh: record('fresh', {}) } }
+ expect(downloadedRecord(before, after, { sha256: SHA, bytes: 10 }).id).toBe('fresh')
+ })
+
+ it('refuses no new record, a different digest, or an unfinished transfer', () => {
+ expect(() => downloadedRecord(before, before, { sha256: SHA, bytes: 10 })).toThrow(/added \[\]/)
+ const other = {
+ schemaVersion: 1,
+ records: { fresh: record('fresh', { sha256: 'b'.repeat(64) }) },
+ }
+ expect(() => downloadedRecord(before, other, { sha256: SHA, bytes: 10 })).toThrow(
+ /expected one/
+ )
+ const partial = {
+ schemaVersion: 1,
+ records: { fresh: record('fresh', { state: 'transferring' }) },
+ }
+ expect(() => downloadedRecord(before, partial, { sha256: SHA, bytes: 10 })).toThrow(
+ /expected one/
+ )
+ })
+})
diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.md
index 97103dfb3..1c7b4d75a 100644
--- a/docs/testing/minikube-t2-runbook.md
+++ b/docs/testing/minikube-t2-runbook.md
@@ -237,6 +237,105 @@ real-subscription interoperability or authorize real account use. The four
public targets and the guarded build body have hermetic boundary coverage in
`scripts/tests/test-minikube-mutation-boundary.sh`.
+### Legacy processing-lease restart lane
+
+Issue #1022. This lane proves that a Host booting on a GFS download store
+ledger that still carries a processing lease written before #1019 (no writer
+session) discards that lease and keeps serving shell commands and GFS downloads.
+It is development-only, uses the image-capabilities provider fixture, and has
+the same requirements as that lane: Node 24, a clean tree, the branch-owned
+`MINIKUBE_PROFILE` with a matching `CONTROL_API_REAL_PG_CONTEXT`, the mutation
+lease, and the single-Host (`mcp-host/chatllm`) profile.
+
+```sh
+make minikube-build-image-capabilities-fixture
+make minikube-t2
+make minikube-run-legacy-lease-restart
+```
+
+The Playwright config is
+`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts`.
+The lane runs five serial scenarios. Each one scales HCC and then the Host
+to 0. It waits until the Host pod is gone, and then seeds one legacy lease. The
+lease is valid and not expired, belongs to a foreign caller, has
+`recordIds: []` and has no `writerSessionId`. The seed pod is labelled
+`evenfire.ai/legacy-lease-seed=<run id>`. It runs the Host image as uid/gid
+1001 and mounts the Host PVC. It does a read-modify-write using the store's own
+publication sequence: a temp file with mode 0600, then fsync, rename, and an
+fsync of the directory. It refuses a ledger that already has leases. The pod
+reads the ledger back and is deleted before the scenario continues.
+
+| Scenario | How the Host is stopped | What runs after the seed |
+| --- | --- | --- |
+| `S-crash` | `delete pod --force --grace-period=0`, then an immediate scale to 0 | The Host and HCC return. The scenario fails with `CRASH_WINDOW_LOST` if a replacement pod started a container in between. |
+| `S-graceful` | Scale to 0 (SIGTERM, full grace) | `rollout restart` of the Host, then the Host and HCC return |
+| `S-hcc` | Scale to 0 | HCC and the Host return, so HCC is a new pod |
+| `S-gfs` | Scale to 0 | Each GFS pod restarts in turn (`gfsc-writer`, then `gfsc-reader`), then the Host and HCC return |
+| `S-update` | Scale to 0 | Upgrade order: GFS, then WRC, then HCC, then the Host |
+
+After each scenario, the lane checks the following:
+
+- **Pods.** The Host has a new pod UID and is Ready. HCC does not replace it
+ after it boots. The Host PVC has the same name and UID before and after.
+- **Store.** The served ledger has no `processingLeases`. The lane then needs
+ one of two outcomes:
+ - a `GfsDownloadStore` warning that 1 legacy lease was discarded, with
+ `clerum_gfs_legacy_processing_leases_discarded_total` at 1;
+ - one warning that the outcome is unknown, with that counter at 0.
+- **Journey.** The Desktop user sends the journey line and approves
+ `shell_exec` by clicking Approve. The fixture then requests
+ `clerum__gfs_download` for a seeded GFS file.
+ - The answer on screen and the fixture ledger must both carry the GFS
+ source's sha256 and byte count.
+ - The Host store must record exactly one new completed download with that
+ digest.
+ - The file in the Host workspace must hash to the same value.
+ - The journey makes no direct RPC calls and uses no storage shortcuts or
+ in-cluster mocks.
+
+**Vacuity.** Run the vacuity lane once, and run it before the fixed lane. The
+pre-fix Host quarantines every ledger record when it sees an inherited executor.
+Those records stay on the PVC and keep counting against the caller's quota. The
+fixed Host then discards the seeded lease when the runner restores it.
+
+```sh
+make minikube-build-legacy-lease-vacuity-fixture
+make minikube-run-legacy-lease-restart-vacuity
+```
+
+The build archives base `74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e` and checks
+that its store has the inherited-executor guard and no legacy-lease discard. It
+builds `clerum/mcp-host:legacy-lease-vacuity` from that archive and puts this
+HEAD's fixture layer on top as
+`clerum/image-capabilities-mcp-host:legacy-lease-vacuity`. It then writes
+`.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json` in the
+canonical checkout. The runner checks the profile, the base revision, the
+fixture-layer revision and the image identities against that manifest before it
+patches anything.
+
+The vacuity lane runs one `S-crash` scenario. It passes only when all of these
+hold:
+
+- the seeded lease is still in the ledger;
+- no discard line was logged and the counter does not exist;
+- the journey stops at the shell with
+ `LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy`, with no download
+ request.
+
+**Restoration.** The runner journals the original HCC and Host replica counts
+before Playwright starts. Zero is never a restored state. Restoration does the
+following:
+
+- deletes any seed pod of the run;
+- scales the Host and then HCC back;
+- waits for GFS, WRC, HCC and the Host to become Ready;
+- restores the production Host image;
+- waits until the Host's ledger has no `processingLeases`;
+- fails when a live `e2e-gfs-legacy-lease-*` GFS fixture remains.
+
+`make minikube-restore-image-capabilities` resumes an interrupted run the same
+way.
+
### Orphaned lock recovery
If `PROFILE_BUSY` reports that the lock has no valid owner PID, or that a stale
diff --git a/scripts/e2e/build-legacy-lease-vacuity-image.sh b/scripts/e2e/build-legacy-lease-vacuity-image.sh
new file mode 100755
index 000000000..54ceac335
--- /dev/null
+++ b/scripts/e2e/build-legacy-lease-vacuity-image.sh
@@ -0,0 +1,143 @@
+#!/usr/bin/env bash
+# Build the two images of the legacy processing-lease vacuity lane (issue #1022)
+# inside the owned minikube profile, under the inherited T2 mutation lease:
+#
+# clerum/mcp-host:legacy-lease-vacuity
+# mcp-host built from the pre-fix base revision (74e0d81d9, before #1019),
+# whose store treats a legacy lease as an inherited executor;
+# clerum/image-capabilities-mcp-host:legacy-lease-vacuity
+# the provider-fixture layer of THIS HEAD on top of that Host.
+#
+# It then writes the manifest `make minikube-run-legacy-lease-restart-vacuity`
+# proves before patching anything:
+# <canonical>/.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json
+#
+# Nothing is published. Run it through
+# `make minikube-build-legacy-lease-vacuity-fixture`.
+set -euo pipefail
+
+root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+profile="${MINIKUBE_PROFILE:?MINIKUBE_PROFILE is required}"
+base_revision=74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e
+base_image=clerum/mcp-host:legacy-lease-vacuity
+fixture_image=clerum/image-capabilities-mcp-host:legacy-lease-vacuity
+deadline="${root}/scripts/minikube/run-with-deadline.mjs"
+
+bash "${root}/scripts/minikube/require-t2-mutation-lock.sh"
+
+[[ "${profile}" =~ ^clerum-.+-[a-f0-9]{8}$ ]] || {
+ printf 'LEGACY_LEASE_VACUITY: %s is not a branch profile\n' "${profile}" >&2
+ exit 1
+}
+[[ -z "$(git -C "${root}" status --porcelain)" ]] || {
+ printf 'LEGACY_LEASE_VACUITY: the worktree must be clean (the fixture layer is built from HEAD)\n' >&2
+ exit 1
+}
+head_revision="$(git -C "${root}" rev-parse --verify HEAD)"
+[[ "$(git -C "${root}" cat-file -t "${base_revision}")" == commit ]] || {
+ printf 'LEGACY_LEASE_VACUITY: base revision %s is not in this repository\n' "${base_revision}" >&2
+ exit 1
+}
+common_dir="$(git -C "${root}" rev-parse --path-format=absolute --git-common-dir)"
+canonical="$(cd -- "$(dirname -- "${common_dir}")" && pwd -P)"
+[[ "$(basename -- "${canonical}")" == evenfire ]] || {
+ printf 'LEGACY_LEASE_VACUITY: canonical Evenfire checkout required\n' >&2
+ exit 1
+}
+
+run_dir="${canonical}/.local-notes/infra/runs/legacy-lease-vacuity/${profile}"
+source_dir="${run_dir}/base-${base_revision:0:12}"
+
+# Docker runs through the repository's isolated CLI environment (an empty task
+# config, no ambient credentials), pointed at the profile's own daemon with
+# `minikube docker-env`, exactly as scripts/minikube/build-images.sh does.
+# shellcheck source=scripts/minikube/docker-cli-env.sh
+source "${root}/scripts/minikube/docker-cli-env.sh"
+cleanup() {
+ local status=$? cleanup_status=0
+ trap - EXIT INT TERM
+ docker_cli_env_cleanup || cleanup_status=$?
+ rm -rf -- "${source_dir}"
+ if [[ "${status}" -eq 0 && "${cleanup_status}" -ne 0 ]]; then status="${cleanup_status}"; fi
+ exit "${status}"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+mkdir -p "${run_dir}"
+rm -rf -- "${source_dir}"
+mkdir -p "${source_dir}"
+git -C "${root}" archive --format=tar "${base_revision}" | tar -x -C "${source_dir}"
+
+# The archive must be the pre-fix store: it keeps the inherited-executor guard
+# and has no legacy-lease discard. Either check failing means a wrong base.
+store="${source_dir}/mcp-host/src/internalTools/gfsDownloadStore.ts"
+grep -q 'hasInheritedExecutors' "${store}" || {
+ printf 'LEGACY_LEASE_VACUITY: base store has no inherited-executor guard\n' >&2
+ exit 1
+}
+if grep -q 'discardLegacyProcessingLeases' "${store}"; then
+ printf 'LEGACY_LEASE_VACUITY: base store already discards legacy leases\n' >&2
+ exit 1
+fi
+
+docker_cli_env_prepare
+docker_env_output="$(node "${deadline}" --timeout-seconds 60 --kill-grace-seconds 5 --label minikube-docker-env -- \
+ minikube -p "${profile}" docker-env --shell bash)"
+[[ -n "${docker_env_output}" ]] || {
+ printf 'LEGACY_LEASE_VACUITY: minikube returned an empty Docker environment\n' >&2
+ exit 1
+}
+eval "${docker_env_output}"
+unset DOCKER_API_VERSION
+
+inspect() {
+ docker_cli_run_public "inspect-$1" 60 docker image inspect "$1" --format "$2" | tr -d '\r\n'
+}
+
+# The base Host image from the archived pre-fix tree. Its build context is the
+# archive root, which carries that revision's own root .dockerignore.
+docker_cli_run_public legacy-lease-vacuity-host-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \
+ docker build --tag "${base_image}" --file "${source_dir}/mcp-host/Dockerfile" "${source_dir}"
+base_id="$(inspect "${base_image}" '{{.Id}}')"
+
+# The provider-fixture layer of THIS HEAD on top of that Host.
+docker_cli_run_public legacy-lease-vacuity-fixture-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \
+ docker build --tag "${fixture_image}" \
+ --build-arg "MCP_HOST_IMAGE=${base_image}" \
+ --file "${root}/tests/e2e/fixtures/image-capabilities/Dockerfile" "${root}"
+fixture_id="$(inspect "${fixture_image}" '{{.Id}}')"
+
+for id in "${base_id}" "${fixture_id}"; do
+ [[ "${id}" =~ ^sha256:[a-f0-9]{64}$ ]] || {
+ printf 'LEGACY_LEASE_VACUITY: unexpected image id "%s"\n' "${id}" >&2
+ exit 1
+ }
+done
+# The fixture image must extend exactly the base just built.
+parent_layers="$(inspect "${base_image}" '{{json .RootFS.Layers}}')"
+fixture_layers="$(inspect "${fixture_image}" '{{json .RootFS.Layers}}')"
+[[ "${fixture_layers}" == "${parent_layers%]},"* ]] || {
+ printf 'LEGACY_LEASE_VACUITY: the fixture layer does not extend the pre-fix Host image\n' >&2
+ exit 1
+}
+
+manifest="${run_dir}/manifest.json"
+temporary="$(mktemp "${run_dir}/manifest.json.XXXXXX")"
+node -e '
+const [file, profile, baseRevision, head, baseImage, baseId, fixtureImage, fixtureId] = process.argv.slice(1)
+const manifest = {
+ kind: "legacy-lease-vacuity",
+ profile,
+ baseRevision,
+ fixtureLayerRevision: head,
+ images: { [baseImage]: baseId, [fixtureImage]: fixtureId },
+ derivedFrom: { [fixtureImage]: { ref: baseImage, id: baseId } },
+ builtAt: new Date().toISOString(),
+}
+require("node:fs").writeFileSync(file, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 })
+' "${temporary}" "${profile}" "${base_revision}" "${head_revision}" \
+ "${base_image}" "${base_id}" "${fixture_image}" "${fixture_id}"
+mv -f -- "${temporary}" "${manifest}"
+printf 'LEGACY_LEASE_VACUITY_BUILT manifest=%s base=%s fixture=%s\n' "${manifest}" "${base_id}" "${fixture_id}"
diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjs
index 91594cc2f..3544bc021 100644
--- a/scripts/e2e/image-capabilities-fixture.mjs
+++ b/scripts/e2e/image-capabilities-fixture.mjs
@@ -42,6 +42,69 @@ export const playwrightLanes = {
label: 'document-upload-playwright',
grep: 'document-upload fixture: ',
},
+ // Issue #1022: legacy processing-lease restart scenarios on the fixed Host.
+ 'legacy-lease-restart': {
+ config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',
+ label: 'legacy-lease-restart-playwright',
+ grep: 'legacy-lease-restart fixture: ',
+ // S-crash, S-graceful, S-hcc, S-gfs and S-update: a shorter list is a lost scenario.
+ expectedSpecs: 5,
+ },
+ // The same journey on a Host built from the pre-fix base revision: it must
+ // fail at the shell step with download_busy, or the lane proves nothing.
+ 'legacy-lease-restart-vacuity': {
+ config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',
+ label: 'legacy-lease-restart-vacuity-playwright',
+ grep: 'legacy-lease-restart vacuity: ',
+ expectedSpecs: 1,
+ },
+}
+
+/**
+ * The pre-fix revision the vacuity lane runs (#1022 base, before #1019), and
+ * the two images `scripts/e2e/build-legacy-lease-vacuity-image.sh` builds from
+ * it. They never share a tag with the HEAD images.
+ */
+export const legacyLeaseVacuity = Object.freeze({
+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
+ baseImage: 'clerum/mcp-host:legacy-lease-vacuity',
+ fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',
+ manifestKind: 'legacy-lease-vacuity',
+})
+
+export function legacyLeaseVacuityManifestPath(canonical, profile) {
+ return path.join(
+ canonical,
+ '.local-notes/infra/runs/legacy-lease-vacuity',
+ profile,
+ 'manifest.json'
+ )
+}
+
+/** Label of the seed pods the legacy-lease spec creates; restoration deletes them. */
+export const legacyLeaseSeedLabel = 'evenfire.ai/legacy-lease-seed'
+
+/**
+ * Runtime of each lane: the Host image HCC is pointed at, the Playwright
+ * deadline, and whether the lane restarts workloads (legacy-lease lanes scale
+ * HCC and the Host to zero and restart GFS and WRC, so restoration must bring
+ * them back).
+ */
+const laneRuntimes = {
+ image: { fixtureImage, timeoutSeconds: 900, legacyLease: null },
+ 'document-upload': { fixtureImage, timeoutSeconds: 900, legacyLease: null },
+ // Five scenarios, each with workload stop/start cycles and one Desktop launch.
+ 'legacy-lease-restart': { fixtureImage, timeoutSeconds: 3600, legacyLease: 'fixed' },
+ 'legacy-lease-restart-vacuity': {
+ fixtureImage: legacyLeaseVacuity.fixtureImage,
+ timeoutSeconds: 900,
+ legacyLease: 'vacuity',
+ },
+}
+
+export function laneRuntime(lane) {
+ if (!Object.hasOwn(laneRuntimes, lane)) throw new Error(`Unknown lane ${JSON.stringify(lane)}`)
+ return laneRuntimes[lane]
}
export function playwrightLaneConfig(env) {
@@ -73,7 +136,7 @@ const isCount = value => Number.isSafeInteger(value) && value >= 0
* test and no unexpected, skipped or flaky outcome.
*/
export function playwrightVerdict(report, { lane, configFile }) {
- const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
+ const { grep, expectedSpecs } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
const reported = report?.config?.configFile
if (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))
throw new Error('Playwright report config mismatch')
@@ -102,6 +165,10 @@ export function playwrightVerdict(report, { lane, configFile }) {
)
if (titles.some(title => !title.startsWith(grep)))
throw new Error(`Playwright report has specs outside the ${lane} lane`)
+ if (expectedSpecs !== undefined && titles.length !== expectedSpecs)
+ throw new Error(
+ `Playwright report lists ${titles.length} ${lane} specs, expected ${expectedSpecs}`
+ )
return 'PASS'
}
@@ -137,6 +204,53 @@ export function proveImages(manifest, observed, head, profile) {
return { head, profile, images: observed }
}
+/**
+ * Proves the vacuity images: the Host layer was built from the pre-fix base
+ * revision, the fixture layer from this HEAD, both for this profile, and the
+ * fixture layer derives from the exact base image present in the node.
+ */
+export function proveVacuityImages(manifest, observed, head, profile) {
+ const { baseRevision, baseImage: vacuityBase, fixtureImage: vacuityFixture } = legacyLeaseVacuity
+ if (
+ !/^[a-f0-9]{40}$/.test(head) ||
+ manifest?.kind !== legacyLeaseVacuity.manifestKind ||
+ manifest.profile !== profile
+ )
+ throw new Error('Vacuity image ownership mismatch')
+ if (manifest.baseRevision !== baseRevision)
+ throw new Error('Vacuity Host image was not built from the pre-fix base revision')
+ if (manifest.fixtureLayerRevision !== head)
+ throw new Error('Vacuity fixture layer was not built from this HEAD')
+ for (const ref of [vacuityBase, vacuityFixture]) {
+ const id = manifest.images?.[ref]
+ if (!/^sha256:[a-f0-9]{64}$/.test(id ?? '') || id !== observed[ref])
+ throw new Error('Vacuity image identity mismatch')
+ }
+ const binding = manifest.derivedFrom?.[vacuityFixture]
+ if (binding?.ref !== vacuityBase || binding.id !== observed[vacuityBase])
+ throw new Error('Vacuity fixture derived-base mismatch')
+ return { head, profile, baseRevision, images: observed }
+}
+
+/**
+ * The legacy-lease lanes scale HCC and the Host. Each must be back at its
+ * recorded replica count; zero is never a restored state.
+ */
+export function legacyLeaseReplicaPlan(recorded, live) {
+ const plan = []
+ // The Host first: HCC sets the Host replicas itself, and a Host already at
+ // its count makes HCC's first reconcile a no-op.
+ for (const key of ['host', 'hcc']) {
+ const want = recorded?.[key]
+ if (!Number.isSafeInteger(want) || want < 1)
+ throw new Error(`Recorded ${key} replicas must be a positive integer`)
+ if (!Number.isSafeInteger(live?.[key]) || live[key] < 0)
+ throw new Error(`Live ${key} replicas are unreadable`)
+ if (live[key] !== want) plan.push({ key, replicas: want })
+ }
+ return plan
+}
+
export function modelInputs(runId) {
if (!/^image-capabilities-[a-f0-9]{12}$/.test(runId)) throw new Error('Invalid fixture run')
return [
@@ -256,6 +370,7 @@ async function main() {
label: laneLabel,
grep: laneGrep,
} = selectFixtureAction(process.argv[2], process.env)
+ const runtime = action === 'run' ? laneRuntime(lane) : null
const head = command('git', ['rev-parse', 'HEAD']).trim()
const branch = command('git', ['branch', '--show-current']).trim()
if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')
@@ -339,6 +454,8 @@ async function main() {
profile,
head,
root,
+ fixtureImage: runtime.fixtureImage,
+ legacyLease: runtime.legacyLease,
rows: [],
changes: {},
restored: false,
@@ -350,6 +467,10 @@ async function main() {
(action === 'run' && state.head !== head)
)
throw new Error('Run state identity mismatch')
+ // Restore reads the lane's Host image from the persisted state, never from the
+ // environment, so a stale lane variable cannot point restoration elsewhere.
+ if (typeof state.fixtureImage !== 'string' || !state.fixtureImage)
+ throw new Error('Run state has no fixture image; restore it with the runner that created it')
const save = () => {
const destination = path.join(evidence, 'state.json')
if (fs.existsSync(destination)) {
@@ -424,8 +545,86 @@ async function main() {
if (!response.ok) throw new Error(`Fixture admin operation failed (${response.status})`)
return response.status === 204 ? null : response.json()
}
+ const deploymentReplicas = (namespace, name) => get(namespace, 'deployment', name).spec.replicas
+ const restoreLegacyLeaseWorkloads = () => {
+ // Seed pods mount the Host PVC; none may outlive the run.
+ kc([
+ '-n',
+ 'mcp-host',
+ 'delete',
+ 'pod',
+ '-l',
+ `${legacyLeaseSeedLabel}=${state.runId}`,
+ '--ignore-not-found',
+ '--wait=true',
+ '--timeout=90s',
+ ])
+ const remaining = kc([
+ '-n',
+ 'mcp-host',
+ 'get',
+ 'pod',
+ '-l',
+ legacyLeaseSeedLabel,
+ '-o',
+ 'jsonpath={.items[*].metadata.name}',
+ ]).trim()
+ if (remaining) throw new Error('Legacy-lease seed pods remain in mcp-host')
+ const liveReplicas = () => ({
+ host: deploymentReplicas('mcp-host', 'chatllm'),
+ hcc: deploymentReplicas('control-plane', 'host-context-controller'),
+ })
+ for (const { key, replicas } of legacyLeaseReplicaPlan(
+ state.legacyLeaseReplicas,
+ liveReplicas()
+ )) {
+ const [namespace, name] =
+ key === 'host' ? ['mcp-host', 'chatllm'] : ['control-plane', 'host-context-controller']
+ kc(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])
+ }
+ for (const [namespace, name] of [
+ ['gfs', 'gfsc-writer'],
+ ['gfs', 'gfsc-reader'],
+ ['control-plane', 'workflow-recipes'],
+ ['control-plane', 'host-context-controller'],
+ ['mcp-host', 'chatllm'],
+ ])
+ kc(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=150s'])
+ if (legacyLeaseReplicaPlan(state.legacyLeaseReplicas, liveReplicas()).length)
+ throw new Error('HCC or the Host is not back at its recorded replicas')
+ }
+ // The Host discards a legacy lease at initialize, which runs at boot and can
+ // wait out writer contention, so the ledger is polled with a bound.
+ const waitLedgerWithoutLegacyLeases = async () => {
+ const until = Date.now() + 240_000
+ let last = 'unread'
+ while (Date.now() < until) {
+ try {
+ const ledger = JSON.parse(
+ kc([
+ '-n',
+ 'mcp-host',
+ 'exec',
+ 'deployment/chatllm',
+ '-c',
+ 'mcp-host',
+ '--',
+ 'cat',
+ '/workspace/.gfs-download-store/ledger-v1.json',
+ ])
+ )
+ if (!Object.hasOwn(ledger, 'processingLeases')) return
+ last = 'processingLeases present'
+ } catch (error) {
+ last = error.message
+ }
+ await new Promise(resolve => setTimeout(resolve, 2_000))
+ }
+ throw new Error(`Restored Host ledger still carries legacy processing leases (${last})`)
+ }
const restoreRuntime = async () => {
if (state.restored) return
+ if (state.legacyLease && state.legacyLeaseReplicas) restoreLegacyLeaseWorkloads()
// Restore the model before removing its fixture-only catalog rows.
if (state.changes.host) {
const current = get('mcp-host', 'host', 'chatllm')
@@ -500,7 +699,7 @@ async function main() {
path: state.hcc.path.replace(/value$/, 'name'),
value: 'CONTEXT_MAPPER_HOST_IMAGE',
},
- { op: 'test', path: state.hcc.path, value: fixtureImage },
+ { op: 'test', path: state.hcc.path, value: state.fixtureImage },
{ op: 'replace', path: state.hcc.path, value: state.hcc.image },
unmark,
])
@@ -582,6 +781,9 @@ async function main() {
'process.stdout.write(String(!process.env.EVENFIRE_IMAGE_CAPABILITIES_FIXTURE && !require("node:fs").existsSync("/tmp/image-capabilities-evidence.json")))',
]).trim()
if (predicate !== 'true') throw new Error('Fixture state remains in Host')
+ // The vacuity lane leaves its seeded lease on the PVC (the pre-fix Host
+ // cannot discard it); the restored fixed Host must discard it at boot.
+ if (state.legacyLease) await waitLedgerWithoutLegacyLeases()
state.restored = true
save()
}
@@ -596,11 +798,20 @@ async function main() {
}
let failure
try {
+ const vacuity = runtime.legacyLease === 'vacuity'
const manifest = JSON.parse(
- fs.readFileSync(path.join(root, 'deploy/minikube/.image-manifest.json'), 'utf8')
+ fs.readFileSync(
+ vacuity
+ ? legacyLeaseVacuityManifestPath(canonical, profile)
+ : path.join(root, 'deploy/minikube/.image-manifest.json'),
+ 'utf8'
+ )
)
+ const proofRefs = vacuity
+ ? [legacyLeaseVacuity.baseImage, legacyLeaseVacuity.fixtureImage]
+ : [baseImage, fixtureImage]
const observed = Object.fromEntries(
- [baseImage, fixtureImage].map(ref => [
+ proofRefs.map(ref => [
ref,
command('minikube', [
'-p',
@@ -616,7 +827,9 @@ async function main() {
]).trim(),
])
)
- state.imageProof = proveImages(manifest, observed, head, profile)
+ state.imageProof = vacuity
+ ? proveVacuityImages(manifest, observed, head, profile)
+ : proveImages(manifest, observed, head, profile)
save()
const host = get('mcp-host', 'host', 'chatllm')
const hosts = JSON.parse(kc(['-n', 'mcp-host', 'get', 'hosts', '-o', 'json']))
@@ -772,7 +985,7 @@ async function main() {
value: 'CONTEXT_MAPPER_HOST_IMAGE',
},
{ op: 'test', path: state.hcc.path, value: state.hcc.image },
- { op: 'replace', path: state.hcc.path, value: fixtureImage },
+ { op: 'replace', path: state.hcc.path, value: state.fixtureImage },
])
kc([
'-n',
@@ -782,7 +995,7 @@ async function main() {
'deployment/host-context-controller',
'--timeout=120s',
])
- await waitHostImage(fixtureImage)
+ await waitHostImage(state.fixtureImage)
const wire = JSON.parse(
kc([
'-n',
@@ -811,6 +1024,26 @@ async function main() {
QA_RECORDER_IMAGE_MODEL_UNSUPPORTED: 'glm-5.3',
QA_RECORDER_IMAGE_MODEL_UNKNOWN: models[2].model,
QA_RECORDER_ROOT: evidence,
+ ...(runtime.legacyLease
+ ? {
+ // The GFS fixture helpers name the cluster through this variable.
+ E2E_K8S_CONTEXT: profile,
+ LEGACY_LEASE_LANE_MODE: runtime.legacyLease,
+ LEGACY_LEASE_SEED_LABEL_VALUE: state.runId,
+ }
+ : {}),
+ }
+ if (runtime.legacyLease) {
+ // The spec scales these to zero; restoration needs the values it found.
+ const replicas = {
+ host: deploymentReplicas('mcp-host', 'chatllm'),
+ hcc: deploymentReplicas('control-plane', 'host-context-controller'),
+ }
+ // Validates the recorded counts: a workload already at zero is refused
+ // here, before the spec runs, instead of becoming the restore target.
+ legacyLeaseReplicaPlan(replicas, replicas)
+ state.legacyLeaseReplicas = replicas
+ save()
}
command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])
process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)
@@ -826,7 +1059,7 @@ async function main() {
[
'scripts/minikube/run-with-deadline.mjs',
'--timeout-seconds',
- '900',
+ String(runtime.timeoutSeconds),
'--heartbeat-seconds',
'20',
'--kill-grace-seconds',
@@ -847,7 +1080,7 @@ async function main() {
...environment,
PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),
},
- timeout: 930_000,
+ timeout: (runtime.timeoutSeconds + 30) * 1000,
}
)
report(output)
@@ -875,7 +1108,9 @@ async function main() {
try {
const current = get('mcp-host', 'deployment', 'chatllm')
if (
- current.spec.template.spec.containers.some(container => container.image === fixtureImage)
+ current.spec.template.spec.containers.some(
+ container => container.image === state.fixtureImage
+ )
) {
const logs = kc(['-n', 'mcp-host', 'logs', 'deployment/chatllm', '--tail=60'])
state.startupDiagnostic = [
@@ -902,6 +1137,32 @@ async function main() {
)
}
uninstallSignalRestore()
+ if (!failure && state.legacyLease) {
+ // The spec removes its GFS fixtures; a live row left behind fails the run.
+ try {
+ const live = kc([
+ '-n',
+ 'control-plane',
+ 'exec',
+ 'deployment/control-postgres',
+ '--',
+ 'psql',
+ '-v',
+ 'ON_ERROR_STOP=1',
+ '-U',
+ 'postgres',
+ '-d',
+ 'profiles',
+ '-t',
+ '-A',
+ '-c',
+ "SELECT count(*) FROM gfs_resources WHERE drive = 'main' AND deleted_at IS NULL AND path_cache LIKE '/e2e-gfs-legacy-lease-%';",
+ ]).trim()
+ if (live !== '0') failure = new Error(`Legacy-lease GFS fixture rows remain (${live})`)
+ } catch (error) {
+ failure = new Error(`Legacy-lease GFS fixture check failed: ${error.message}`)
+ }
+ }
if (failure) throw failure
process.stdout.write(
fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })
diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjs
index 13ae290e9..0e81c46f5 100644
--- a/scripts/e2e/image-capabilities-fixture.test.mjs
+++ b/scripts/e2e/image-capabilities-fixture.test.mjs
@@ -8,10 +8,15 @@ import {
fixtureImage,
fixturePassLine,
installSignalRestore,
+ laneRuntime,
+ legacyLeaseReplicaPlan,
+ legacyLeaseVacuity,
+ legacyLeaseVacuityManifestPath,
modelInputs,
playwrightLaneConfig,
playwrightVerdict,
proveImages,
+ proveVacuityImages,
requireOwnedResource,
runAnnotation,
sanitizeFixtureReport,
@@ -352,3 +357,190 @@ test('the playwright log redacts the admin password, the session cookie and bear
assert.ok(!report.includes('opaque-bearer-value'))
assert.match(report, /3 passed \(41\.2s\)/)
})
+
+// ---------------------------------------------------------------------------
+// Issue #1022: legacy processing-lease restart lanes.
+// ---------------------------------------------------------------------------
+
+test('the legacy-lease lanes share one config and select disjoint journeys', () => {
+ const fixed = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart' })
+ const vacuity = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart-vacuity' })
+ assert.equal(
+ fixed.config,
+ 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts'
+ )
+ assert.equal(vacuity.config, fixed.config)
+ assert.equal(fixed.grep, 'legacy-lease-restart fixture: ')
+ assert.equal(vacuity.grep, 'legacy-lease-restart vacuity: ')
+ assert.equal(vacuity.grep.startsWith(fixed.grep), false)
+ assert.equal(fixed.grep.startsWith(vacuity.grep), false)
+ const options = {
+ lane: 'legacy-lease-restart',
+ configFile: laneConfigFile('legacy-lease-restart'),
+ }
+ const scenarioTitles = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'].map(
+ scenario => `legacy-lease-restart fixture: ${scenario}`
+ )
+ // Witness: the lane's own five scenario titles are PASS.
+ assert.equal(
+ playwrightVerdict(
+ playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles }),
+ options
+ ),
+ 'PASS'
+ )
+ // A clean report that lost scenarios is not the lane.
+ assert.throws(
+ () =>
+ playwrightVerdict(
+ playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles.slice(0, 2) }),
+ options
+ ),
+ /lists 2 legacy-lease-restart specs, expected 5/
+ )
+ // The vacuity lane is exactly its one S-crash journey.
+ assert.equal(
+ playwrightVerdict(
+ playwrightReport({
+ lane: 'legacy-lease-restart-vacuity',
+ titles: ['legacy-lease-restart vacuity: S-crash'],
+ }),
+ {
+ lane: 'legacy-lease-restart-vacuity',
+ configFile: laneConfigFile('legacy-lease-restart-vacuity'),
+ }
+ ),
+ 'PASS'
+ )
+ // A vacuity title in the fixed lane's report is outside that lane.
+ assert.throws(
+ () =>
+ playwrightVerdict(
+ playwrightReport({
+ lane: 'legacy-lease-restart',
+ titles: ['legacy-lease-restart vacuity: S-crash'],
+ }),
+ options
+ ),
+ /outside the legacy-lease-restart lane/
+ )
+})
+
+test('each lane runtime names its Host image and deadline; existing lanes are unchanged', () => {
+ for (const lane of ['image', 'document-upload']) {
+ assert.deepEqual(laneRuntime(lane), { fixtureImage, timeoutSeconds: 900, legacyLease: null })
+ }
+ assert.deepEqual(laneRuntime('legacy-lease-restart'), {
+ fixtureImage,
+ timeoutSeconds: 3600,
+ legacyLease: 'fixed',
+ })
+ assert.deepEqual(laneRuntime('legacy-lease-restart-vacuity'), {
+ fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',
+ timeoutSeconds: 900,
+ legacyLease: 'vacuity',
+ })
+ // The vacuity images never reuse a HEAD tag.
+ assert.notEqual(legacyLeaseVacuity.fixtureImage, fixtureImage)
+ assert.notEqual(legacyLeaseVacuity.baseImage, baseImage)
+ for (const lane of ['typo', '__proto__', '']) {
+ assert.throws(() => laneRuntime(lane), /Unknown lane/, lane)
+ }
+})
+
+test('the vacuity manifest lives under the canonical run root, per profile', () => {
+ assert.equal(
+ legacyLeaseVacuityManifestPath('/repo/evenfire', profile),
+ `/repo/evenfire/.local-notes/infra/runs/legacy-lease-vacuity/${profile}/manifest.json`
+ )
+})
+
+const vacuityIds = {
+ [legacyLeaseVacuity.baseImage]: `sha256:${'e'.repeat(64)}`,
+ [legacyLeaseVacuity.fixtureImage]: `sha256:${'f'.repeat(64)}`,
+}
+function vacuityManifest() {
+ return {
+ kind: 'legacy-lease-vacuity',
+ profile,
+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
+ fixtureLayerRevision: head,
+ images: { ...vacuityIds },
+ derivedFrom: {
+ [legacyLeaseVacuity.fixtureImage]: {
+ ref: legacyLeaseVacuity.baseImage,
+ id: vacuityIds[legacyLeaseVacuity.baseImage],
+ },
+ },
+ }
+}
+
+test('vacuity images must come from the pre-fix base and this HEAD, for this profile', () => {
+ // Witness: the well-formed manifest is proven.
+ assert.deepEqual(proveVacuityImages(vacuityManifest(), vacuityIds, head, profile), {
+ head,
+ profile,
+ baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
+ images: vacuityIds,
+ })
+ for (const [label, mutate, pattern] of [
+ ['other kind', value => (value.kind = 'image-manifest'), /ownership/],
+ ['other profile', value => (value.profile = 'clerum-other-12345678'), /ownership/],
+ ['built from HEAD', value => (value.baseRevision = head), /pre-fix base/],
+ ['stale fixture layer', value => (value.fixtureLayerRevision = 'd'.repeat(40)), /this HEAD/],
+ [
+ 'other base id',
+ value => (value.images[legacyLeaseVacuity.baseImage] = `sha256:${'1'.repeat(64)}`),
+ /identity/,
+ ],
+ [
+ 'fixture from another base',
+ value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].id = `sha256:${'2'.repeat(64)}`),
+ /derived-base/,
+ ],
+ [
+ 'fixture from the HEAD Host',
+ value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].ref = baseImage),
+ /derived-base/,
+ ],
+ ]) {
+ const changed = vacuityManifest()
+ mutate(changed)
+ assert.throws(() => proveVacuityImages(changed, vacuityIds, head, profile), pattern, label)
+ }
+ // The node holds a different image than the manifest recorded.
+ assert.throws(
+ () =>
+ proveVacuityImages(
+ vacuityManifest(),
+ { ...vacuityIds, [legacyLeaseVacuity.fixtureImage]: `sha256:${'3'.repeat(64)}` },
+ head,
+ profile
+ ),
+ /identity/
+ )
+ // A HEAD image manifest is never accepted as a vacuity proof.
+ assert.throws(() => proveVacuityImages(manifest(), ids, head, profile), /ownership/)
+})
+
+test('restoration scales the Host before HCC and never accepts zero as restored', () => {
+ const recorded = { host: 1, hcc: 1 }
+ // Witness: both at zero yields a plan, Host first.
+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 0, hcc: 0 }), [
+ { key: 'host', replicas: 1 },
+ { key: 'hcc', replicas: 1 },
+ ])
+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 0 }), [
+ { key: 'hcc', replicas: 1 },
+ ])
+ assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 1 }), [])
+ for (const [label, bad] of [
+ ['zero host', { host: 0, hcc: 1 }],
+ ['zero hcc', { host: 1, hcc: 0 }],
+ ['missing', {}],
+ ['fractional', { host: 1.5, hcc: 1 }],
+ ]) {
+ assert.throws(() => legacyLeaseReplicaPlan(bad, { host: 1, hcc: 1 }), /Recorded/, label)
+ }
+ assert.throws(() => legacyLeaseReplicaPlan(recorded, { host: undefined, hcc: 1 }), /unreadable/)
+})
diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.md
index b3a207519..624c1216b 100644
--- a/tests/e2e/fixtures/image-capabilities/README.md
+++ b/tests/e2e/fixtures/image-capabilities/README.md
@@ -55,6 +55,39 @@ file chip, sends, and requires the digest of the attached bytes in the answer,
one `document-read-requested` row, one `document-answer` row carrying that
digest, no image row and no `documentFailures`.
+## Legacy processing-lease restart journey (issue #1022)
+
+`make minikube-run-legacy-lease-restart` is the same runner with
+`IMAGE_CAPABILITIES_LANE=legacy-lease-restart`; the config is
+`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts` and
+the cluster side lives in `helpers/legacyLeaseRestart.ts`. The user message
+carries one line:
+
+```
+LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>
+```
+
+The fixture then scripts the turns from the message list alone: it asks for
+`shell_exec` with a command whose output is `<marker>-shell-ok` (a token absent
+from the command), then for `clerum__gfs_download` of the named resource once
+the shell result echoes the token, and answers
+`LEGACY-LEASE-FIXTURE-OK marker=… sha256=… bytes=…` from the download receipt.
+A shell or download result without what the next step needs is answered with
+`LEGACY-LEASE-FIXTURE-SHELL-FAILED code=<code>` or
+`LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=<code>`, where the code is a GFS
+download store error code (for example `download_busy`) or `unrecognized`.
+`download_busy` is what the pre-fix Host returns for a legacy lease. The ledger
+rows are `legacy-lease-shell-requested`, `legacy-lease-download-requested`,
+`legacy-lease-answer` (with `downloadSha256` and `downloadBytes`),
+`legacy-lease-shell-failed` and `legacy-lease-download-failed` (with
+`failureCode`). It never records command output.
+
+The vacuity lane, `make minikube-run-legacy-lease-restart-vacuity`, runs the same
+fixture layer on a Host built from the pre-fix base by
+`make minikube-build-legacy-lease-vacuity-fixture`. The procedure and the
+signals checked after each scenario are in
+`docs/testing/minikube-t2-runbook.md` (Legacy processing-lease restart lane).
+
## Local execution
Use Node 24 and Bash 5, a clean development branch based on current `origin/dev`,
diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjs
index 85bf71b60..1ecd30441 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs
@@ -160,6 +160,12 @@ export const FIXTURE_REJECTION_REASONS = Object.freeze([
'document-tool-output-unreadable',
'document-page-range-malformed',
'document-stream-unsupported',
+ 'legacy-lease-shell-not-offered',
+ 'legacy-lease-download-not-offered',
+ 'legacy-lease-tool-result-unexpected',
+ 'legacy-lease-tool-result-count',
+ 'legacy-lease-download-result-malformed',
+ 'legacy-lease-stream-unsupported',
])
/** A refusal with a stable machine-readable reason code for the evidence tally. */
@@ -512,6 +518,187 @@ function classifyDocumentTurn(body) {
}
}
+/*
+ * Legacy processing-lease restart journey (issue #1022, `make
+ * minikube-run-legacy-lease-restart`). The user message carries one line
+ *
+ * LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>
+ *
+ * and the fixture scripts two tool calls from the message list alone:
+ *
+ * 1. no tool result yet: call `shell_exec` with a command whose output is
+ * `<marker>-shell-ok`. That token does not appear in the command, so only
+ * a command that really ran can echo it back;
+ * 2. one shell result carrying that token: call `clerum__gfs_download` for the
+ * named resource;
+ * 3. a shell result and a download result: answer with the sha256 and byte
+ * count the Host's download receipt reports.
+ *
+ * A shell or download result that does not carry what step 2 or 3 needs is
+ * answered with a failure line naming a closed-set code (a store error code such
+ * as `download_busy`, or `unrecognized`), so the journey fails on screen at the
+ * step that broke. The ledger records only the step, the code, and for the
+ * answer the receipt's digest and byte count, never the command output.
+ */
+const SHELL_TOOL = 'shell_exec'
+const GFS_DOWNLOAD_TOOL = 'clerum__gfs_download'
+const LEGACY_LEASE_JOURNEY_LINE =
+ /LEGACY_LEASE_JOURNEY marker=(legacy-lease-[a-f0-9]{16}) drive=([a-z0-9][a-z0-9-]{0,62}) resourceId=([a-f0-9]{32})/
+const LEGACY_LEASE_SHELL_TOKEN_SUFFIX = '-shell-ok'
+export const LEGACY_LEASE_ANSWER_PREFIX = 'LEGACY-LEASE-FIXTURE-OK'
+export const LEGACY_LEASE_SHELL_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-SHELL-FAILED'
+export const LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED'
+
+/**
+ * Codes a failed shell or download step may report: the GFS download store's
+ * error codes (`GfsDownloadStoreErrorCode`), plus `unrecognized` for a result
+ * that names none of them. A closed set of fixed strings, never tool output.
+ */
+export const LEGACY_LEASE_FAILURE_CODES = Object.freeze([
+ 'caller_mismatch',
+ 'caller_quota_exceeded',
+ 'corrupt_store_ledger',
+ 'download_expired',
+ 'download_missing',
+ 'download_busy',
+ 'host_quota_exceeded',
+ 'publication_cancelled',
+ 'storage_write_failed',
+ 'unsupported_store_schema',
+ 'workspace_unavailable',
+ 'writer_locked',
+ 'unrecognized',
+])
+
+/** The store error text (`GfsDownloadStoreError`): `GFS download store failed (<code>)`. */
+const STORE_FAILURE = /GFS download store failed \(([a-z_]+)\)/
+
+function toolOutputPattern(name) {
+ return new RegExp(
+ `^<tool_output name="${name}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`
+ )
+}
+const SHELL_TOOL_OUTPUT = toolOutputPattern(SHELL_TOOL)
+const GFS_DOWNLOAD_TOOL_OUTPUT = toolOutputPattern(GFS_DOWNLOAD_TOOL)
+
+class FixtureLegacyLeaseError extends Error {
+ constructor(code, message) {
+ super(message)
+ this.name = 'FixtureLegacyLeaseError'
+ this.code = code
+ }
+}
+
+function failureCode(content) {
+ const code = STORE_FAILURE.exec(content)?.[1]
+ return code && LEGACY_LEASE_FAILURE_CODES.includes(code) ? code : 'unrecognized'
+}
+
+function offeredTool(body, name) {
+ const tools = Array.isArray(body.tools) ? body.tools : []
+ return tools.some(tool => tool?.function?.name === name)
+}
+
+/**
+ * Classifies a legacy-lease journey turn.
+ *
+ * - `null`: the last user message carries no journey line, or no tool is offered
+ * (a title or summary request), so the request is not part of the journey.
+ * - `{ stage: 'shell', marker }`: ask for the shell command.
+ * - `{ stage: 'download', drive, resourceId }`: the shell echoed the token.
+ * - `{ stage: 'shell-failed', code }`: the shell result lacks the token.
+ * - `{ stage: 'answer', marker, sha256, sizeBytes }`: the download receipt.
+ * - `{ stage: 'download-failed', code }`: the download result is an error.
+ */
+function classifyLegacyLeaseTurn(body) {
+ if (!Array.isArray(body.tools) || body.tools.length === 0) return null
+ const userIndex = lastUserIndex(body.messages)
+ if (userIndex < 0) return null
+ const user = body.messages[userIndex]
+ if (typeof user.content !== 'string') return null
+ const journey = LEGACY_LEASE_JOURNEY_LINE.exec(user.content)
+ if (!journey) return null
+ const [, marker, drive, resourceId] = journey
+
+ const toolMessages = body.messages
+ .slice(userIndex + 1)
+ .filter(message => message?.role === 'tool')
+ if (toolMessages.length > 2)
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-tool-result-count',
+ `the journey makes two tool calls, received ${toolMessages.length} results`
+ )
+ const contents = toolMessages.map(message => message.content)
+ if (contents.some(content => typeof content !== 'string'))
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-tool-result-unexpected',
+ 'a tool result is not a string'
+ )
+
+ if (contents.length === 0) {
+ if (!offeredTool(body, SHELL_TOOL))
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-shell-not-offered',
+ 'the Host did not offer shell_exec for the journey'
+ )
+ return { stage: 'shell', marker }
+ }
+
+ const shell = SHELL_TOOL_OUTPUT.exec(contents[0])
+ if (!shell)
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-tool-result-unexpected',
+ 'the first tool result is not a shell_exec result'
+ )
+ const token = `${marker}${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}`
+ const echoed = shell[1].split('\n').some(line => line.trim() === token)
+ if (contents.length === 1) {
+ if (!echoed) return { stage: 'shell-failed', code: failureCode(shell[1]) }
+ if (!offeredTool(body, GFS_DOWNLOAD_TOOL))
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-download-not-offered',
+ 'the Host did not offer clerum__gfs_download after the shell step'
+ )
+ return { stage: 'download', drive, resourceId }
+ }
+ if (!echoed)
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-tool-result-unexpected',
+ 'a download result follows a shell result without the token'
+ )
+
+ const download = GFS_DOWNLOAD_TOOL_OUTPUT.exec(contents[1])
+ if (!download)
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-tool-result-unexpected',
+ 'the second tool result is not a clerum__gfs_download result'
+ )
+ let receipt
+ try {
+ receipt = JSON.parse(download[1])
+ } catch {
+ return { stage: 'download-failed', code: failureCode(download[1]) }
+ }
+ if (
+ !receipt ||
+ typeof receipt !== 'object' ||
+ receipt.delivery !== 'workspace_file' ||
+ typeof receipt.sha256 !== 'string' ||
+ !SHA256_HEX.test(receipt.sha256) ||
+ !isByteCount(receipt.sizeBytes)
+ )
+ throw new FixtureLegacyLeaseError(
+ 'legacy-lease-download-result-malformed',
+ 'the download result is not a workspace_file receipt with sha256 and sizeBytes'
+ )
+ return {
+ stage: 'answer',
+ marker,
+ sha256: receipt.sha256.toLowerCase(),
+ sizeBytes: receipt.sizeBytes,
+ }
+}
+
const isByteCount = value => Number.isSafeInteger(value) && value >= 0
const INTERNAL_HOST_SUFFIXES = [
@@ -758,6 +945,10 @@ export function createImageFixtureFetch(
documentReadRequests: 0,
documentAnswers: 0,
documentFailures: 0,
+ legacyLeaseShellRequests: 0,
+ legacyLeaseDownloadRequests: 0,
+ legacyLeaseAnswers: 0,
+ legacyLeaseFailures: 0,
},
attempts: [],
}
@@ -911,6 +1102,74 @@ export function createImageFixtureFetch(
}
)
}
+
+ let legacyTurn
+ try {
+ legacyTurn = classifyLegacyLeaseTurn(body)
+ } catch (error) {
+ if (!(error instanceof FixtureLegacyLeaseError)) throw error
+ state.counters.legacyLeaseFailures += 1
+ return refuse(model, error.code, { code: error.code })
+ }
+ if (legacyTurn) {
+ if (stream) {
+ // A tool call cannot be answered in this fixture's SSE, exactly as for
+ // the document read.
+ state.counters.legacyLeaseFailures += 1
+ return refuse(model, 'legacy-lease-stream-unsupported', {
+ code: 'legacy_lease_stream_unsupported',
+ })
+ }
+ sequence += 1
+ const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
+ const reply = (responseKind, content, fields = {}) => {
+ recordCall(model, null, responseKind, fields)
+ publish()
+ return jsonResponse(completionBody(completionId, model, content))
+ }
+ const call = (responseKind, name, args) => {
+ recordCall(model, null, responseKind)
+ publish()
+ return jsonResponse(toolCallCompletionBody(completionId, model, name, args))
+ }
+ switch (legacyTurn.stage) {
+ case 'shell':
+ state.counters.legacyLeaseShellRequests += 1
+ return call('legacy-lease-shell-requested', SHELL_TOOL, {
+ command: `printf '%s${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}\\n' ${legacyTurn.marker}`,
+ })
+ case 'download':
+ state.counters.legacyLeaseDownloadRequests += 1
+ return call('legacy-lease-download-requested', GFS_DOWNLOAD_TOOL, {
+ drive: legacyTurn.drive,
+ resourceId: legacyTurn.resourceId,
+ })
+ case 'shell-failed':
+ state.counters.legacyLeaseFailures += 1
+ return reply(
+ 'legacy-lease-shell-failed',
+ `${LEGACY_LEASE_SHELL_FAILED_PREFIX} code=${legacyTurn.code}`,
+ { failureCode: legacyTurn.code }
+ )
+ case 'download-failed':
+ state.counters.legacyLeaseFailures += 1
+ return reply(
+ 'legacy-lease-download-failed',
+ `${LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX} code=${legacyTurn.code}`,
+ { failureCode: legacyTurn.code }
+ )
+ case 'answer':
+ state.counters.legacyLeaseAnswers += 1
+ return reply(
+ 'legacy-lease-answer',
+ `${LEGACY_LEASE_ANSWER_PREFIX} marker=${legacyTurn.marker} ` +
+ `sha256=${legacyTurn.sha256} bytes=${legacyTurn.sizeBytes}`,
+ { downloadSha256: legacyTurn.sha256, downloadBytes: legacyTurn.sizeBytes }
+ )
+ default:
+ throw new Error(`Unhandled legacy-lease stage ${String(legacyTurn.stage)}`)
+ }
+ }
state.counters.textOnlyResponses += 1
return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })
}
diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
index a527950d2..3beca6da5 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
@@ -1091,6 +1091,12 @@ const EXPECTED_REJECTION_REASONS = [
'image-png-malformed',
'image-png-unsupported-form',
'image-tile-grid-mismatch',
+ 'legacy-lease-download-not-offered',
+ 'legacy-lease-download-result-malformed',
+ 'legacy-lease-shell-not-offered',
+ 'legacy-lease-stream-unsupported',
+ 'legacy-lease-tool-result-count',
+ 'legacy-lease-tool-result-unexpected',
'provider-auth-mismatch',
'provider-body-invalid',
'provider-body-unsupported',
@@ -1227,3 +1233,308 @@ test('every rejected row records its closed-set reason and no request content',
assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')
assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)
})
+
+// ---------------------------------------------------------------------------
+// Legacy processing-lease restart journey (issue #1022): the fixture scripts
+// shell_exec, then clerum__gfs_download, then answers from the receipt.
+// ---------------------------------------------------------------------------
+
+const LEGACY_MARKER = 'legacy-lease-0123456789abcdef'
+const LEGACY_RESOURCE = 'a'.repeat(32)
+const LEGACY_SHA = 'b'.repeat(64)
+const LEGACY_TOKEN = `${LEGACY_MARKER}-shell-ok`
+
+const toolDefinition = name => ({
+ type: 'function',
+ function: { name, description: name, parameters: { type: 'object' } },
+})
+const LEGACY_TOOLS = [toolDefinition('shell_exec'), toolDefinition('clerum__gfs_download')]
+
+function legacyUserMessage(marker = LEGACY_MARKER) {
+ return {
+ role: 'user',
+ content:
+ '<turn-context>\ndate: 2026-10-08\n</turn-context>\n\n' +
+ `Run the journey. LEGACY_LEASE_JOURNEY marker=${marker} drive=main resourceId=${LEGACY_RESOURCE}`,
+ }
+}
+
+function legacyMessages(results) {
+ const messages = [legacyUserMessage()]
+ for (const [index, [name, content]] of results.entries()) {
+ messages.push({
+ role: 'assistant',
+ content: null,
+ tool_calls: [{ id: `call-${index}`, type: 'function', function: { name, arguments: '{}' } }],
+ })
+ messages.push({ role: 'tool', tool_call_id: `call-${index}`, content })
+ }
+ return messages
+}
+
+const wrapped = (name, content) =>
+ `<tool_output name="${name}" sanitized="false">\n${content}\n</tool_output>`
+const shellResult = content => ['shell_exec', wrapped('shell_exec', content)]
+const downloadResult = content => ['clerum__gfs_download', wrapped('clerum__gfs_download', content)]
+const receipt = (overrides = {}) =>
+ JSON.stringify({
+ delivery: 'workspace_file',
+ id: '11111111-2222-4333-8444-555555555555',
+ path: '.gfs-downloads/input-1/report.pdf',
+ sizeBytes: 42,
+ sha256: LEGACY_SHA,
+ ...overrides,
+ })
+
+test('legacy-lease step 1: asks for a shell command whose output token is not in the command', async () => {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS })
+ )
+ assert.equal(response.status, 200)
+ const body = await response.json()
+ assert.equal(body.choices[0].finish_reason, 'tool_calls')
+ const [toolCall] = body.choices[0].message.tool_calls
+ assert.equal(toolCall.function.name, 'shell_exec')
+ const { command } = JSON.parse(toolCall.function.arguments)
+ // The approval preview shows the marker, but only an executed command can
+ // produce the joined token.
+ assert.ok(command.includes(LEGACY_MARKER))
+ assert.equal(command.includes(LEGACY_TOKEN), false)
+ assert.equal(command, `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.legacyLeaseShellRequests, 1)
+ assert.deepEqual(evidence.attempts, [
+ { model: VISUAL_MODEL, imageSha256: null, responseKind: 'legacy-lease-shell-requested' },
+ ])
+})
+
+test('legacy-lease step 2: a shell result with the token asks for the GFS download', async () => {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(`${LEGACY_TOKEN}\n`)]), {
+ tools: LEGACY_TOOLS,
+ })
+ )
+ const body = await response.json()
+ const [toolCall] = body.choices[0].message.tool_calls
+ assert.equal(toolCall.function.name, 'clerum__gfs_download')
+ assert.deepEqual(JSON.parse(toolCall.function.arguments), {
+ drive: 'main',
+ resourceId: LEGACY_RESOURCE,
+ })
+ assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 1)
+ assert.equal(h.getEvidence().attempts[0].responseKind, 'legacy-lease-download-requested')
+})
+
+test('legacy-lease step 2: a lease failure before command start is answered with its store code', async () => {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(
+ VISUAL_MODEL,
+ legacyMessages([
+ shellResult(
+ 'Processing lease failed before command start: GFS download store failed (download_busy)'
+ ),
+ ]),
+ { tools: LEGACY_TOOLS }
+ )
+ )
+ const body = await response.json()
+ assert.equal(body.choices[0].finish_reason, 'stop')
+ assert.equal(
+ body.choices[0].message.content,
+ 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy'
+ )
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.legacyLeaseFailures, 1)
+ assert.equal(evidence.counters.legacyLeaseDownloadRequests, 0)
+ assert.deepEqual(evidence.attempts, [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'legacy-lease-shell-failed',
+ failureCode: 'download_busy',
+ },
+ ])
+})
+
+test('legacy-lease step 2: output without the token or a known code is unrecognized, never a download', async () => {
+ for (const content of [
+ // The command echoed back is not the token.
+ `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`,
+ 'GFS download store failed (not_a_store_code)',
+ `${LEGACY_TOKEN}-extra`,
+ ]) {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(content)]), { tools: LEGACY_TOOLS })
+ )
+ const body = await response.json()
+ assert.equal(
+ body.choices[0].message.content,
+ 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=unrecognized',
+ content
+ )
+ assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 0, content)
+ }
+})
+
+test('legacy-lease step 3: the answer carries the receipt sha256 and byte count', async () => {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt())]), {
+ tools: LEGACY_TOOLS,
+ })
+ )
+ const body = await response.json()
+ assert.equal(
+ body.choices[0].message.content,
+ `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${LEGACY_SHA} bytes=42`
+ )
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.legacyLeaseAnswers, 1)
+ assert.deepEqual(evidence.attempts, [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'legacy-lease-answer',
+ downloadSha256: LEGACY_SHA,
+ downloadBytes: 42,
+ },
+ ])
+})
+
+test('legacy-lease step 3: a different receipt gives a different answer (not a constant)', async () => {
+ const other = 'c'.repeat(64)
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(
+ VISUAL_MODEL,
+ legacyMessages([
+ shellResult(LEGACY_TOKEN),
+ downloadResult(receipt({ sha256: other, sizeBytes: 7 })),
+ ]),
+ { tools: LEGACY_TOOLS }
+ )
+ )
+ const body = await response.json()
+ assert.equal(
+ body.choices[0].message.content,
+ `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${other} bytes=7`
+ )
+})
+
+test('legacy-lease step 3: a download error is answered with its store code', async () => {
+ const h = harness()
+ const response = await call(
+ h,
+ chatBody(
+ VISUAL_MODEL,
+ legacyMessages([
+ shellResult(LEGACY_TOKEN),
+ downloadResult('Error: GFS download store failed (workspace_unavailable)'),
+ ]),
+ { tools: LEGACY_TOOLS }
+ )
+ )
+ const body = await response.json()
+ assert.equal(
+ body.choices[0].message.content,
+ 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=workspace_unavailable'
+ )
+ assert.deepEqual(h.getEvidence().attempts, [
+ {
+ model: VISUAL_MODEL,
+ imageSha256: null,
+ responseKind: 'legacy-lease-download-failed',
+ failureCode: 'workspace_unavailable',
+ },
+ ])
+})
+
+test('legacy-lease turns the fixture cannot use are refused with a closed-set reason', async () => {
+ const cases = [
+ [
+ 'shell not offered',
+ chatBody(VISUAL_MODEL, [legacyUserMessage()], {
+ tools: [toolDefinition('clerum__gfs_download')],
+ }),
+ 'legacy-lease-shell-not-offered',
+ ],
+ [
+ 'download not offered',
+ chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN)]), {
+ tools: [toolDefinition('shell_exec')],
+ }),
+ 'legacy-lease-download-not-offered',
+ ],
+ [
+ 'first result from another tool',
+ chatBody(VISUAL_MODEL, legacyMessages([downloadResult(receipt())]), { tools: LEGACY_TOOLS }),
+ 'legacy-lease-tool-result-unexpected',
+ ],
+ [
+ 'three results',
+ chatBody(
+ VISUAL_MODEL,
+ legacyMessages([
+ shellResult(LEGACY_TOKEN),
+ downloadResult(receipt()),
+ downloadResult(receipt()),
+ ]),
+ { tools: LEGACY_TOOLS }
+ ),
+ 'legacy-lease-tool-result-count',
+ ],
+ [
+ 'receipt without sha256',
+ chatBody(
+ VISUAL_MODEL,
+ legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt({ sha256: 'x' }))]),
+ { tools: LEGACY_TOOLS }
+ ),
+ 'legacy-lease-download-result-malformed',
+ ],
+ [
+ 'streamed turn',
+ chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS, stream: true }),
+ 'legacy-lease-stream-unsupported',
+ ],
+ ]
+ for (const [label, body, reason] of cases) {
+ const h = harness()
+ const response = await call(h, body)
+ assert.ok(response.status >= 400, label)
+ const evidence = h.getEvidence()
+ assert.equal(evidence.counters.legacyLeaseFailures, 1, label)
+ assert.deepEqual(
+ evidence.attempts,
+ [{ model: VISUAL_MODEL, imageSha256: null, responseKind: 'rejected', reason }],
+ label
+ )
+ }
+})
+
+test('legacy-lease: a journey line without offered tools is an ordinary text-only request', async () => {
+ const h = harness()
+ const response = await call(h, chatBody(VISUAL_MODEL, [legacyUserMessage()]))
+ const body = await response.json()
+ assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)
+ // Witness: the request reached the fixture and was answered as text.
+ assert.equal(h.getEvidence().counters.textOnlyResponses, 1)
+ assert.equal(h.getEvidence().counters.legacyLeaseShellRequests, 0)
+})
+
+test('legacy-lease failure codes are a frozen closed set that includes download_busy', () => {
+ const codes = providerModule.LEGACY_LEASE_FAILURE_CODES
+ assert.equal(Object.isFrozen(codes), true)
+ assert.ok(codes.includes('download_busy'))
+ assert.ok(codes.includes('unrecognized'))
+})
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 86c865e52..5153d8811 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.107",
+ "version": "1.0.108",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.107",
+ "version": "1.0.108",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index 18d0e2843..a1819b160 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.107",
+ "version": "1.0.108",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",