Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

revert: "test(e2e): add the minikube legacy processing lease restart lane"

Alfredo Lopez3bfa59ae60d7
diff --git a/Makefile b/Makefile
index 46dbb3fcc..b74d68007 100644
--- a/Makefile
+++ b/Makefile
@@ -321,25 +321,6 @@ minikube-run-document-upload: ## Run the visible Desktop document-upload journey
IMAGE_CAPABILITIES_LANE=document-upload \
bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
-.PHONY: minikube-run-legacy-lease-restart minikube-run-legacy-lease-restart-vacuity
-minikube-run-legacy-lease-restart: ## Run the legacy processing-lease restart scenarios (issue #1022) with the Desktop shell+GFS download journey and restore the Host
- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
- IMAGE_CAPABILITIES_LANE=legacy-lease-restart \
- bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
-
-minikube-run-legacy-lease-restart-vacuity: ## Prove the legacy-lease lane fails on the pre-fix Host (base 74e0d81d9): S-crash must stop at the shell with download_busy
- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
- IMAGE_CAPABILITIES_LANE=legacy-lease-restart-vacuity \
- bash scripts/minikube/with-t2-mutation-lock.sh -- node scripts/e2e/image-capabilities-fixture.mjs run
-
-.PHONY: minikube-build-legacy-lease-vacuity-fixture
-minikube-build-legacy-lease-vacuity-fixture: ## Build the pre-fix Host (base 74e0d81d9) and its provider-fixture layer for the legacy-lease vacuity lane under the owned profile lease
- @T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
- T2_SKIP_LOCK="$(T2_SKIP_LOCK)" T2_LOCK_TOKEN="$(T2_LOCK_TOKEN)" \
- bash scripts/minikube/with-t2-mutation-lock.sh -- bash scripts/e2e/build-legacy-lease-vacuity-image.sh
-
minikube-restore-image-capabilities: ## Resume restoration of a recorded image fixture run (IMAGE_CAPABILITIES_RUN_DIR=<dir printed by minikube-run-image-capabilities>)
@test -n "$(IMAGE_CAPABILITIES_RUN_DIR)" || { echo "IMAGE_CAPABILITIES_RUN_DIR is required: the run directory printed by 'make minikube-run-image-capabilities'"; exit 1; }
@T2_PROJECT_DIR="$(CURDIR)" T2_PROFILE="$(MINIKUBE_PROFILE)" T2_CONTEXT="$(MINIKUBE_PROFILE)" \
diff --git a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
index 23dec419f..d70615959 100644
--- a/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
+++ b/desktop-app/test/e2e-playwright/helpers/imageCapabilityEvidence.ts
@@ -57,14 +57,6 @@ export const FIXTURE_RESPONSE_KIND = {
// delivered.
documentReadRequested: 'document-read-requested',
documentAnswer: 'document-answer',
- // Issue #1022: the legacy processing-lease restart journey. shell_exec first,
- // then clerum__gfs_download, then an answer built from the download receipt;
- // a failed step is answered with the store code it reported.
- legacyLeaseShellRequested: 'legacy-lease-shell-requested',
- legacyLeaseDownloadRequested: 'legacy-lease-download-requested',
- legacyLeaseAnswer: 'legacy-lease-answer',
- legacyLeaseShellFailed: 'legacy-lease-shell-failed',
- legacyLeaseDownloadFailed: 'legacy-lease-download-failed',
} as const
/**
@@ -93,33 +85,6 @@ export const FIXTURE_REJECTION_REASONS: readonly string[] = Object.freeze([
'document-tool-output-unreadable',
'document-page-range-malformed',
'document-stream-unsupported',
- 'legacy-lease-shell-not-offered',
- 'legacy-lease-download-not-offered',
- 'legacy-lease-tool-result-unexpected',
- 'legacy-lease-tool-result-count',
- 'legacy-lease-download-result-malformed',
- 'legacy-lease-stream-unsupported',
-])
-
-/**
- * Store failure codes a legacy-lease failure row may carry (the
- * `GfsDownloadStoreError` codes plus `unrecognized`). Declared independently of
- * the fixture; `test/imageCapabilityEvidence.test.ts` checks the lists agree.
- */
-export const FIXTURE_LEGACY_LEASE_FAILURE_CODES: readonly string[] = Object.freeze([
- 'caller_mismatch',
- 'caller_quota_exceeded',
- 'corrupt_store_ledger',
- 'download_expired',
- 'download_missing',
- 'download_busy',
- 'host_quota_exceeded',
- 'publication_cancelled',
- 'storage_write_failed',
- 'unsupported_store_schema',
- 'workspace_unavailable',
- 'writer_locked',
- 'unrecognized',
])
/** The fixture's text-only answer. It names no image and no color. */
@@ -347,12 +312,6 @@ export interface FixtureProviderAttempt {
truncated?: boolean
/** One of {@link FIXTURE_REJECTION_REASONS}. Present only on rejected rows. */
reason?: string
- /** One of {@link FIXTURE_LEGACY_LEASE_FAILURE_CODES}. Legacy-lease failure rows only. */
- failureCode?: string
- /** sha256 the download receipt reported. Legacy-lease answer row only. */
- downloadSha256?: string
- /** Byte count the download receipt reported. Legacy-lease answer row only. */
- downloadBytes?: number
}
export interface FixtureEvidenceCounters {
@@ -367,10 +326,6 @@ export interface FixtureEvidenceCounters {
documentReadRequests: number
documentAnswers: number
documentFailures: number
- legacyLeaseShellRequests: number
- legacyLeaseDownloadRequests: number
- legacyLeaseAnswers: number
- legacyLeaseFailures: number
}
export interface ImageCapabilityEvidenceSnapshot {
@@ -397,10 +352,6 @@ const COUNTER_KEYS: readonly (keyof FixtureEvidenceCounters)[] = [
'documentReadRequests',
'documentAnswers',
'documentFailures',
- 'legacyLeaseShellRequests',
- 'legacyLeaseDownloadRequests',
- 'legacyLeaseAnswers',
- 'legacyLeaseFailures',
]
function requireCounters(value: unknown, source: string): FixtureEvidenceCounters {
@@ -483,53 +434,6 @@ function requireAttemptRow(value: unknown, index: number, source: string): Fixtu
)
}
- // failureCode is present on every legacy-lease failure row and on no other row.
- const legacyFailure =
- responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed ||
- responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadFailed
- if (legacyFailure) {
- const failureCode = value.failureCode
- if (
- typeof failureCode !== 'string' ||
- !FIXTURE_LEGACY_LEASE_FAILURE_CODES.includes(failureCode)
- ) {
- throw new Error(
- `${source}: attempts[${index}].failureCode must be one of ` +
- `${FIXTURE_LEGACY_LEASE_FAILURE_CODES.join(', ')} on a ${responseKind} row`
- )
- }
- row.failureCode = failureCode
- } else if ('failureCode' in value) {
- throw new Error(
- `${source}: attempts[${index}].failureCode is only valid on a legacy-lease failure row`
- )
- }
-
- // downloadSha256 and downloadBytes are present on every legacy-lease answer row only.
- if (responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer) {
- const downloadSha256 = value.downloadSha256
- const downloadBytes = value.downloadBytes
- if (typeof downloadSha256 !== 'string' || !SHA256_HEX.test(downloadSha256)) {
- throw new Error(
- `${source}: attempts[${index}].downloadSha256 must be a sha256 hex digest on a ` +
- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
- )
- }
- if (!isByteCount(downloadBytes)) {
- throw new Error(
- `${source}: attempts[${index}].downloadBytes must be a non-negative integer on a ` +
- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
- )
- }
- row.downloadSha256 = downloadSha256
- row.downloadBytes = downloadBytes
- } else if ('downloadSha256' in value || 'downloadBytes' in value) {
- throw new Error(
- `${source}: attempts[${index}].downloadSha256 and downloadBytes are only valid on a ` +
- `${FIXTURE_RESPONSE_KIND.legacyLeaseAnswer} row`
- )
- }
-
// Present only on document rows; an image row must not grow the field.
if ('documentSha256' in value) {
const documentSha256 = value.documentSha256
@@ -719,10 +623,7 @@ export function appendedAttempts(
current.reason !== row.reason ||
current.byteRange?.offset !== row.byteRange?.offset ||
current.byteRange?.length !== row.byteRange?.length ||
- current.truncated !== row.truncated ||
- current.failureCode !== row.failureCode ||
- current.downloadSha256 !== row.downloadSha256 ||
- current.downloadBytes !== row.downloadBytes
+ current.truncated !== row.truncated
) {
throw new Error(
`[image-capabilities] ledger row ${index} changed between reads; the ledger must be append-only.`
diff --git a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts b/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
deleted file mode 100644
index 24f3df659..000000000
--- a/desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
+++ /dev/null
@@ -1,892 +0,0 @@
-// desktop-app/test/e2e-playwright/helpers/legacyLeaseRestart.ts
-//
-// E2E_GUARDIAN_IPC_FLOW: this module drives no browser transition. It owns the
-// cluster side of the legacy processing-lease restart lane (issue #1022): the
-// workload stop/start scenarios, the seed of one legacy lease on the Host PVC,
-// and read-only readers for the three signals the spec asserts after each
-// scenario. The Desktop journey itself lives in the spec.
-//
-// Every kubectl call names the validated branch profile as its context and is
-// bounded by a timeout. Arguments travel as a vector, never through a shell.
-// Waits poll a live condition with a deadline; none of them is a fixed sleep.
-import { execFileSync } from 'node:child_process'
-import { randomUUID } from 'node:crypto'
-
-const PROFILE_PATTERN = /^clerum-.+-[a-f0-9]{8}$/
-const RUN_ID_PATTERN = /^image-capabilities-[a-f0-9]{12}$/
-const SHA256_HEX = /^[a-f0-9]{64}$/
-
-export const LEGACY_LEASE_LANE_MODES = ['fixed', 'vacuity'] as const
-export type LegacyLeaseLaneMode = (typeof LEGACY_LEASE_LANE_MODES)[number]
-
-/** The Host whose PVC carries the GFS download store (the fixture lane's Host). */
-export const HOST = {
- namespace: 'mcp-host',
- deployment: 'chatllm',
- container: 'mcp-host',
- selector: 'app=chatllm',
-} as const
-export const HCC = {
- namespace: 'control-plane',
- deployment: 'host-context-controller',
- selector: 'app=host-context-controller',
-} as const
-export const WRC = {
- namespace: 'control-plane',
- deployment: 'workflow-recipes',
- selector: 'app=workflow-recipes',
-} as const
-export const GFS_WORKLOADS = [
- {
- namespace: 'gfs',
- deployment: 'gfsc-writer',
- selector: 'app=gfs-controller,clerum.io/gfsc-role=writer',
- },
- {
- namespace: 'gfs',
- deployment: 'gfsc-reader',
- selector: 'app=gfs-controller,clerum.io/gfsc-role=reader',
- },
-] as const
-
-/** The store ledger on the Host PVC, mounted at /workspace. */
-export const STORE_LEDGER_PATH = '/workspace/.gfs-download-store/ledger-v1.json'
-export const LEGACY_DISCARD_COUNTER = 'clerum_gfs_legacy_processing_leases_discarded_total'
-const STORE_LOG_COMPONENT = 'GfsDownloadStore'
-const DISCARD_MSG_PREFIX = 'GFS download store discarded '
-const UNKNOWN_MSG_PREFIX = 'GFS download store could not confirm the discard of '
-
-export const SEED_LABEL_KEY = 'evenfire.ai/legacy-lease-seed'
-
-export interface LegacyLeaseLaneEnv {
- profile: string
- mode: LegacyLeaseLaneMode
- runId: string
-}
-
-/** Reads and validates the runner-owned bindings of this lane. */
-export function requireLegacyLeaseLaneEnv(
- env: NodeJS.ProcessEnv = process.env
-): LegacyLeaseLaneEnv {
- const problems: string[] = []
- const profile = (env.MINIKUBE_PROFILE ?? '').trim()
- if (!PROFILE_PATTERN.test(profile))
- problems.push(`MINIKUBE_PROFILE is not a branch profile ("${profile}")`)
- if ((env.CONTROL_API_REAL_PG_CONTEXT ?? '').trim() !== profile)
- problems.push('CONTROL_API_REAL_PG_CONTEXT must equal MINIKUBE_PROFILE')
- if ((env.E2E_K8S_CONTEXT ?? '').trim() !== profile)
- problems.push('E2E_K8S_CONTEXT must equal MINIKUBE_PROFILE (the GFS fixture helpers read it)')
- const mode = (env.LEGACY_LEASE_LANE_MODE ?? '').trim()
- if (!(LEGACY_LEASE_LANE_MODES as readonly string[]).includes(mode))
- problems.push(`LEGACY_LEASE_LANE_MODE must be fixed or vacuity ("${mode}")`)
- const runId = (env.LEGACY_LEASE_SEED_LABEL_VALUE ?? '').trim()
- if (!RUN_ID_PATTERN.test(runId) || runId !== (env.IMAGE_CAPABILITIES_RUN_ID ?? '').trim())
- problems.push('LEGACY_LEASE_SEED_LABEL_VALUE must equal IMAGE_CAPABILITIES_RUN_ID')
- if (problems.length > 0) {
- throw new Error(
- 'legacy-lease-restart lane is not configured:\n' +
- problems.map(problem => ` - ${problem}`).join('\n') +
- '\nRun it through `make minikube-run-legacy-lease-restart` (or the -vacuity target).'
- )
- }
- return { profile, mode: mode as LegacyLeaseLaneMode, runId }
-}
-
-export type Kubectl = (args: string[], options?: { timeoutMs?: number; input?: string }) => string
-
-export function kubectlFor(profile: string): Kubectl {
- if (!PROFILE_PATTERN.test(profile)) throw new Error(`refusing kubectl for "${profile}"`)
- return (args, options = {}) => {
- const timeoutMs = options.timeoutMs ?? 30_000
- try {
- return execFileSync(
- 'kubectl',
- ['--context', profile, `--request-timeout=${Math.ceil(timeoutMs / 1000)}s`, ...args],
- {
- encoding: 'utf8',
- timeout: timeoutMs + 5_000,
- input: options.input,
- maxBuffer: 16 * 1024 * 1024,
- stdio: ['pipe', 'pipe', 'pipe'],
- }
- )
- } catch (error) {
- const stderr =
- typeof (error as { stderr?: unknown }).stderr === 'string'
- ? (error as { stderr: string }).stderr.trim()
- : ''
- throw new Error(`kubectl ${args.join(' ')} failed: ${(error as Error).message}\n${stderr}`)
- }
- }
-}
-
-/** Polls `probe` until it returns a value, or fails with the last observation. */
-export async function waitFor<T>(
- label: string,
- timeoutMs: number,
- probe: () => { done: true; value: T } | { done: false; observed: string }
-): Promise<T> {
- const deadline = Date.now() + timeoutMs
- let observed = 'not yet observed'
- for (;;) {
- const result = probe()
- if (result.done) return result.value
- observed = result.observed
- if (Date.now() >= deadline)
- throw new Error(`${label}: not reached within ${timeoutMs} ms (${observed})`)
- await new Promise(resolve => setTimeout(resolve, 500))
- }
-}
-
-export interface PodState {
- name: string
- uid: string
- phase: string
- terminating: boolean
- ready: boolean
- /** A container of this pod has started running at least once. */
- started: boolean
- image: string
-}
-
-export function readPods(kubectl: Kubectl, namespace: string, selector: string): PodState[] {
- const list = JSON.parse(
- kubectl(['-n', namespace, 'get', 'pod', '-l', selector, '-o', 'json'])
- ) as {
- items: Array<{
- metadata: { name: string; uid: string; deletionTimestamp?: string }
- spec: { containers: Array<{ image: string }> }
- status: {
- phase?: string
- conditions?: Array<{ type: string; status: string }>
- containerStatuses?: Array<{
- started?: boolean
- state?: { running?: unknown; terminated?: unknown }
- lastState?: { running?: unknown; terminated?: unknown }
- }>
- }
- }>
- }
- return list.items.map(item => ({
- name: item.metadata.name,
- uid: item.metadata.uid,
- phase: item.status.phase ?? 'Unknown',
- terminating: Boolean(item.metadata.deletionTimestamp),
- ready: (item.status.conditions ?? []).some(c => c.type === 'Ready' && c.status === 'True'),
- started: (item.status.containerStatuses ?? []).some(
- c =>
- c.started === true ||
- Boolean(c.state?.running) ||
- Boolean(c.state?.terminated) ||
- Boolean(c.lastState?.running) ||
- Boolean(c.lastState?.terminated)
- ),
- image: item.spec.containers[0]?.image ?? '',
- }))
-}
-
-function describePods(pods: PodState[]): string {
- return pods.length === 0
- ? 'no pods'
- : pods
- .map(
- p =>
- `${p.name}(${p.uid.slice(0, 8)} ${p.phase}${p.terminating ? ' terminating' : ''}${p.ready ? ' ready' : ''})`
- )
- .join(', ')
-}
-
-export function deploymentReplicas(kubectl: Kubectl, namespace: string, name: string): number {
- const replicas = JSON.parse(kubectl(['-n', namespace, 'get', 'deployment', name, '-o', 'json']))
- .spec.replicas
- if (!Number.isSafeInteger(replicas)) throw new Error(`${namespace}/${name} has no spec.replicas`)
- return replicas as number
-}
-
-export function scale(kubectl: Kubectl, namespace: string, name: string, replicas: number): void {
- kubectl(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])
-}
-
-export function rolloutStatus(kubectl: Kubectl, namespace: string, name: string): void {
- kubectl(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=180s'], {
- timeoutMs: 190_000,
- })
-}
-
-export function rolloutRestart(kubectl: Kubectl, namespace: string, name: string): void {
- kubectl(['-n', namespace, 'rollout', 'restart', `deployment/${name}`])
-}
-
-export async function waitNoPods(
- kubectl: Kubectl,
- namespace: string,
- selector: string,
- label: string
-): Promise<void> {
- await waitFor(`${label} has no pods`, 120_000, () => {
- const pods = readPods(kubectl, namespace, selector)
- return pods.length === 0
- ? { done: true, value: undefined }
- : { done: false, observed: describePods(pods) }
- })
-}
-
-/** Exactly one Ready, non-terminating pod whose UID is not in `previous`. */
-export async function waitSingleNewReadyPod(
- kubectl: Kubectl,
- namespace: string,
- selector: string,
- previous: ReadonlySet<string>,
- label: string,
- timeoutMs = 300_000
-): Promise<PodState> {
- return waitFor(`${label} has one new Ready pod`, timeoutMs, () => {
- const pods = readPods(kubectl, namespace, selector)
- const [only] = pods
- if (pods.length === 1 && only && only.ready && !only.terminating && !previous.has(only.uid))
- return { done: true, value: only }
- return { done: false, observed: describePods(pods) }
- })
-}
-
-export interface PvcIdentity {
- name: string
- uid: string
-}
-
-/** The PVC the Host deployment mounts as its workspace, by claim name and UID. */
-export function hostPvcIdentity(kubectl: Kubectl): PvcIdentity {
- const deployment = JSON.parse(
- kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])
- ) as {
- spec: {
- template: {
- spec: { volumes?: Array<{ name: string; persistentVolumeClaim?: { claimName: string } }> }
- }
- }
- }
- const claims = (deployment.spec.template.spec.volumes ?? [])
- .filter(volume => volume.name === 'workspace' && volume.persistentVolumeClaim)
- .map(volume => volume.persistentVolumeClaim!.claimName)
- if (claims.length !== 1) throw new Error(`Host deployment mounts ${claims.length} workspace PVCs`)
- const name = claims[0]!
- const uid = kubectl([
- '-n',
- HOST.namespace,
- 'get',
- 'pvc',
- name,
- '-o',
- 'jsonpath={.metadata.uid}',
- ]).trim()
- if (!uid) throw new Error(`PVC ${name} has no UID`)
- return { name, uid }
-}
-
-// ---------------------------------------------------------------------------
-// Seeding one legacy lease on the stopped Host's PVC.
-// ---------------------------------------------------------------------------
-
-export interface LegacyLeaseRecord {
- leaseId: string
- callerIdentity: string
- recordIds: string[]
- acquiredAt: string
- expiresAt: string
-}
-
-/**
- * A valid, non-expired lease owned by a foreign caller, with no writer session
- * (written by an earlier boot) and no records: exactly the shape a pre-#1019
- * shell left behind when its Host died mid-command.
- */
-export function legacyLease(now = Date.now()): LegacyLeaseRecord {
- const leaseId = randomUUID()
- return {
- leaseId,
- callerIdentity: `legacy-lease-e2e-foreign-${leaseId.slice(0, 8)}`,
- recordIds: [],
- acquiredAt: new Date(now - 60_000).toISOString(),
- expiresAt: new Date(now + 60 * 60_000).toISOString(),
- }
-}
-
-/**
- * Runs inside the seed pod as the Host's own uid/gid. Read-modify-write with
- * the store's own publication sequence (temp file 0600, fsync, rename, fsync of
- * the directory), then a re-read that must show the lease and the unchanged
- * record set. It refuses a ledger that already carries leases.
- */
-const SEED_SCRIPT = `
-const fs = require('node:fs')
-const path = require('node:path')
-const { randomUUID } = require('node:crypto')
-const ledgerPath = ${JSON.stringify(STORE_LEDGER_PATH)}
-const lease = JSON.parse(process.env.LEGACY_LEASE)
-const before = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))
-if (before.schemaVersion !== 1) throw new Error('ledger schemaVersion is not 1')
-if (before.processingLeases !== undefined) throw new Error('ledger already carries processingLeases')
-const recordIds = Object.keys(before.records || {}).sort()
-const next = { ...before, processingLeases: { [lease.leaseId]: lease } }
-const temporary = ledgerPath + '.tmp-' + randomUUID()
-const fd = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, 0o600)
-try { fs.writeSync(fd, JSON.stringify(next)); fs.fsyncSync(fd) } finally { fs.closeSync(fd) }
-fs.renameSync(temporary, ledgerPath)
-const dir = fs.openSync(path.dirname(ledgerPath), 'r')
-try { fs.fsyncSync(dir) } finally { fs.closeSync(dir) }
-const after = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'))
-const afterIds = Object.keys(after.records || {}).sort()
-const seeded = after.processingLeases && after.processingLeases[lease.leaseId]
-if (!seeded || JSON.stringify(seeded) !== JSON.stringify(lease)) throw new Error('seeded lease not read back')
-if (JSON.stringify(afterIds) !== JSON.stringify(recordIds)) throw new Error('record set changed')
-const info = fs.statSync(ledgerPath)
-process.stdout.write('LEGACY_LEASE_SEED ' + JSON.stringify({ leaseId: lease.leaseId, records: afterIds.length, mode: (info.mode & 0o777).toString(8), uid: info.uid, gid: info.gid }) + '\\n')
-`
-
-export interface SeedSummary {
- leaseId: string
- records: number
- mode: string
- uid: number
- gid: number
-}
-
-export function parseSeedSummary(logs: string, leaseId: string): SeedSummary {
- const lines = logs.split('\n').filter(line => line.startsWith('LEGACY_LEASE_SEED '))
- if (lines.length !== 1) throw new Error(`seed pod printed ${lines.length} summaries`)
- const summary = JSON.parse(lines[0]!.slice('LEGACY_LEASE_SEED '.length)) as SeedSummary
- if (summary.leaseId !== leaseId) throw new Error('seed summary names another lease')
- if (summary.mode !== '600' || summary.uid !== 1001 || summary.gid !== 1001)
- throw new Error(`seeded ledger has mode ${summary.mode} owner ${summary.uid}:${summary.gid}`)
- return summary
-}
-
-/**
- * Seeds the lease while HCC and the Host are at zero and the Host pod is gone.
- * The pod runs the Host's image as uid/gid 1001 (the Host's own identity) and
- * mounts the same PVC. It is deleted before this returns, on every path.
- */
-export async function seedLegacyLease(
- kubectl: Kubectl,
- input: { runId: string; scenario: string; pvc: PvcIdentity; lease: LegacyLeaseRecord }
-): Promise<SeedSummary> {
- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 0)
- throw new Error('refusing to seed: the Host is not scaled to zero')
- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 0)
- throw new Error('refusing to seed: HCC is not scaled to zero')
- const hostPods = readPods(kubectl, HOST.namespace, HOST.selector)
- if (hostPods.length !== 0)
- throw new Error(`refusing to seed: Host pods remain (${describePods(hostPods)})`)
- const image = JSON.parse(
- kubectl(['-n', HOST.namespace, 'get', 'deployment', HOST.deployment, '-o', 'json'])
- ).spec.template.spec.containers.find((c: { name: string }) => c.name === HOST.container)?.image
- if (typeof image !== 'string' || !image) throw new Error('Host container image not found')
- const name = `legacy-lease-seed-${input.runId.slice(-12)}-${input.scenario.toLowerCase()}`
- const pod = {
- apiVersion: 'v1',
- kind: 'Pod',
- metadata: {
- name,
- namespace: HOST.namespace,
- labels: { [SEED_LABEL_KEY]: input.runId },
- },
- spec: {
- restartPolicy: 'Never',
- automountServiceAccountToken: false,
- securityContext: {
- runAsUser: 1001,
- runAsGroup: 1001,
- fsGroup: 1001,
- runAsNonRoot: true,
- seccompProfile: { type: 'RuntimeDefault' },
- },
- containers: [
- {
- name: 'seed',
- image,
- imagePullPolicy: 'IfNotPresent',
- command: ['node', '-e', SEED_SCRIPT],
- env: [{ name: 'LEGACY_LEASE', value: JSON.stringify(input.lease) }],
- securityContext: {
- allowPrivilegeEscalation: false,
- capabilities: { drop: ['ALL'] },
- },
- resources: {
- limits: { cpu: '200m', memory: '128Mi' },
- requests: { cpu: '50m', memory: '64Mi' },
- },
- volumeMounts: [{ name: 'workspace', mountPath: '/workspace' }],
- },
- ],
- volumes: [{ name: 'workspace', persistentVolumeClaim: { claimName: input.pvc.name } }],
- },
- }
- kubectl(['-n', HOST.namespace, 'create', '-f', '-'], { input: JSON.stringify(pod) })
- try {
- const phase = await waitFor(`seed pod ${name} finishes`, 120_000, () => {
- const current = kubectl([
- '-n',
- HOST.namespace,
- 'get',
- 'pod',
- name,
- '-o',
- 'jsonpath={.status.phase}',
- ]).trim()
- return current === 'Succeeded' || current === 'Failed'
- ? { done: true, value: current }
- : { done: false, observed: current || 'Pending' }
- })
- const logs = kubectl(['-n', HOST.namespace, 'logs', name, '-c', 'seed'])
- if (phase !== 'Succeeded') throw new Error(`seed pod ${name} ${phase}:\n${logs.slice(-2000)}`)
- return parseSeedSummary(logs, input.lease.leaseId)
- } finally {
- kubectl(
- [
- '-n',
- HOST.namespace,
- 'delete',
- 'pod',
- name,
- '--ignore-not-found',
- '--wait=true',
- '--timeout=90s',
- ],
- {
- timeoutMs: 100_000,
- }
- )
- }
-}
-
-// ---------------------------------------------------------------------------
-// Scenarios.
-// ---------------------------------------------------------------------------
-
-export const SCENARIOS = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'] as const
-export type Scenario = (typeof SCENARIOS)[number]
-
-export interface ScenarioResult {
- scenario: Scenario
- pvc: PvcIdentity
- previousHostUid: string
- host: PodState
- seed: SeedSummary
- lease: LegacyLeaseRecord
- /** Workloads the scenario replaced, with their pod UIDs before and after. */
- replaced: Array<{ workload: string; before: string[]; after: string }>
-}
-
-function singleReadyPod(
- kubectl: Kubectl,
- namespace: string,
- selector: string,
- label: string
-): PodState {
- const pods = readPods(kubectl, namespace, selector).filter(p => !p.terminating)
- const [only] = pods
- if (pods.length !== 1 || !only?.ready)
- throw new Error(`${label} is not one Ready pod (${describePods(pods)})`)
- return only
-}
-
-/**
- * Stops the Host. `crash` kills the running pod without grace and scales the
- * deployment to zero at once, so the replacement the ReplicaSet creates in
- * between must never start a container: if one does, the crash window was
- * lost and the scenario fails instead of silently becoming a graceful stop.
- */
-async function stopHost(
- kubectl: Kubectl,
- mode: 'crash' | 'graceful',
- running: PodState
-): Promise<void> {
- if (mode === 'graceful') {
- scale(kubectl, HOST.namespace, HOST.deployment, 0)
- await waitNoPods(kubectl, HOST.namespace, HOST.selector, 'Host')
- return
- }
- kubectl([
- '-n',
- HOST.namespace,
- 'delete',
- 'pod',
- running.name,
- '--force',
- '--grace-period=0',
- '--wait=false',
- ])
- scale(kubectl, HOST.namespace, HOST.deployment, 0)
- const startedReplacements = new Set<string>()
- await waitFor('Host has no pods after the crash', 120_000, () => {
- const pods = readPods(kubectl, HOST.namespace, HOST.selector)
- for (const pod of pods)
- if (pod.uid !== running.uid && pod.started) startedReplacements.add(pod.name)
- return pods.length === 0
- ? { done: true, value: undefined }
- : { done: false, observed: describePods(pods) }
- })
- if (startedReplacements.size > 0)
- throw new Error(
- `CRASH_WINDOW_LOST: a replacement Host pod started before the scale to zero (${[...startedReplacements].join(', ')})`
- )
-}
-
-async function restartWorkload(
- kubectl: Kubectl,
- workload: { namespace: string; deployment: string; selector: string }
-): Promise<{ workload: string; before: string[]; after: string }> {
- const before = readPods(kubectl, workload.namespace, workload.selector).map(p => p.uid)
- if (before.length === 0) throw new Error(`${workload.deployment} has no pod to restart`)
- rolloutRestart(kubectl, workload.namespace, workload.deployment)
- rolloutStatus(kubectl, workload.namespace, workload.deployment)
- const pod = await waitSingleNewReadyPod(
- kubectl,
- workload.namespace,
- workload.selector,
- new Set(before),
- workload.deployment
- )
- return { workload: workload.deployment, before, after: pod.uid }
-}
-
-/**
- * Runs one scenario end to end: HCC to zero, Host stopped, lease seeded, the
- * scenario's own restarts, then HCC and the Host back. Returns the new Host pod.
- * Every step leaves HCC and the Host at their pre-scenario replicas on success;
- * on failure, `ensureWorkloadsUp` (and the runner's restoration) bring them back.
- */
-export async function runScenario(
- kubectl: Kubectl,
- input: { scenario: Scenario; runId: string }
-): Promise<ScenarioResult> {
- const { scenario } = input
- const replaced: ScenarioResult['replaced'] = []
- const pvcBefore = hostPvcIdentity(kubectl)
- const running = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')
- const hccBefore = singleReadyPod(kubectl, HCC.namespace, HCC.selector, 'HCC')
- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)
- throw new Error('Host deployment is not at one replica before the scenario')
- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)
- throw new Error('HCC deployment is not at one replica before the scenario')
-
- // HCC first: it owns the Host deployment and would scale it back up.
- scale(kubectl, HCC.namespace, HCC.deployment, 0)
- await waitNoPods(kubectl, HCC.namespace, HCC.selector, 'HCC')
- await stopHost(kubectl, scenario === 'S-crash' ? 'crash' : 'graceful', running)
-
- const lease = legacyLease()
- const seed = await seedLegacyLease(kubectl, {
- runId: input.runId,
- scenario,
- pvc: pvcBefore,
- lease,
- })
-
- const startHostThenHcc = async () => {
- scale(kubectl, HOST.namespace, HOST.deployment, 1)
- scale(kubectl, HCC.namespace, HCC.deployment, 1)
- }
- switch (scenario) {
- case 'S-crash':
- await startHostThenHcc()
- break
- case 'S-graceful':
- // A new template while stopped: the next pod is a fresh rollout, the way
- // a graceful `rollout restart` replaces the Host.
- rolloutRestart(kubectl, HOST.namespace, HOST.deployment)
- await startHostThenHcc()
- break
- case 'S-hcc':
- await startHostThenHcc()
- break
- case 'S-gfs':
- for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))
- await startHostThenHcc()
- break
- case 'S-update':
- // Upgrade order: GFS, WRC, HCC, then the Host.
- for (const workload of GFS_WORKLOADS) replaced.push(await restartWorkload(kubectl, workload))
- replaced.push(await restartWorkload(kubectl, WRC))
- rolloutRestart(kubectl, HOST.namespace, HOST.deployment)
- scale(kubectl, HCC.namespace, HCC.deployment, 1)
- rolloutStatus(kubectl, HCC.namespace, HCC.deployment)
- scale(kubectl, HOST.namespace, HOST.deployment, 1)
- break
- }
-
- const hccPod = await waitSingleNewReadyPod(
- kubectl,
- HCC.namespace,
- HCC.selector,
- new Set([hccBefore.uid]),
- 'HCC'
- )
- replaced.push({ workload: HCC.deployment, before: [hccBefore.uid], after: hccPod.uid })
- const host = await waitSingleNewReadyPod(
- kubectl,
- HOST.namespace,
- HOST.selector,
- new Set([running.uid]),
- 'Host'
- )
- rolloutStatus(kubectl, HOST.namespace, HOST.deployment)
- // HCC's first reconcile must not roll the Host again: the pod that booted on
- // the seeded ledger is the one whose signals are read.
- const settled = singleReadyPod(kubectl, HOST.namespace, HOST.selector, 'Host')
- if (settled.uid !== host.uid)
- throw new Error(`HCC replaced the Host pod after it booted (${host.uid} -> ${settled.uid})`)
- const pvcAfter = hostPvcIdentity(kubectl)
- if (pvcAfter.name !== pvcBefore.name || pvcAfter.uid !== pvcBefore.uid)
- throw new Error(
- `Host PVC changed across ${scenario}: ${JSON.stringify(pvcBefore)} -> ${JSON.stringify(pvcAfter)}`
- )
- return { scenario, pvc: pvcAfter, previousHostUid: running.uid, host, seed, lease, replaced }
-}
-
-/** Brings HCC and the Host back to one replica after a failed scenario. */
-export function ensureWorkloadsUp(kubectl: Kubectl): void {
- if (deploymentReplicas(kubectl, HOST.namespace, HOST.deployment) !== 1)
- scale(kubectl, HOST.namespace, HOST.deployment, 1)
- if (deploymentReplicas(kubectl, HCC.namespace, HCC.deployment) !== 1)
- scale(kubectl, HCC.namespace, HCC.deployment, 1)
- rolloutStatus(kubectl, HCC.namespace, HCC.deployment)
- rolloutStatus(kubectl, HOST.namespace, HOST.deployment)
-}
-
-// ---------------------------------------------------------------------------
-// Signal readers (read-only) and pure verdicts.
-// ---------------------------------------------------------------------------
-
-export function readStoreLedger(kubectl: Kubectl, podName: string): Record<string, unknown> {
- const raw = kubectl([
- '-n',
- HOST.namespace,
- 'exec',
- podName,
- '-c',
- HOST.container,
- '--',
- 'cat',
- STORE_LEDGER_PATH,
- ])
- const parsed = JSON.parse(raw) as unknown
- if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed))
- throw new Error('store ledger is not a JSON object')
- return parsed as Record<string, unknown>
-}
-
-export function readHostLogLines(kubectl: Kubectl, podName: string): string[] {
- return kubectl(['-n', HOST.namespace, 'logs', podName, '-c', HOST.container], {
- timeoutMs: 60_000,
- })
- .split('\n')
- .filter(line => line.trim().length > 0)
-}
-
-export function readHostMetrics(kubectl: Kubectl, podName: string): string {
- return kubectl([
- '-n',
- HOST.namespace,
- 'exec',
- podName,
- '-c',
- HOST.container,
- '--',
- 'node',
- '-e',
- "fetch('http://127.0.0.1:8080/metrics').then(r=>{if(!r.ok)throw new Error('metrics '+r.status);return r.text()}).then(t=>process.stdout.write(t))",
- ])
-}
-
-/**
- * The value of an unlabelled metric, or null when the exposition has no such
- * sample. A metric with labels, or more than one sample, is refused.
- */
-export function parseMetricValue(text: string, name: string): number | null {
- const samples = text
- .split('\n')
- .filter(
- line => !line.startsWith('#') && (line.startsWith(`${name} `) || line.startsWith(`${name}{`))
- )
- if (samples.length === 0) return null
- if (samples.length !== 1) throw new Error(`${name} has ${samples.length} samples`)
- const match = new RegExp(`^${name}(?:\\{\\})? ([0-9.eE+-]+)$`).exec(samples[0]!.trim())
- if (!match) throw new Error(`${name} sample is not an unlabelled number: ${samples[0]}`)
- const value = Number(match[1])
- if (!Number.isFinite(value)) throw new Error(`${name} sample is not finite`)
- return value
-}
-
-interface StoreLogLine {
- component?: unknown
- msg?: unknown
- discarded?: unknown
- legacyProcessingLeases?: unknown
- outcome?: unknown
-}
-
-function storeLogLines(logLines: string[]): StoreLogLine[] {
- const parsed: StoreLogLine[] = []
- for (const line of logLines) {
- if (!line.startsWith('{')) continue
- let value: unknown
- try {
- value = JSON.parse(line)
- } catch {
- continue
- }
- if (
- typeof value === 'object' &&
- value !== null &&
- (value as StoreLogLine).component === STORE_LOG_COMPONENT
- )
- parsed.push(value as StoreLogLine)
- }
- return parsed
-}
-
-export type RecoveryOutcome = 'discarded' | 'unknown-outcome'
-
-/**
- * Signal 2 on the fixed Host. The ledger must have no `processingLeases`, and
- * exactly one of the two outcomes must be visible: one confirmed discard of
- * one lease with the counter at 1, or one unknown-outcome warning with the
- * counter at 0. The counter is read from a pod that just booted, so its value
- * is the delta. A missing counter is a failure: it is the liveness witness
- * that the metric registry of the fixed build is the one that answered.
- */
-export function legacyLeaseRecoveryVerdict(input: {
- ledger: Record<string, unknown>
- logLines: string[]
- counter: number | null
-}): RecoveryOutcome {
- if (Object.hasOwn(input.ledger, 'processingLeases'))
- throw new Error('the ledger still carries processingLeases after the restart')
- if (input.counter === null)
- throw new Error(`${LEGACY_DISCARD_COUNTER} is not exposed by the Host`)
- const lines = storeLogLines(input.logLines)
- const discards = lines.filter(
- line =>
- typeof line.msg === 'string' &&
- line.msg.startsWith(DISCARD_MSG_PREFIX) &&
- typeof line.discarded === 'number'
- )
- const unknown = lines.filter(
- line =>
- line.outcome === 'unknown' &&
- typeof line.msg === 'string' &&
- line.msg.startsWith(UNKNOWN_MSG_PREFIX) &&
- typeof line.legacyProcessingLeases === 'number'
- )
- if (
- discards.length === 1 &&
- unknown.length === 0 &&
- discards[0]!.discarded === 1 &&
- input.counter === 1
- )
- return 'discarded'
- if (
- unknown.length === 1 &&
- discards.length === 0 &&
- unknown[0]!.legacyProcessingLeases === 1 &&
- input.counter === 0
- )
- return 'unknown-outcome'
- throw new Error(
- `no recovery outcome: ${discards.length} discard line(s) ` +
- `(${discards.map(line => String(line.discarded)).join(',') || '-'}), ` +
- `${unknown.length} unknown-outcome line(s), counter ${input.counter}`
- )
-}
-
-/**
- * The vacuity witness on the pre-fix Host: the seeded lease is still in the
- * ledger, nothing was discarded, and the counter does not exist. Together with
- * the journey failing at the shell with download_busy, this shows the lane can
- * tell the fixed build from the broken one.
- */
-export function vacuityStoreVerdict(input: {
- ledger: Record<string, unknown>
- logLines: string[]
- counter: number | null
- leaseId: string
-}): 'lease-retained' {
- const leases = input.ledger.processingLeases as Record<string, unknown> | undefined
- if (!leases || typeof leases !== 'object' || !Object.hasOwn(leases, input.leaseId))
- throw new Error('the pre-fix Host no longer carries the seeded lease')
- const discards = storeLogLines(input.logLines).filter(
- line => typeof line.msg === 'string' && line.msg.startsWith(DISCARD_MSG_PREFIX)
- )
- if (discards.length !== 0) throw new Error('the pre-fix Host logged a legacy-lease discard')
- if (input.counter !== null) throw new Error(`the pre-fix Host exposes ${LEGACY_DISCARD_COUNTER}`)
- return 'lease-retained'
-}
-
-export interface StoreRecord {
- id: string
- hostPath: string
- sizeBytes: number
- sha256?: string
- state: string
-}
-
-export function storeRecords(ledger: Record<string, unknown>): StoreRecord[] {
- const records = ledger.records
- if (typeof records !== 'object' || records === null || Array.isArray(records))
- throw new Error('store ledger has no records object')
- return Object.values(records as Record<string, StoreRecord>)
-}
-
-/** The one completed record this journey added, matching the GFS source bytes. */
-export function downloadedRecord(
- before: Record<string, unknown>,
- after: Record<string, unknown>,
- source: { sha256: string; bytes: number }
-): StoreRecord {
- if (!SHA256_HEX.test(source.sha256)) throw new Error('source sha256 is not a digest')
- const known = new Set(storeRecords(before).map(record => record.id))
- const added = storeRecords(after).filter(record => !known.has(record.id))
- const matching = added.filter(
- record =>
- record.state === 'completed' &&
- record.sha256 === source.sha256 &&
- record.sizeBytes === source.bytes
- )
- if (added.length !== 1 || matching.length !== 1)
- throw new Error(
- `expected one new completed record for ${source.sha256}/${source.bytes} bytes; ` +
- `added ${JSON.stringify(added.map(r => ({ state: r.state, sha256: r.sha256, sizeBytes: r.sizeBytes })))}`
- )
- return matching[0]!
-}
-
-/** sha256 and size of a file on the Host PVC, computed in the Host container. */
-export function hostFileDigest(
- kubectl: Kubectl,
- podName: string,
- hostPath: string
-): { sha256: string; bytes: number } {
- if (path_is_unsafe(hostPath)) throw new Error(`refusing to read ${hostPath}`)
- const out = kubectl([
- '-n',
- HOST.namespace,
- 'exec',
- podName,
- '-c',
- HOST.container,
- '--',
- 'node',
- '-e',
- "const b=require('node:fs').readFileSync(process.argv[1]);process.stdout.write(require('node:crypto').createHash('sha256').update(b).digest('hex')+' '+b.length)",
- `/workspace/${hostPath}`,
- ]).trim()
- const [sha256, bytes] = out.split(' ')
- if (!sha256 || !SHA256_HEX.test(sha256) || !/^\d+$/.test(bytes ?? ''))
- throw new Error(`unexpected digest output: ${out}`)
- return { sha256, bytes: Number(bytes) }
-}
-
-function path_is_unsafe(hostPath: string): boolean {
- return hostPath.startsWith('/') || hostPath.split('/').some(part => part === '..' || part === '')
-}
diff --git a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts b/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts
deleted file mode 100644
index 41fb82cd4..000000000
--- a/desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts
+++ /dev/null
@@ -1,59 +0,0 @@
-import { defineConfig } from '@playwright/test'
-import path from 'node:path'
-import { requireImageCapabilitiesFixtureEnv } from './helpers/imageCapabilityEvidence'
-import { requireLegacyLeaseLaneEnv } from './helpers/legacyLeaseRestart'
-
-// E2E_GUARDIAN_IPC_FLOW: this file only configures the runner. The spec owns
-// visible IPC transitions and reads the external provider ledger as its wire oracle.
-
-/*
- * Desktop App legacy processing-lease restart lane (issue #1022).
- *
- * It reuses the image-capabilities fixture stack (derived Host image, provider
- * fixture, seeded catalog and login) and adds the cluster side: before each
- * scenario the Host and HCC are stopped, one legacy foreign processing lease is
- * seeded on the Host PVC, the scenario's restarts run, and the Desktop journey
- * (approved `shell_exec`, then `clerum__gfs_download`) must succeed on the Host
- * that booted on that ledger.
- *
- * - It loads NO env file. Every binding is exported by the runner
- * (`make minikube-run-legacy-lease-restart` or the -vacuity target) and a
- * missing or inconsistent one fails here, at config load.
- * - The runner selects the scenarios with --grep: the fixed lane runs the five
- * `legacy-lease-restart fixture:` tests, the vacuity lane runs the one
- * `legacy-lease-restart vacuity:` test against the pre-fix Host.
- */
-
-const repoRoot = path.resolve(__dirname, '../../..')
-
-if (process.env.NODE_ENV !== undefined && process.env.NODE_ENV !== 'test') {
- throw new Error(
- `NODE_ENV must be "test" (or unset) for the legacy-lease-restart fixture lane; ` +
- `received "${process.env.NODE_ENV}".`
- )
-}
-process.env.NODE_ENV = 'test'
-
-// Fail at config load, with every missing binding listed at once.
-requireImageCapabilitiesFixtureEnv(process.env)
-requireLegacyLeaseLaneEnv(process.env)
-
-const recorderRoot = process.env.QA_RECORDER_ROOT
- ? path.resolve(process.env.QA_RECORDER_ROOT)
- : path.join(repoRoot, '.local-notes', 'qa-recorder')
-
-export default defineConfig({
- testDir: '.',
- testMatch: /qa-recorder-legacy-lease-restart\.spec\.ts/,
- // Per scenario: HCC and Host stop, seed pod, the scenario's own rollouts
- // (S-update replaces GFS, WRC, HCC and the Host), one Electron launch and a
- // two-tool journey with an approval.
- timeout: 660_000,
- expect: { timeout: 30_000 },
- retries: 0,
- workers: 1,
- fullyParallel: false,
- reporter: [['list']],
- outputDir: path.join(recorderRoot, 'runs', 'desktop-app-legacy-lease-restart'),
- preserveOutput: 'always',
-})
diff --git a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts b/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts
deleted file mode 100644
index e0e1a5dcb..000000000
--- a/desktop-app/test/e2e-playwright/qa-recorder-legacy-lease-restart.spec.ts
+++ /dev/null
@@ -1,440 +0,0 @@
-import { type ElectronApplication, type Page, type TestInfo, expect, test } from '@playwright/test'
-import { createHash, randomBytes } from 'node:crypto'
-import {
- GFS_E2E_DRIVE,
- type GfsFileFixture,
- cleanupGfsFixture,
- getE2EUserId,
- gfsWriterPod,
- kubectlOut,
- ridOf,
- seedGfsFileFixture,
- seedGfsGrant,
- uniqueGfsFixtureName,
-} from '../../../tests/e2e/gfsUiFixtures'
-import { discoverManagedGfsAgent } from './helpers/gfsAgentDiscovery'
-import {
- FIXTURE_RESPONSE_KIND,
- IMAGE_CAPABILITY_FIXTURE_MODELS,
- type ImageCapabilityEvidenceSnapshot,
- appendedAttempts,
- readImageCapabilityEvidence,
- requireImageCapabilitiesFixtureEnv,
-} from './helpers/imageCapabilityEvidence'
-import {
- HOST,
- type Kubectl,
- LEGACY_DISCARD_COUNTER,
- type LegacyLeaseLaneMode,
- SCENARIOS,
- type Scenario,
- type ScenarioResult,
- downloadedRecord,
- ensureWorkloadsUp,
- hostFileDigest,
- kubectlFor,
- legacyLeaseRecoveryVerdict,
- parseMetricValue,
- readHostLogLines,
- readHostMetrics,
- readPods,
- readStoreLedger,
- requireLegacyLeaseLaneEnv,
- runScenario,
- vacuityStoreVerdict,
-} from './helpers/legacyLeaseRestart'
-import {
- EXTERNAL_REST_API_BASE_URL,
- RPC_PROXY_BASE_URL,
- assertAllowedTarget,
- desktopCredentials,
- finalizeRecording,
- launchDesktopApp,
- login,
- screenshotAndLog,
-} from './qa-recorder-helpers'
-
-/*
- * E2E_GUARDIAN_IPC_FLOW: Desktop chat, the agent switch, the model popover and
- * the approval card all travel over Electron IPC, so the renderer has no HTTP
- * request to await for any transition. The oracles are the visible chat (the
- * fixture's answer line and the approval card), the provider-boundary ledger of
- * the derived in-cluster fixture, and read-only reads of the Host's own store
- * ledger, logs, metrics and workspace file.
- *
- * Issue #1022: a processing lease written by a Host build before #1019 has no
- * writer session. A Host that boots on such a ledger must discard it at
- * initialize (log `discarded 1`, counter 1) or report that it could not confirm
- * the discard; either way the ledger it serves carries no `processingLeases`,
- * and a shell command plus a GFS download work. The pre-fix build treats the
- * lease as an inherited executor, so its shell refuses with `download_busy`.
- *
- * Each scenario stops HCC and the Host, seeds one foreign lease on the Host PVC
- * from a pod running as the Host's uid/gid, performs its own restarts, and
- * brings HCC and the Host back. Then the Desktop journey runs: the user sends a
- * journey line, approves `shell_exec` by clicking Approve, and the fixture asks
- * for `clerum__gfs_download` of a seeded GFS file. The answer on screen carries
- * the sha256 and byte count of the Host's receipt, which must equal the GFS
- * source, as must the file the Host wrote to its workspace.
- *
- * Setup shortcuts (named, outside the behaviour under test): the GFS file and
- * its grants are seeded with SQL and the writer blob with kubectl cp, and the
- * legacy lease is written by the seed pod. Only the external provider peer is
- * simulated; nothing is mocked inside the cluster or the app.
- */
-
-const SCENARIO_TITLES: Record<Scenario, string> = {
- 'S-crash':
- 'S-crash: a Host killed without grace boots on a legacy lease and serves shell then download',
- 'S-graceful':
- 'S-graceful: a Host stopped and rolled out boots on a legacy lease and serves shell then download',
- 'S-hcc':
- 'S-hcc: an HCC restart with the Host boots on a legacy lease and serves shell then download',
- 'S-gfs':
- 'S-gfs: each GFS pod restarts while the Host boots on a legacy lease, then shell and download work',
- 'S-update':
- 'S-update: GFS, WRC, HCC then Host restart in upgrade order on a legacy lease, then shell and download work',
-}
-
-const JOURNEY_RESPONSE_TIMEOUT_MS = 240_000
-const ANY_TERMINAL_ANSWER = /LEGACY-LEASE-FIXTURE-(?:OK|SHELL-FAILED|DOWNLOAD-FAILED)/
-
-interface LaneContext {
- mode: LegacyLeaseLaneMode
- runId: string
- kubectl: Kubectl
- env: ReturnType<typeof requireImageCapabilitiesFixtureEnv>
- file: GfsFileFixture
- source: { sha256: string; bytes: number }
-}
-
-let lane: LaneContext | undefined
-
-function requireLane(expected: LegacyLeaseLaneMode): LaneContext {
- if (!lane) throw new Error('legacy-lease-restart lane context was not prepared')
- // A mode mismatch is a runner defect: it fails, it never skips.
- if (lane.mode !== expected)
- throw new Error(
- `this test belongs to the ${expected} lane, but LEGACY_LEASE_LANE_MODE is ${lane.mode}`
- )
- return lane
-}
-
-test.describe.configure({ mode: 'serial' })
-
-test.beforeAll(async () => {
- const env = requireImageCapabilitiesFixtureEnv()
- const laneEnv = requireLegacyLeaseLaneEnv()
- await assertAllowedTarget('EXTERNAL_REST_API_BASE_URL', env.externalRestApiBaseUrl)
- await assertAllowedTarget('RPC_PROXY_BASE_URL', env.rpcProxyBaseUrl)
- const kubectl = kubectlFor(laneEnv.profile)
-
- // The GFS subject of the Host this lane drives: exactly the chatllm Host.
- const agent = discoverManagedGfsAgent()
- if (agent.namespace !== HOST.namespace || agent.name !== HOST.deployment)
- throw new Error(`GFS agent discovery returned ${agent.subjectId}, not the lane's Host`)
- const ownerUserId = getE2EUserId(desktopCredentials().email)
-
- const file = seedGfsFileFixture(uniqueGfsFixtureName('e2e-gfs-legacy-lease'))
- try {
- seedGfsGrant({
- resourceId: file.fileResourceId,
- subjectType: 'user',
- subjectId: ownerUserId,
- permissions: ['read'],
- inherit: true,
- grantedBy: 'e2e:legacy-lease-restart',
- })
- seedGfsGrant({
- resourceId: file.fileResourceId,
- subjectType: 'host',
- subjectId: agent.subjectId,
- permissions: ['read'],
- inherit: true,
- grantedBy: 'e2e:legacy-lease-restart',
- })
- // The source the journey must reproduce: the bytes the fixture seeded, and
- // the same digest read back from the writer's blob.
- const content = Buffer.from(`E2E GFS file fixture: ${file.name}\n`, 'utf8')
- const sha256 = createHash('sha256').update(content).digest('hex')
- const blob = kubectlOut([
- '-n',
- 'gfs',
- 'exec',
- gfsWriterPod(),
- '--',
- 'sha256sum',
- `/data/gfs/${ridOf(file.fileResourceId)}`,
- ]).trim()
- if (blob.split(/\s+/)[0] !== sha256)
- throw new Error(`writer blob digest ${blob} differs from the seeded ${sha256}`)
- lane = {
- mode: laneEnv.mode,
- runId: laneEnv.runId,
- kubectl,
- env,
- file,
- source: { sha256, bytes: content.length },
- }
- } catch (error) {
- cleanupGfsFixture(file.name)
- throw error
- }
-})
-
-test.afterAll(async () => {
- if (lane) cleanupGfsFixture(lane.file.name)
-})
-
-/** Reach the authenticated shell through the real sign-in form. */
-async function signIn(page: Page) {
- await expect(page.locator('.boot-overlay')).toBeHidden({ timeout: 30_000 })
- const emailInput = page.locator('#email-input')
- const settingsMenu = page.getByTestId('nav-settings-menu')
- await expect(emailInput.or(settingsMenu)).toBeVisible({ timeout: 30_000 })
- if (await settingsMenu.isVisible()) {
- if ((await settingsMenu.getAttribute('aria-expanded')) !== 'true') await settingsMenu.click()
- await expect(settingsMenu).toHaveAttribute('aria-expanded', 'true')
- await page.getByTestId('logout-btn').click()
- }
- await expect(emailInput).toBeVisible({ timeout: 20_000 })
- await expect(page.getByTestId('nav-chat')).toHaveCount(0)
- await login(page, desktopCredentials())
- await expect(page.getByTestId('nav-chat')).toBeVisible({ timeout: 30_000 })
-}
-
-async function openFixtureChat(page: Page, hostRef: string) {
- const escaped = hostRef.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
- await page.getByTestId('nav-chat').click()
- const switchAgent = page.getByRole('button', { name: 'Switch chat agent' })
- await expect(switchAgent).toBeVisible({ timeout: 30_000 })
- await switchAgent.click()
- const item = page.getByRole('menuitem', { name: new RegExp(`^${escaped}$`, 'i') })
- await expect(item).toHaveCount(1)
- await item.click()
- await expect(switchAgent).toContainText(new RegExp(escaped, 'i'))
-
- await page.getByTestId('nav-new-chat').click()
- const composer = page.getByRole('textbox', { name: 'Agent message composer' })
- await expect(composer).toBeVisible({ timeout: 30_000 })
- await expect(page.locator('[data-chat-message-id]')).toHaveCount(0, { timeout: 20_000 })
-
- const model = IMAGE_CAPABILITY_FIXTURE_MODELS.supported
- const chip = page.getByTestId('selected-chat-model')
- await expect(chip).toBeVisible({ timeout: 30_000 })
- await chip.click()
- await expect(page.getByRole('menu', { name: 'Select model' })).toBeVisible({ timeout: 20_000 })
- const row = page.getByTestId(`model-option-${model}`)
- await expect(row).toHaveCount(1)
- await row.click()
- await expect(chip).toHaveAttribute('data-model-id', model)
- return composer
-}
-
-interface JourneyOutcome {
- answer: string
- approvals: string[]
- before: ImageCapabilityEvidenceSnapshot
- after: ImageCapabilityEvidenceSnapshot
-}
-
-/**
- * Sends the journey line and approves every tool approval the Host raises by
- * clicking Approve, until the fixture's terminal answer is on screen. Returns
- * the answer text and the input preview of each approval clicked.
- */
-async function runJourney(page: Page, ctx: LaneContext, marker: string): Promise<JourneyOutcome> {
- const composer = await openFixtureChat(page, ctx.env.hostRef)
- const line = `LEGACY_LEASE_JOURNEY marker=${marker} drive=${GFS_E2E_DRIVE} resourceId=${ridOf(ctx.file.fileResourceId)}`
- const before = readImageCapabilityEvidence(ctx.env)
- expect(before.runId).toBe(ctx.env.runId)
- await composer.fill(line)
- const send = page.getByTestId('send-button')
- await expect(send).toBeEnabled({ timeout: 20_000 })
- await send.click()
-
- const approve = page.getByTestId('approval-approve-btn')
- const preview = page.getByTestId('approval-input-preview')
- const response = page.getByTestId('agent-response').locator('.message-block.markdown-content')
- const terminal = response.filter({ hasText: ANY_TERMINAL_ANSWER })
- const approvals: string[] = []
- // The journey has at most two approvable tools (shell, then download).
- for (let round = 0; round < 3; round += 1) {
- await expect(approve.or(terminal)).toBeVisible({ timeout: JOURNEY_RESPONSE_TIMEOUT_MS })
- if ((await terminal.count()) > 0) break
- if (round === 2) throw new Error('more approvals than the journey has tools')
- await expect(preview).toBeVisible({ timeout: 20_000 })
- const shown = await preview.innerText()
- approvals.push(shown)
- await approve.click()
- // The clicked card leaves, or is replaced by the next tool's approval.
- await expect
- .poll(
- async () =>
- (await approve.count()) === 0 ||
- ((await preview.count()) > 0 && (await preview.innerText()) !== shown),
- { timeout: 60_000 }
- )
- .toBe(true)
- }
- await expect(terminal).toHaveCount(1)
- const answer = (await terminal.innerText()).trim()
- const after = readImageCapabilityEvidence(ctx.env)
- return { answer, approvals, before, after }
-}
-
-/** The shell approval the user clicked is the fixture's command for this marker. */
-function expectShellApproved(approvals: string[], marker: string) {
- expect(approvals.length).toBeGreaterThanOrEqual(1)
- expect(approvals[0]).toContain('printf')
- expect(approvals[0]).toContain(marker)
-}
-
-async function withDesktop(
- testInfo: TestInfo,
- label: string,
- body: (page: Page) => Promise<void>
-): Promise<void> {
- let app: ElectronApplication | undefined
- let recordedPage: Page | undefined
- try {
- const launched = await launchDesktopApp(
- testInfo,
- `issue1022-${label}-minikube${new URL(EXTERNAL_REST_API_BASE_URL).port}`
- )
- app = launched.app
- recordedPage = launched.page
- await signIn(launched.page)
- await body(launched.page)
- await screenshotAndLog(launched.page, testInfo, `desktop-legacy-lease-${label}`)
- } catch (error) {
- if (recordedPage && !recordedPage.isClosed()) {
- await recordedPage
- .screenshot({ path: testInfo.outputPath('failure.png') })
- .catch(() => undefined)
- }
- throw error
- } finally {
- await finalizeRecording(app, recordedPage)
- }
-}
-
-/** The Host pod that booted on the seeded ledger is still the only Ready one. */
-function expectHostUnchanged(ctx: LaneContext, result: ScenarioResult) {
- const pods = readPods(ctx.kubectl, HOST.namespace, HOST.selector).filter(p => !p.terminating)
- expect(pods.map(p => p.uid)).toEqual([result.host.uid])
- expect(pods[0]?.ready).toBe(true)
- expect(result.host.uid).not.toBe(result.previousHostUid)
-}
-
-function attachScenario(testInfo: TestInfo, data: unknown) {
- return testInfo.attach('scenario.json', {
- body: JSON.stringify(data, null, 2),
- contentType: 'application/json',
- })
-}
-
-for (const scenario of SCENARIOS) {
- test(`legacy-lease-restart fixture: ${SCENARIO_TITLES[scenario]}`, async ({}, testInfo) => {
- const ctx = requireLane('fixed')
- try {
- const result = await runScenario(ctx.kubectl, { scenario, runId: ctx.runId })
- expect(result.seed.leaseId).toBe(result.lease.leaseId)
-
- // Signal 2: the store the new Host serves has no legacy lease, and it
- // reports one of the two recovery outcomes.
- const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)
- const metrics = readHostMetrics(ctx.kubectl, result.host.name)
- const outcome = legacyLeaseRecoveryVerdict({
- ledger: ledgerAtBoot,
- logLines: readHostLogLines(ctx.kubectl, result.host.name),
- counter: parseMetricValue(metrics, LEGACY_DISCARD_COUNTER),
- })
- await attachScenario(testInfo, { ...result, outcome })
-
- // Signal 3: the Desktop journey on that Host.
- const marker = `legacy-lease-${randomBytes(8).toString('hex')}`
- await withDesktop(testInfo, scenario.toLowerCase(), async page => {
- const journey = await runJourney(page, ctx, marker)
- expectShellApproved(journey.approvals, marker)
- expect(journey.answer).toContain(
- `LEGACY-LEASE-FIXTURE-OK marker=${marker} sha256=${ctx.source.sha256} bytes=${ctx.source.bytes}`
- )
- const rows = appendedAttempts(journey.before, journey.after)
- const kinds = rows.map(row => row.responseKind)
- expect(
- kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)
- ).toHaveLength(1)
- expect(
- kinds.filter(k => k === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)
- ).toHaveLength(1)
- const answers = rows.filter(
- row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer
- )
- expect(answers).toHaveLength(1)
- expect(answers[0]?.downloadSha256).toBe(ctx.source.sha256)
- expect(answers[0]?.downloadBytes).toBe(ctx.source.bytes)
- const delta = (key: keyof ImageCapabilityEvidenceSnapshot['counters']) =>
- journey.after.counters[key] - journey.before.counters[key]
- expect(delta('legacyLeaseShellRequests')).toBe(1)
- expect(delta('legacyLeaseDownloadRequests')).toBe(1)
- expect(delta('legacyLeaseAnswers')).toBe(1)
- expect(delta('legacyLeaseFailures')).toBe(0)
- })
-
- // The store recorded exactly this download, and the workspace file holds
- // the GFS source bytes.
- const ledgerAfter = readStoreLedger(ctx.kubectl, result.host.name)
- const record = downloadedRecord(ledgerAtBoot, ledgerAfter, ctx.source)
- expect(hostFileDigest(ctx.kubectl, result.host.name, record.hostPath)).toEqual(ctx.source)
- expect(Object.hasOwn(ledgerAfter, 'processingLeases')).toBe(false)
- expectHostUnchanged(ctx, result)
- } finally {
- ensureWorkloadsUp(ctx.kubectl)
- }
- })
-}
-
-test('legacy-lease-restart vacuity: S-crash on base 74e0d81d9 fails at the shell with download_busy', async ({}, testInfo) => {
- const ctx = requireLane('vacuity')
- try {
- const result = await runScenario(ctx.kubectl, { scenario: 'S-crash', runId: ctx.runId })
- const ledgerAtBoot = readStoreLedger(ctx.kubectl, result.host.name)
- const verdict = vacuityStoreVerdict({
- ledger: ledgerAtBoot,
- logLines: readHostLogLines(ctx.kubectl, result.host.name),
- counter: parseMetricValue(
- readHostMetrics(ctx.kubectl, result.host.name),
- LEGACY_DISCARD_COUNTER
- ),
- leaseId: result.lease.leaseId,
- })
- await attachScenario(testInfo, { ...result, verdict })
-
- const marker = `legacy-lease-${randomBytes(8).toString('hex')}`
- await withDesktop(testInfo, 'vacuity', async page => {
- const journey = await runJourney(page, ctx, marker)
- expectShellApproved(journey.approvals, marker)
- expect(journey.answer).toContain('LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy')
- const rows = appendedAttempts(journey.before, journey.after)
- const shellFailed = rows.filter(
- row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellFailed
- )
- // Positive witness first: the shell step ran and failed with the store code.
- expect(shellFailed).toHaveLength(1)
- expect(shellFailed[0]?.failureCode).toBe('download_busy')
- expect(
- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseShellRequested)
- ).toHaveLength(1)
- expect(
- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseDownloadRequested)
- ).toHaveLength(0)
- expect(
- rows.filter(row => row.responseKind === FIXTURE_RESPONSE_KIND.legacyLeaseAnswer)
- ).toHaveLength(0)
- })
- expectHostUnchanged(ctx, result)
- } finally {
- ensureWorkloadsUp(ctx.kubectl)
- }
-})
diff --git a/desktop-app/test/imageCapabilityEvidence.test.ts b/desktop-app/test/imageCapabilityEvidence.test.ts
index 31c621059..7912bc0d6 100644
--- a/desktop-app/test/imageCapabilityEvidence.test.ts
+++ b/desktop-app/test/imageCapabilityEvidence.test.ts
@@ -44,24 +44,6 @@ const rows = {
truncated: true,
}),
textOnly: (): Row => ({ model: 'glm-5.3-flash', imageSha256: null, responseKind: 'text-only' }),
- shellRequested: (): Row => ({
- model: 'glm-5.3-flash',
- imageSha256: null,
- responseKind: 'legacy-lease-shell-requested',
- }),
- legacyAnswer: (): Row => ({
- model: 'glm-5.3-flash',
- imageSha256: null,
- responseKind: 'legacy-lease-answer',
- downloadSha256: DIGEST,
- downloadBytes: 42,
- }),
- shellFailed: (): Row => ({
- model: 'glm-5.3-flash',
- imageSha256: null,
- responseKind: 'legacy-lease-shell-failed',
- failureCode: 'download_busy',
- }),
}
function ledger(attempts: Row[]): string {
@@ -81,10 +63,6 @@ function ledger(attempts: Row[]): string {
documentReadRequests: 0,
documentAnswers: 0,
documentFailures: 0,
- legacyLeaseShellRequests: 0,
- legacyLeaseDownloadRequests: 0,
- legacyLeaseAnswers: 0,
- legacyLeaseFailures: 0,
},
attempts,
})
@@ -142,47 +120,6 @@ describe('imageCapabilityEvidence ledger rows', () => {
})
})
-describe('legacy-lease ledger rows', () => {
- it('keeps the failure code and the download receipt on the rows that carry them', () => {
- const snapshot = parse([rows.shellRequested(), rows.legacyAnswer(), rows.shellFailed()])
- expect(snapshot.attempts).toEqual([
- rows.shellRequested(),
- rows.legacyAnswer(),
- rows.shellFailed(),
- ])
- })
-
- it('requires a closed-set failure code on failure rows and nowhere else', () => {
- // Witness: the well-formed failure row parses with its code.
- expect(parse([rows.shellFailed()]).attempts[0]?.failureCode).toBe('download_busy')
- const { failureCode: _code, ...withoutCode } = rows.shellFailed()
- for (const [label, row] of [
- ['missing failureCode', withoutCode],
- ['free-text failureCode', { ...rows.shellFailed(), failureCode: 'disk on fire' }],
- ['failureCode on an answer row', { ...rows.legacyAnswer(), failureCode: 'download_busy' }],
- ['failureCode on a text row', { ...rows.textOnly(), failureCode: 'download_busy' }],
- ] as const) {
- expect(() => parse([row]), label).toThrow(/failureCode/)
- }
- })
-
- it('requires the download receipt on every answer row and nowhere else', () => {
- // Witness: the well-formed answer row parses with its receipt.
- expect(parse([rows.legacyAnswer()]).attempts[0]?.downloadBytes).toBe(42)
- const { downloadSha256: _sha, ...withoutSha } = rows.legacyAnswer()
- const { downloadBytes: _bytes, ...withoutBytes } = rows.legacyAnswer()
- for (const [label, row] of [
- ['missing downloadSha256', withoutSha],
- ['short downloadSha256', { ...rows.legacyAnswer(), downloadSha256: 'abc' }],
- ['missing downloadBytes', withoutBytes],
- ['negative downloadBytes', { ...rows.legacyAnswer(), downloadBytes: -1 }],
- ['receipt on a shell row', { ...rows.shellRequested(), downloadSha256: DIGEST }],
- ] as const) {
- expect(() => parse([row]), label).toThrow(/downloadSha256|downloadBytes/)
- }
- })
-})
-
describe('appendedAttempts', () => {
it('refuses a row whose reason or delivered page changed between reads', () => {
const before = parse([rows.rejected(), rows.answer()])
@@ -199,21 +136,6 @@ describe('appendedAttempts', () => {
expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)
}
})
-
- it('refuses a legacy-lease row whose receipt or failure code changed between reads', () => {
- const before = parse([rows.legacyAnswer(), rows.shellFailed()])
- // Witness: an unchanged prefix yields the appended row.
- expect(
- appendedAttempts(before, parse([rows.legacyAnswer(), rows.shellFailed(), rows.textOnly()]))
- ).toEqual([rows.textOnly()])
- for (const [label, changed] of [
- ['sha', [{ ...rows.legacyAnswer(), downloadSha256: 'c'.repeat(64) }, rows.shellFailed()]],
- ['bytes', [{ ...rows.legacyAnswer(), downloadBytes: 43 }, rows.shellFailed()]],
- ['code', [rows.legacyAnswer(), { ...rows.shellFailed(), failureCode: 'writer_locked' }]],
- ] as const) {
- expect(() => appendedAttempts(before, parse([...changed])), label).toThrow(/append-only/)
- }
- })
})
describe('rejection reason parity', () => {
@@ -231,17 +153,4 @@ describe('rejection reason parity', () => {
expect((declared as readonly string[]).length).toBeGreaterThan(0)
expect(FIXTURE_RESPONSE_KIND.rejected).toBe('rejected')
})
-
- it('declares the same legacy-lease failure codes as the fixture, independently', async () => {
- const providerModule = (await import(PROVIDER_MODULE_URL)) as {
- LEGACY_LEASE_FAILURE_CODES: readonly string[]
- }
- const declared = (evidenceModule as Record<string, unknown>).FIXTURE_LEGACY_LEASE_FAILURE_CODES
- expect(Array.isArray(declared)).toBe(true)
- expect(declared).not.toBe(providerModule.LEGACY_LEASE_FAILURE_CODES)
- expect([...(declared as readonly string[])].sort()).toEqual(
- [...providerModule.LEGACY_LEASE_FAILURE_CODES].sort()
- )
- expect(declared as readonly string[]).toContain('download_busy')
- })
})
diff --git a/desktop-app/test/legacyLeaseRestart.test.ts b/desktop-app/test/legacyLeaseRestart.test.ts
deleted file mode 100644
index 9c9d358e1..000000000
--- a/desktop-app/test/legacyLeaseRestart.test.ts
+++ /dev/null
@@ -1,238 +0,0 @@
-import { describe, expect, it } from 'vitest'
-import {
- LEGACY_DISCARD_COUNTER,
- downloadedRecord,
- legacyLease,
- legacyLeaseRecoveryVerdict,
- parseMetricValue,
- parseSeedSummary,
- requireLegacyLeaseLaneEnv,
- vacuityStoreVerdict,
-} from './e2e-playwright/helpers/legacyLeaseRestart'
-
-const PROFILE = 'clerum-fix-issue-1022-processing-lease-lifecycle-2e0feeb9'
-const RUN_ID = 'image-capabilities-0123456789ab'
-const SHA = 'a'.repeat(64)
-
-const discardLine = JSON.stringify({
- level: 40,
- component: 'GfsDownloadStore',
- discarded: 1,
- msg: 'GFS download store discarded 1 legacy processing lease(s) at initialize',
-})
-const unknownLine = JSON.stringify({
- level: 40,
- component: 'GfsDownloadStore',
- legacyProcessingLeases: 1,
- outcome: 'unknown',
- msg: 'GFS download store could not confirm the discard of 1 legacy processing lease(s)',
-})
-const unrelatedLine = JSON.stringify({ level: 30, component: 'Server', msg: 'listening' })
-
-describe('requireLegacyLeaseLaneEnv', () => {
- const valid = {
- MINIKUBE_PROFILE: PROFILE,
- CONTROL_API_REAL_PG_CONTEXT: PROFILE,
- E2E_K8S_CONTEXT: PROFILE,
- LEGACY_LEASE_LANE_MODE: 'fixed',
- LEGACY_LEASE_SEED_LABEL_VALUE: RUN_ID,
- IMAGE_CAPABILITIES_RUN_ID: RUN_ID,
- }
-
- it('accepts the runner bindings', () => {
- expect(requireLegacyLeaseLaneEnv(valid)).toEqual({
- profile: PROFILE,
- mode: 'fixed',
- runId: RUN_ID,
- })
- })
-
- it('lists every inconsistent binding at once', () => {
- let message = ''
- try {
- requireLegacyLeaseLaneEnv({
- ...valid,
- E2E_K8S_CONTEXT: 'minikube',
- LEGACY_LEASE_LANE_MODE: 'skip',
- LEGACY_LEASE_SEED_LABEL_VALUE: 'image-capabilities-ffffffffffff',
- })
- } catch (error) {
- message = (error as Error).message
- }
- expect(message).toContain('E2E_K8S_CONTEXT')
- expect(message).toContain('LEGACY_LEASE_LANE_MODE')
- expect(message).toContain('LEGACY_LEASE_SEED_LABEL_VALUE')
- })
-})
-
-describe('legacyLease', () => {
- it('is a valid, unexpired, foreign lease with no records and no writer session', () => {
- const now = Date.parse('2026-10-08T10:00:00.000Z')
- const lease = legacyLease(now)
- expect(lease.leaseId).toMatch(/^[0-9a-f-]{36}$/)
- expect(lease.recordIds).toEqual([])
- expect(lease.callerIdentity).not.toBe('')
- expect(Date.parse(lease.expiresAt)).toBeGreaterThan(now)
- expect(Date.parse(lease.expiresAt)).toBeGreaterThan(Date.parse(lease.acquiredAt))
- expect(Object.hasOwn(lease, 'writerSessionId')).toBe(false)
- })
-})
-
-describe('parseSeedSummary', () => {
- const summary = (fields: Record<string, unknown>) =>
- `LEGACY_LEASE_SEED ${JSON.stringify({ leaseId: 'l1', records: 0, mode: '600', uid: 1001, gid: 1001, ...fields })}\n`
-
- it('reads the one summary of the seeded lease', () => {
- expect(parseSeedSummary(summary({}), 'l1').records).toBe(0)
- })
-
- it('refuses a ledger left with another mode or owner', () => {
- expect(() => parseSeedSummary(summary({ mode: '644' }), 'l1')).toThrow(/mode 644/)
- expect(() => parseSeedSummary(summary({ uid: 0 }), 'l1')).toThrow(/owner 0:1001/)
- })
-
- it('refuses no summary, two summaries, or another lease', () => {
- expect(() => parseSeedSummary('', 'l1')).toThrow(/0 summaries/)
- expect(() => parseSeedSummary(summary({}) + summary({}), 'l1')).toThrow(/2 summaries/)
- expect(() => parseSeedSummary(summary({}), 'l2')).toThrow(/another lease/)
- })
-})
-
-describe('parseMetricValue', () => {
- const exposition = [
- `# HELP ${LEGACY_DISCARD_COUNTER} Legacy processing leases discarded.`,
- `# TYPE ${LEGACY_DISCARD_COUNTER} counter`,
- `${LEGACY_DISCARD_COUNTER} 1`,
- 'process_cpu_seconds_total 3.2',
- ].join('\n')
-
- it('reads an unlabelled sample', () => {
- expect(parseMetricValue(exposition, LEGACY_DISCARD_COUNTER)).toBe(1)
- })
-
- it('returns null when the metric is absent', () => {
- expect(parseMetricValue('process_cpu_seconds_total 3.2\n', LEGACY_DISCARD_COUNTER)).toBeNull()
- })
-
- it('refuses duplicate or labelled samples', () => {
- expect(() =>
- parseMetricValue(`${exposition}\n${LEGACY_DISCARD_COUNTER} 2`, LEGACY_DISCARD_COUNTER)
- ).toThrow(/2 samples/)
- expect(() =>
- parseMetricValue(`${LEGACY_DISCARD_COUNTER}{a="b"} 1`, LEGACY_DISCARD_COUNTER)
- ).toThrow(/not an unlabelled number/)
- })
-})
-
-describe('legacyLeaseRecoveryVerdict', () => {
- const ledger = { schemaVersion: 1, records: {} }
-
- it('accepts one confirmed discard with the counter at 1', () => {
- expect(
- legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine, discardLine], counter: 1 })
- ).toBe('discarded')
- })
-
- it('accepts one unknown-outcome warning with the counter at 0', () => {
- expect(legacyLeaseRecoveryVerdict({ ledger, logLines: [unknownLine], counter: 0 })).toBe(
- 'unknown-outcome'
- )
- })
-
- it('refuses a ledger that still carries processingLeases', () => {
- expect(() =>
- legacyLeaseRecoveryVerdict({
- ledger: { ...ledger, processingLeases: {} },
- logLines: [discardLine],
- counter: 1,
- })
- ).toThrow(/still carries processingLeases/)
- })
-
- it('refuses when no outcome was logged (the Host never ran the discard)', () => {
- expect(() =>
- legacyLeaseRecoveryVerdict({ ledger, logLines: [unrelatedLine], counter: 0 })
- ).toThrow(/0 discard line\(s\).*0 unknown-outcome/)
- })
-
- it('refuses a discard whose counter disagrees, and a missing counter', () => {
- expect(() =>
- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: 0 })
- ).toThrow(/counter 0/)
- expect(() =>
- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine], counter: null })
- ).toThrow(/not exposed/)
- })
-
- it('refuses both outcomes at once, and a discard of more than one lease', () => {
- expect(() =>
- legacyLeaseRecoveryVerdict({ ledger, logLines: [discardLine, unknownLine], counter: 1 })
- ).toThrow(/no recovery outcome/)
- const two = discardLine.replace('"discarded":1', '"discarded":2')
- expect(() => legacyLeaseRecoveryVerdict({ ledger, logLines: [two], counter: 2 })).toThrow(
- /no recovery outcome/
- )
- })
-})
-
-describe('vacuityStoreVerdict', () => {
- const ledger = { schemaVersion: 1, records: {}, processingLeases: { l1: {} } }
-
- it('accepts the pre-fix Host keeping the lease without a counter', () => {
- expect(
- vacuityStoreVerdict({ ledger, logLines: [unrelatedLine], counter: null, leaseId: 'l1' })
- ).toBe('lease-retained')
- })
-
- it('refuses a discarded lease, a discard line, or an exposed counter', () => {
- expect(() =>
- vacuityStoreVerdict({
- ledger: { schemaVersion: 1, records: {} },
- logLines: [],
- counter: null,
- leaseId: 'l1',
- })
- ).toThrow(/no longer carries/)
- expect(() =>
- vacuityStoreVerdict({ ledger, logLines: [discardLine], counter: null, leaseId: 'l1' })
- ).toThrow(/logged a legacy-lease discard/)
- expect(() => vacuityStoreVerdict({ ledger, logLines: [], counter: 0, leaseId: 'l1' })).toThrow(
- /exposes/
- )
- })
-})
-
-describe('downloadedRecord', () => {
- const record = (id: string, fields: Record<string, unknown>) => ({
- id,
- hostPath: `downloads/${id}`,
- sizeBytes: 10,
- sha256: SHA,
- state: 'completed',
- ...fields,
- })
- const before = { schemaVersion: 1, records: { old: record('old', {}) } }
-
- it('returns the one new completed record with the source digest and size', () => {
- const after = { schemaVersion: 1, records: { ...before.records, fresh: record('fresh', {}) } }
- expect(downloadedRecord(before, after, { sha256: SHA, bytes: 10 }).id).toBe('fresh')
- })
-
- it('refuses no new record, a different digest, or an unfinished transfer', () => {
- expect(() => downloadedRecord(before, before, { sha256: SHA, bytes: 10 })).toThrow(/added \[\]/)
- const other = {
- schemaVersion: 1,
- records: { fresh: record('fresh', { sha256: 'b'.repeat(64) }) },
- }
- expect(() => downloadedRecord(before, other, { sha256: SHA, bytes: 10 })).toThrow(
- /expected one/
- )
- const partial = {
- schemaVersion: 1,
- records: { fresh: record('fresh', { state: 'transferring' }) },
- }
- expect(() => downloadedRecord(before, partial, { sha256: SHA, bytes: 10 })).toThrow(
- /expected one/
- )
- })
-})
diff --git a/docs/testing/minikube-t2-runbook.md b/docs/testing/minikube-t2-runbook.md
index 1c7b4d75a..97103dfb3 100644
--- a/docs/testing/minikube-t2-runbook.md
+++ b/docs/testing/minikube-t2-runbook.md
@@ -237,105 +237,6 @@ real-subscription interoperability or authorize real account use. The four
public targets and the guarded build body have hermetic boundary coverage in
`scripts/tests/test-minikube-mutation-boundary.sh`.
-### Legacy processing-lease restart lane
-
-Issue #1022. This lane proves that a Host booting on a GFS download store
-ledger that still carries a processing lease written before #1019 (no writer
-session) discards that lease and keeps serving shell commands and GFS downloads.
-It is development-only, uses the image-capabilities provider fixture, and has
-the same requirements as that lane: Node 24, a clean tree, the branch-owned
-`MINIKUBE_PROFILE` with a matching `CONTROL_API_REAL_PG_CONTEXT`, the mutation
-lease, and the single-Host (`mcp-host/chatllm`) profile.
-
-```sh
-make minikube-build-image-capabilities-fixture
-make minikube-t2
-make minikube-run-legacy-lease-restart
-```
-
-The Playwright config is
-`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts`.
-The lane runs five serial scenarios. Each one scales HCC and then the Host
-to 0. It waits until the Host pod is gone, and then seeds one legacy lease. The
-lease is valid and not expired, belongs to a foreign caller, has
-`recordIds: []` and has no `writerSessionId`. The seed pod is labelled
-`evenfire.ai/legacy-lease-seed=<run id>`. It runs the Host image as uid/gid
-1001 and mounts the Host PVC. It does a read-modify-write using the store's own
-publication sequence: a temp file with mode 0600, then fsync, rename, and an
-fsync of the directory. It refuses a ledger that already has leases. The pod
-reads the ledger back and is deleted before the scenario continues.
-
-| Scenario | How the Host is stopped | What runs after the seed |
-| --- | --- | --- |
-| `S-crash` | `delete pod --force --grace-period=0`, then an immediate scale to 0 | The Host and HCC return. The scenario fails with `CRASH_WINDOW_LOST` if a replacement pod started a container in between. |
-| `S-graceful` | Scale to 0 (SIGTERM, full grace) | `rollout restart` of the Host, then the Host and HCC return |
-| `S-hcc` | Scale to 0 | HCC and the Host return, so HCC is a new pod |
-| `S-gfs` | Scale to 0 | Each GFS pod restarts in turn (`gfsc-writer`, then `gfsc-reader`), then the Host and HCC return |
-| `S-update` | Scale to 0 | Upgrade order: GFS, then WRC, then HCC, then the Host |
-
-After each scenario, the lane checks the following:
-
-- **Pods.** The Host has a new pod UID and is Ready. HCC does not replace it
- after it boots. The Host PVC has the same name and UID before and after.
-- **Store.** The served ledger has no `processingLeases`. The lane then needs
- one of two outcomes:
- - a `GfsDownloadStore` warning that 1 legacy lease was discarded, with
- `clerum_gfs_legacy_processing_leases_discarded_total` at 1;
- - one warning that the outcome is unknown, with that counter at 0.
-- **Journey.** The Desktop user sends the journey line and approves
- `shell_exec` by clicking Approve. The fixture then requests
- `clerum__gfs_download` for a seeded GFS file.
- - The answer on screen and the fixture ledger must both carry the GFS
- source's sha256 and byte count.
- - The Host store must record exactly one new completed download with that
- digest.
- - The file in the Host workspace must hash to the same value.
- - The journey makes no direct RPC calls and uses no storage shortcuts or
- in-cluster mocks.
-
-**Vacuity.** Run the vacuity lane once, and run it before the fixed lane. The
-pre-fix Host quarantines every ledger record when it sees an inherited executor.
-Those records stay on the PVC and keep counting against the caller's quota. The
-fixed Host then discards the seeded lease when the runner restores it.
-
-```sh
-make minikube-build-legacy-lease-vacuity-fixture
-make minikube-run-legacy-lease-restart-vacuity
-```
-
-The build archives base `74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e` and checks
-that its store has the inherited-executor guard and no legacy-lease discard. It
-builds `clerum/mcp-host:legacy-lease-vacuity` from that archive and puts this
-HEAD's fixture layer on top as
-`clerum/image-capabilities-mcp-host:legacy-lease-vacuity`. It then writes
-`.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json` in the
-canonical checkout. The runner checks the profile, the base revision, the
-fixture-layer revision and the image identities against that manifest before it
-patches anything.
-
-The vacuity lane runs one `S-crash` scenario. It passes only when all of these
-hold:
-
-- the seeded lease is still in the ledger;
-- no discard line was logged and the counter does not exist;
-- the journey stops at the shell with
- `LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy`, with no download
- request.
-
-**Restoration.** The runner journals the original HCC and Host replica counts
-before Playwright starts. Zero is never a restored state. Restoration does the
-following:
-
-- deletes any seed pod of the run;
-- scales the Host and then HCC back;
-- waits for GFS, WRC, HCC and the Host to become Ready;
-- restores the production Host image;
-- waits until the Host's ledger has no `processingLeases`;
-- fails when a live `e2e-gfs-legacy-lease-*` GFS fixture remains.
-
-`make minikube-restore-image-capabilities` resumes an interrupted run the same
-way.
-
### Orphaned lock recovery
If `PROFILE_BUSY` reports that the lock has no valid owner PID, or that a stale
diff --git a/scripts/e2e/build-legacy-lease-vacuity-image.sh b/scripts/e2e/build-legacy-lease-vacuity-image.sh
deleted file mode 100755
index 54ceac335..000000000
--- a/scripts/e2e/build-legacy-lease-vacuity-image.sh
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/usr/bin/env bash
-# Build the two images of the legacy processing-lease vacuity lane (issue #1022)
-# inside the owned minikube profile, under the inherited T2 mutation lease:
-#
-# clerum/mcp-host:legacy-lease-vacuity
-# mcp-host built from the pre-fix base revision (74e0d81d9, before #1019),
-# whose store treats a legacy lease as an inherited executor;
-# clerum/image-capabilities-mcp-host:legacy-lease-vacuity
-# the provider-fixture layer of THIS HEAD on top of that Host.
-#
-# It then writes the manifest `make minikube-run-legacy-lease-restart-vacuity`
-# proves before patching anything:
-# <canonical>/.local-notes/infra/runs/legacy-lease-vacuity/<profile>/manifest.json
-#
-# Nothing is published. Run it through
-# `make minikube-build-legacy-lease-vacuity-fixture`.
-set -euo pipefail
-
-root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../.." && pwd -P)"
-profile="${MINIKUBE_PROFILE:?MINIKUBE_PROFILE is required}"
-base_revision=74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e
-base_image=clerum/mcp-host:legacy-lease-vacuity
-fixture_image=clerum/image-capabilities-mcp-host:legacy-lease-vacuity
-deadline="${root}/scripts/minikube/run-with-deadline.mjs"
-
-bash "${root}/scripts/minikube/require-t2-mutation-lock.sh"
-
-[[ "${profile}" =~ ^clerum-.+-[a-f0-9]{8}$ ]] || {
- printf 'LEGACY_LEASE_VACUITY: %s is not a branch profile\n' "${profile}" >&2
- exit 1
-}
-[[ -z "$(git -C "${root}" status --porcelain)" ]] || {
- printf 'LEGACY_LEASE_VACUITY: the worktree must be clean (the fixture layer is built from HEAD)\n' >&2
- exit 1
-}
-head_revision="$(git -C "${root}" rev-parse --verify HEAD)"
-[[ "$(git -C "${root}" cat-file -t "${base_revision}")" == commit ]] || {
- printf 'LEGACY_LEASE_VACUITY: base revision %s is not in this repository\n' "${base_revision}" >&2
- exit 1
-}
-common_dir="$(git -C "${root}" rev-parse --path-format=absolute --git-common-dir)"
-canonical="$(cd -- "$(dirname -- "${common_dir}")" && pwd -P)"
-[[ "$(basename -- "${canonical}")" == evenfire ]] || {
- printf 'LEGACY_LEASE_VACUITY: canonical Evenfire checkout required\n' >&2
- exit 1
-}
-
-run_dir="${canonical}/.local-notes/infra/runs/legacy-lease-vacuity/${profile}"
-source_dir="${run_dir}/base-${base_revision:0:12}"
-
-# Docker runs through the repository's isolated CLI environment (an empty task
-# config, no ambient credentials), pointed at the profile's own daemon with
-# `minikube docker-env`, exactly as scripts/minikube/build-images.sh does.
-# shellcheck source=scripts/minikube/docker-cli-env.sh
-source "${root}/scripts/minikube/docker-cli-env.sh"
-cleanup() {
- local status=$? cleanup_status=0
- trap - EXIT INT TERM
- docker_cli_env_cleanup || cleanup_status=$?
- rm -rf -- "${source_dir}"
- if [[ "${status}" -eq 0 && "${cleanup_status}" -ne 0 ]]; then status="${cleanup_status}"; fi
- exit "${status}"
-}
-trap cleanup EXIT
-trap 'exit 130' INT
-trap 'exit 143' TERM
-
-mkdir -p "${run_dir}"
-rm -rf -- "${source_dir}"
-mkdir -p "${source_dir}"
-git -C "${root}" archive --format=tar "${base_revision}" | tar -x -C "${source_dir}"
-
-# The archive must be the pre-fix store: it keeps the inherited-executor guard
-# and has no legacy-lease discard. Either check failing means a wrong base.
-store="${source_dir}/mcp-host/src/internalTools/gfsDownloadStore.ts"
-grep -q 'hasInheritedExecutors' "${store}" || {
- printf 'LEGACY_LEASE_VACUITY: base store has no inherited-executor guard\n' >&2
- exit 1
-}
-if grep -q 'discardLegacyProcessingLeases' "${store}"; then
- printf 'LEGACY_LEASE_VACUITY: base store already discards legacy leases\n' >&2
- exit 1
-fi
-
-docker_cli_env_prepare
-docker_env_output="$(node "${deadline}" --timeout-seconds 60 --kill-grace-seconds 5 --label minikube-docker-env -- \
- minikube -p "${profile}" docker-env --shell bash)"
-[[ -n "${docker_env_output}" ]] || {
- printf 'LEGACY_LEASE_VACUITY: minikube returned an empty Docker environment\n' >&2
- exit 1
-}
-eval "${docker_env_output}"
-unset DOCKER_API_VERSION
-
-inspect() {
- docker_cli_run_public "inspect-$1" 60 docker image inspect "$1" --format "$2" | tr -d '\r\n'
-}
-
-# The base Host image from the archived pre-fix tree. Its build context is the
-# archive root, which carries that revision's own root .dockerignore.
-docker_cli_run_public legacy-lease-vacuity-host-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \
- docker build --tag "${base_image}" --file "${source_dir}/mcp-host/Dockerfile" "${source_dir}"
-base_id="$(inspect "${base_image}" '{{.Id}}')"
-
-# The provider-fixture layer of THIS HEAD on top of that Host.
-docker_cli_run_public legacy-lease-vacuity-fixture-build "${MINIKUBE_DOCKER_BUILD_TIMEOUT_SECONDS}" \
- docker build --tag "${fixture_image}" \
- --build-arg "MCP_HOST_IMAGE=${base_image}" \
- --file "${root}/tests/e2e/fixtures/image-capabilities/Dockerfile" "${root}"
-fixture_id="$(inspect "${fixture_image}" '{{.Id}}')"
-
-for id in "${base_id}" "${fixture_id}"; do
- [[ "${id}" =~ ^sha256:[a-f0-9]{64}$ ]] || {
- printf 'LEGACY_LEASE_VACUITY: unexpected image id "%s"\n' "${id}" >&2
- exit 1
- }
-done
-# The fixture image must extend exactly the base just built.
-parent_layers="$(inspect "${base_image}" '{{json .RootFS.Layers}}')"
-fixture_layers="$(inspect "${fixture_image}" '{{json .RootFS.Layers}}')"
-[[ "${fixture_layers}" == "${parent_layers%]},"* ]] || {
- printf 'LEGACY_LEASE_VACUITY: the fixture layer does not extend the pre-fix Host image\n' >&2
- exit 1
-}
-
-manifest="${run_dir}/manifest.json"
-temporary="$(mktemp "${run_dir}/manifest.json.XXXXXX")"
-node -e '
-const [file, profile, baseRevision, head, baseImage, baseId, fixtureImage, fixtureId] = process.argv.slice(1)
-const manifest = {
- kind: "legacy-lease-vacuity",
- profile,
- baseRevision,
- fixtureLayerRevision: head,
- images: { [baseImage]: baseId, [fixtureImage]: fixtureId },
- derivedFrom: { [fixtureImage]: { ref: baseImage, id: baseId } },
- builtAt: new Date().toISOString(),
-}
-require("node:fs").writeFileSync(file, JSON.stringify(manifest, null, 2) + "\n", { mode: 0o600 })
-' "${temporary}" "${profile}" "${base_revision}" "${head_revision}" \
- "${base_image}" "${base_id}" "${fixture_image}" "${fixture_id}"
-mv -f -- "${temporary}" "${manifest}"
-printf 'LEGACY_LEASE_VACUITY_BUILT manifest=%s base=%s fixture=%s\n' "${manifest}" "${base_id}" "${fixture_id}"
diff --git a/scripts/e2e/image-capabilities-fixture.mjs b/scripts/e2e/image-capabilities-fixture.mjs
index 3544bc021..91594cc2f 100644
--- a/scripts/e2e/image-capabilities-fixture.mjs
+++ b/scripts/e2e/image-capabilities-fixture.mjs
@@ -42,69 +42,6 @@ export const playwrightLanes = {
label: 'document-upload-playwright',
grep: 'document-upload fixture: ',
},
- // Issue #1022: legacy processing-lease restart scenarios on the fixed Host.
- 'legacy-lease-restart': {
- config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',
- label: 'legacy-lease-restart-playwright',
- grep: 'legacy-lease-restart fixture: ',
- // S-crash, S-graceful, S-hcc, S-gfs and S-update: a shorter list is a lost scenario.
- expectedSpecs: 5,
- },
- // The same journey on a Host built from the pre-fix base revision: it must
- // fail at the shell step with download_busy, or the lane proves nothing.
- 'legacy-lease-restart-vacuity': {
- config: 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts',
- label: 'legacy-lease-restart-vacuity-playwright',
- grep: 'legacy-lease-restart vacuity: ',
- expectedSpecs: 1,
- },
-}
-
-/**
- * The pre-fix revision the vacuity lane runs (#1022 base, before #1019), and
- * the two images `scripts/e2e/build-legacy-lease-vacuity-image.sh` builds from
- * it. They never share a tag with the HEAD images.
- */
-export const legacyLeaseVacuity = Object.freeze({
- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
- baseImage: 'clerum/mcp-host:legacy-lease-vacuity',
- fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',
- manifestKind: 'legacy-lease-vacuity',
-})
-
-export function legacyLeaseVacuityManifestPath(canonical, profile) {
- return path.join(
- canonical,
- '.local-notes/infra/runs/legacy-lease-vacuity',
- profile,
- 'manifest.json'
- )
-}
-
-/** Label of the seed pods the legacy-lease spec creates; restoration deletes them. */
-export const legacyLeaseSeedLabel = 'evenfire.ai/legacy-lease-seed'
-
-/**
- * Runtime of each lane: the Host image HCC is pointed at, the Playwright
- * deadline, and whether the lane restarts workloads (legacy-lease lanes scale
- * HCC and the Host to zero and restart GFS and WRC, so restoration must bring
- * them back).
- */
-const laneRuntimes = {
- image: { fixtureImage, timeoutSeconds: 900, legacyLease: null },
- 'document-upload': { fixtureImage, timeoutSeconds: 900, legacyLease: null },
- // Five scenarios, each with workload stop/start cycles and one Desktop launch.
- 'legacy-lease-restart': { fixtureImage, timeoutSeconds: 3600, legacyLease: 'fixed' },
- 'legacy-lease-restart-vacuity': {
- fixtureImage: legacyLeaseVacuity.fixtureImage,
- timeoutSeconds: 900,
- legacyLease: 'vacuity',
- },
-}
-
-export function laneRuntime(lane) {
- if (!Object.hasOwn(laneRuntimes, lane)) throw new Error(`Unknown lane ${JSON.stringify(lane)}`)
- return laneRuntimes[lane]
}
export function playwrightLaneConfig(env) {
@@ -136,7 +73,7 @@ const isCount = value => Number.isSafeInteger(value) && value >= 0
* test and no unexpected, skipped or flaky outcome.
*/
export function playwrightVerdict(report, { lane, configFile }) {
- const { grep, expectedSpecs } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
+ const { grep } = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: lane })
const reported = report?.config?.configFile
if (typeof reported !== 'string' || fs.realpathSync(reported) !== fs.realpathSync(configFile))
throw new Error('Playwright report config mismatch')
@@ -165,10 +102,6 @@ export function playwrightVerdict(report, { lane, configFile }) {
)
if (titles.some(title => !title.startsWith(grep)))
throw new Error(`Playwright report has specs outside the ${lane} lane`)
- if (expectedSpecs !== undefined && titles.length !== expectedSpecs)
- throw new Error(
- `Playwright report lists ${titles.length} ${lane} specs, expected ${expectedSpecs}`
- )
return 'PASS'
}
@@ -204,53 +137,6 @@ export function proveImages(manifest, observed, head, profile) {
return { head, profile, images: observed }
}
-/**
- * Proves the vacuity images: the Host layer was built from the pre-fix base
- * revision, the fixture layer from this HEAD, both for this profile, and the
- * fixture layer derives from the exact base image present in the node.
- */
-export function proveVacuityImages(manifest, observed, head, profile) {
- const { baseRevision, baseImage: vacuityBase, fixtureImage: vacuityFixture } = legacyLeaseVacuity
- if (
- !/^[a-f0-9]{40}$/.test(head) ||
- manifest?.kind !== legacyLeaseVacuity.manifestKind ||
- manifest.profile !== profile
- )
- throw new Error('Vacuity image ownership mismatch')
- if (manifest.baseRevision !== baseRevision)
- throw new Error('Vacuity Host image was not built from the pre-fix base revision')
- if (manifest.fixtureLayerRevision !== head)
- throw new Error('Vacuity fixture layer was not built from this HEAD')
- for (const ref of [vacuityBase, vacuityFixture]) {
- const id = manifest.images?.[ref]
- if (!/^sha256:[a-f0-9]{64}$/.test(id ?? '') || id !== observed[ref])
- throw new Error('Vacuity image identity mismatch')
- }
- const binding = manifest.derivedFrom?.[vacuityFixture]
- if (binding?.ref !== vacuityBase || binding.id !== observed[vacuityBase])
- throw new Error('Vacuity fixture derived-base mismatch')
- return { head, profile, baseRevision, images: observed }
-}
-
-/**
- * The legacy-lease lanes scale HCC and the Host. Each must be back at its
- * recorded replica count; zero is never a restored state.
- */
-export function legacyLeaseReplicaPlan(recorded, live) {
- const plan = []
- // The Host first: HCC sets the Host replicas itself, and a Host already at
- // its count makes HCC's first reconcile a no-op.
- for (const key of ['host', 'hcc']) {
- const want = recorded?.[key]
- if (!Number.isSafeInteger(want) || want < 1)
- throw new Error(`Recorded ${key} replicas must be a positive integer`)
- if (!Number.isSafeInteger(live?.[key]) || live[key] < 0)
- throw new Error(`Live ${key} replicas are unreadable`)
- if (live[key] !== want) plan.push({ key, replicas: want })
- }
- return plan
-}
-
export function modelInputs(runId) {
if (!/^image-capabilities-[a-f0-9]{12}$/.test(runId)) throw new Error('Invalid fixture run')
return [
@@ -370,7 +256,6 @@ async function main() {
label: laneLabel,
grep: laneGrep,
} = selectFixtureAction(process.argv[2], process.env)
- const runtime = action === 'run' ? laneRuntime(lane) : null
const head = command('git', ['rev-parse', 'HEAD']).trim()
const branch = command('git', ['branch', '--show-current']).trim()
if (command('git', ['status', '--porcelain']).trim()) throw new Error('Clean checkout required')
@@ -454,8 +339,6 @@ async function main() {
profile,
head,
root,
- fixtureImage: runtime.fixtureImage,
- legacyLease: runtime.legacyLease,
rows: [],
changes: {},
restored: false,
@@ -467,10 +350,6 @@ async function main() {
(action === 'run' && state.head !== head)
)
throw new Error('Run state identity mismatch')
- // Restore reads the lane's Host image from the persisted state, never from the
- // environment, so a stale lane variable cannot point restoration elsewhere.
- if (typeof state.fixtureImage !== 'string' || !state.fixtureImage)
- throw new Error('Run state has no fixture image; restore it with the runner that created it')
const save = () => {
const destination = path.join(evidence, 'state.json')
if (fs.existsSync(destination)) {
@@ -545,86 +424,8 @@ async function main() {
if (!response.ok) throw new Error(`Fixture admin operation failed (${response.status})`)
return response.status === 204 ? null : response.json()
}
- const deploymentReplicas = (namespace, name) => get(namespace, 'deployment', name).spec.replicas
- const restoreLegacyLeaseWorkloads = () => {
- // Seed pods mount the Host PVC; none may outlive the run.
- kc([
- '-n',
- 'mcp-host',
- 'delete',
- 'pod',
- '-l',
- `${legacyLeaseSeedLabel}=${state.runId}`,
- '--ignore-not-found',
- '--wait=true',
- '--timeout=90s',
- ])
- const remaining = kc([
- '-n',
- 'mcp-host',
- 'get',
- 'pod',
- '-l',
- legacyLeaseSeedLabel,
- '-o',
- 'jsonpath={.items[*].metadata.name}',
- ]).trim()
- if (remaining) throw new Error('Legacy-lease seed pods remain in mcp-host')
- const liveReplicas = () => ({
- host: deploymentReplicas('mcp-host', 'chatllm'),
- hcc: deploymentReplicas('control-plane', 'host-context-controller'),
- })
- for (const { key, replicas } of legacyLeaseReplicaPlan(
- state.legacyLeaseReplicas,
- liveReplicas()
- )) {
- const [namespace, name] =
- key === 'host' ? ['mcp-host', 'chatllm'] : ['control-plane', 'host-context-controller']
- kc(['-n', namespace, 'scale', `deployment/${name}`, `--replicas=${replicas}`])
- }
- for (const [namespace, name] of [
- ['gfs', 'gfsc-writer'],
- ['gfs', 'gfsc-reader'],
- ['control-plane', 'workflow-recipes'],
- ['control-plane', 'host-context-controller'],
- ['mcp-host', 'chatllm'],
- ])
- kc(['-n', namespace, 'rollout', 'status', `deployment/${name}`, '--timeout=150s'])
- if (legacyLeaseReplicaPlan(state.legacyLeaseReplicas, liveReplicas()).length)
- throw new Error('HCC or the Host is not back at its recorded replicas')
- }
- // The Host discards a legacy lease at initialize, which runs at boot and can
- // wait out writer contention, so the ledger is polled with a bound.
- const waitLedgerWithoutLegacyLeases = async () => {
- const until = Date.now() + 240_000
- let last = 'unread'
- while (Date.now() < until) {
- try {
- const ledger = JSON.parse(
- kc([
- '-n',
- 'mcp-host',
- 'exec',
- 'deployment/chatllm',
- '-c',
- 'mcp-host',
- '--',
- 'cat',
- '/workspace/.gfs-download-store/ledger-v1.json',
- ])
- )
- if (!Object.hasOwn(ledger, 'processingLeases')) return
- last = 'processingLeases present'
- } catch (error) {
- last = error.message
- }
- await new Promise(resolve => setTimeout(resolve, 2_000))
- }
- throw new Error(`Restored Host ledger still carries legacy processing leases (${last})`)
- }
const restoreRuntime = async () => {
if (state.restored) return
- if (state.legacyLease && state.legacyLeaseReplicas) restoreLegacyLeaseWorkloads()
// Restore the model before removing its fixture-only catalog rows.
if (state.changes.host) {
const current = get('mcp-host', 'host', 'chatllm')
@@ -699,7 +500,7 @@ async function main() {
path: state.hcc.path.replace(/value$/, 'name'),
value: 'CONTEXT_MAPPER_HOST_IMAGE',
},
- { op: 'test', path: state.hcc.path, value: state.fixtureImage },
+ { op: 'test', path: state.hcc.path, value: fixtureImage },
{ op: 'replace', path: state.hcc.path, value: state.hcc.image },
unmark,
])
@@ -781,9 +582,6 @@ async function main() {
'process.stdout.write(String(!process.env.EVENFIRE_IMAGE_CAPABILITIES_FIXTURE && !require("node:fs").existsSync("/tmp/image-capabilities-evidence.json")))',
]).trim()
if (predicate !== 'true') throw new Error('Fixture state remains in Host')
- // The vacuity lane leaves its seeded lease on the PVC (the pre-fix Host
- // cannot discard it); the restored fixed Host must discard it at boot.
- if (state.legacyLease) await waitLedgerWithoutLegacyLeases()
state.restored = true
save()
}
@@ -798,20 +596,11 @@ async function main() {
}
let failure
try {
- const vacuity = runtime.legacyLease === 'vacuity'
const manifest = JSON.parse(
- fs.readFileSync(
- vacuity
- ? legacyLeaseVacuityManifestPath(canonical, profile)
- : path.join(root, 'deploy/minikube/.image-manifest.json'),
- 'utf8'
- )
+ fs.readFileSync(path.join(root, 'deploy/minikube/.image-manifest.json'), 'utf8')
)
- const proofRefs = vacuity
- ? [legacyLeaseVacuity.baseImage, legacyLeaseVacuity.fixtureImage]
- : [baseImage, fixtureImage]
const observed = Object.fromEntries(
- proofRefs.map(ref => [
+ [baseImage, fixtureImage].map(ref => [
ref,
command('minikube', [
'-p',
@@ -827,9 +616,7 @@ async function main() {
]).trim(),
])
)
- state.imageProof = vacuity
- ? proveVacuityImages(manifest, observed, head, profile)
- : proveImages(manifest, observed, head, profile)
+ state.imageProof = proveImages(manifest, observed, head, profile)
save()
const host = get('mcp-host', 'host', 'chatllm')
const hosts = JSON.parse(kc(['-n', 'mcp-host', 'get', 'hosts', '-o', 'json']))
@@ -985,7 +772,7 @@ async function main() {
value: 'CONTEXT_MAPPER_HOST_IMAGE',
},
{ op: 'test', path: state.hcc.path, value: state.hcc.image },
- { op: 'replace', path: state.hcc.path, value: state.fixtureImage },
+ { op: 'replace', path: state.hcc.path, value: fixtureImage },
])
kc([
'-n',
@@ -995,7 +782,7 @@ async function main() {
'deployment/host-context-controller',
'--timeout=120s',
])
- await waitHostImage(state.fixtureImage)
+ await waitHostImage(fixtureImage)
const wire = JSON.parse(
kc([
'-n',
@@ -1024,26 +811,6 @@ async function main() {
QA_RECORDER_IMAGE_MODEL_UNSUPPORTED: 'glm-5.3',
QA_RECORDER_IMAGE_MODEL_UNKNOWN: models[2].model,
QA_RECORDER_ROOT: evidence,
- ...(runtime.legacyLease
- ? {
- // The GFS fixture helpers name the cluster through this variable.
- E2E_K8S_CONTEXT: profile,
- LEGACY_LEASE_LANE_MODE: runtime.legacyLease,
- LEGACY_LEASE_SEED_LABEL_VALUE: state.runId,
- }
- : {}),
- }
- if (runtime.legacyLease) {
- // The spec scales these to zero; restoration needs the values it found.
- const replicas = {
- host: deploymentReplicas('mcp-host', 'chatllm'),
- hcc: deploymentReplicas('control-plane', 'host-context-controller'),
- }
- // Validates the recorded counts: a workload already at zero is refused
- // here, before the spec runs, instead of becoming the restore target.
- legacyLeaseReplicaPlan(replicas, replicas)
- state.legacyLeaseReplicas = replicas
- save()
}
command('npm', ['run', 'verify:electron', '--prefix', 'desktop-app'])
process.stdout.write(`Image capability fixture ready: ${state.runId} (lane ${lane})\n`)
@@ -1059,7 +826,7 @@ async function main() {
[
'scripts/minikube/run-with-deadline.mjs',
'--timeout-seconds',
- String(runtime.timeoutSeconds),
+ '900',
'--heartbeat-seconds',
'20',
'--kill-grace-seconds',
@@ -1080,7 +847,7 @@ async function main() {
...environment,
PLAYWRIGHT_JSON_OUTPUT_FILE: path.join(evidence, 'playwright-report.json'),
},
- timeout: (runtime.timeoutSeconds + 30) * 1000,
+ timeout: 930_000,
}
)
report(output)
@@ -1108,9 +875,7 @@ async function main() {
try {
const current = get('mcp-host', 'deployment', 'chatllm')
if (
- current.spec.template.spec.containers.some(
- container => container.image === state.fixtureImage
- )
+ current.spec.template.spec.containers.some(container => container.image === fixtureImage)
) {
const logs = kc(['-n', 'mcp-host', 'logs', 'deployment/chatllm', '--tail=60'])
state.startupDiagnostic = [
@@ -1137,32 +902,6 @@ async function main() {
)
}
uninstallSignalRestore()
- if (!failure && state.legacyLease) {
- // The spec removes its GFS fixtures; a live row left behind fails the run.
- try {
- const live = kc([
- '-n',
- 'control-plane',
- 'exec',
- 'deployment/control-postgres',
- '--',
- 'psql',
- '-v',
- 'ON_ERROR_STOP=1',
- '-U',
- 'postgres',
- '-d',
- 'profiles',
- '-t',
- '-A',
- '-c',
- "SELECT count(*) FROM gfs_resources WHERE drive = 'main' AND deleted_at IS NULL AND path_cache LIKE '/e2e-gfs-legacy-lease-%';",
- ]).trim()
- if (live !== '0') failure = new Error(`Legacy-lease GFS fixture rows remain (${live})`)
- } catch (error) {
- failure = new Error(`Legacy-lease GFS fixture check failed: ${error.message}`)
- }
- }
if (failure) throw failure
process.stdout.write(
fixturePassLine(JSON.parse(readOwnedFile(evidence, 'state.json')), { lane, evidence })
diff --git a/scripts/e2e/image-capabilities-fixture.test.mjs b/scripts/e2e/image-capabilities-fixture.test.mjs
index 0e81c46f5..13ae290e9 100644
--- a/scripts/e2e/image-capabilities-fixture.test.mjs
+++ b/scripts/e2e/image-capabilities-fixture.test.mjs
@@ -8,15 +8,10 @@ import {
fixtureImage,
fixturePassLine,
installSignalRestore,
- laneRuntime,
- legacyLeaseReplicaPlan,
- legacyLeaseVacuity,
- legacyLeaseVacuityManifestPath,
modelInputs,
playwrightLaneConfig,
playwrightVerdict,
proveImages,
- proveVacuityImages,
requireOwnedResource,
runAnnotation,
sanitizeFixtureReport,
@@ -357,190 +352,3 @@ test('the playwright log redacts the admin password, the session cookie and bear
assert.ok(!report.includes('opaque-bearer-value'))
assert.match(report, /3 passed \(41\.2s\)/)
})
-
-// ---------------------------------------------------------------------------
-// Issue #1022: legacy processing-lease restart lanes.
-// ---------------------------------------------------------------------------
-
-test('the legacy-lease lanes share one config and select disjoint journeys', () => {
- const fixed = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart' })
- const vacuity = playwrightLaneConfig({ IMAGE_CAPABILITIES_LANE: 'legacy-lease-restart-vacuity' })
- assert.equal(
- fixed.config,
- 'desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts'
- )
- assert.equal(vacuity.config, fixed.config)
- assert.equal(fixed.grep, 'legacy-lease-restart fixture: ')
- assert.equal(vacuity.grep, 'legacy-lease-restart vacuity: ')
- assert.equal(vacuity.grep.startsWith(fixed.grep), false)
- assert.equal(fixed.grep.startsWith(vacuity.grep), false)
- const options = {
- lane: 'legacy-lease-restart',
- configFile: laneConfigFile('legacy-lease-restart'),
- }
- const scenarioTitles = ['S-crash', 'S-graceful', 'S-hcc', 'S-gfs', 'S-update'].map(
- scenario => `legacy-lease-restart fixture: ${scenario}`
- )
- // Witness: the lane's own five scenario titles are PASS.
- assert.equal(
- playwrightVerdict(
- playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles }),
- options
- ),
- 'PASS'
- )
- // A clean report that lost scenarios is not the lane.
- assert.throws(
- () =>
- playwrightVerdict(
- playwrightReport({ lane: 'legacy-lease-restart', titles: scenarioTitles.slice(0, 2) }),
- options
- ),
- /lists 2 legacy-lease-restart specs, expected 5/
- )
- // The vacuity lane is exactly its one S-crash journey.
- assert.equal(
- playwrightVerdict(
- playwrightReport({
- lane: 'legacy-lease-restart-vacuity',
- titles: ['legacy-lease-restart vacuity: S-crash'],
- }),
- {
- lane: 'legacy-lease-restart-vacuity',
- configFile: laneConfigFile('legacy-lease-restart-vacuity'),
- }
- ),
- 'PASS'
- )
- // A vacuity title in the fixed lane's report is outside that lane.
- assert.throws(
- () =>
- playwrightVerdict(
- playwrightReport({
- lane: 'legacy-lease-restart',
- titles: ['legacy-lease-restart vacuity: S-crash'],
- }),
- options
- ),
- /outside the legacy-lease-restart lane/
- )
-})
-
-test('each lane runtime names its Host image and deadline; existing lanes are unchanged', () => {
- for (const lane of ['image', 'document-upload']) {
- assert.deepEqual(laneRuntime(lane), { fixtureImage, timeoutSeconds: 900, legacyLease: null })
- }
- assert.deepEqual(laneRuntime('legacy-lease-restart'), {
- fixtureImage,
- timeoutSeconds: 3600,
- legacyLease: 'fixed',
- })
- assert.deepEqual(laneRuntime('legacy-lease-restart-vacuity'), {
- fixtureImage: 'clerum/image-capabilities-mcp-host:legacy-lease-vacuity',
- timeoutSeconds: 900,
- legacyLease: 'vacuity',
- })
- // The vacuity images never reuse a HEAD tag.
- assert.notEqual(legacyLeaseVacuity.fixtureImage, fixtureImage)
- assert.notEqual(legacyLeaseVacuity.baseImage, baseImage)
- for (const lane of ['typo', '__proto__', '']) {
- assert.throws(() => laneRuntime(lane), /Unknown lane/, lane)
- }
-})
-
-test('the vacuity manifest lives under the canonical run root, per profile', () => {
- assert.equal(
- legacyLeaseVacuityManifestPath('/repo/evenfire', profile),
- `/repo/evenfire/.local-notes/infra/runs/legacy-lease-vacuity/${profile}/manifest.json`
- )
-})
-
-const vacuityIds = {
- [legacyLeaseVacuity.baseImage]: `sha256:${'e'.repeat(64)}`,
- [legacyLeaseVacuity.fixtureImage]: `sha256:${'f'.repeat(64)}`,
-}
-function vacuityManifest() {
- return {
- kind: 'legacy-lease-vacuity',
- profile,
- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
- fixtureLayerRevision: head,
- images: { ...vacuityIds },
- derivedFrom: {
- [legacyLeaseVacuity.fixtureImage]: {
- ref: legacyLeaseVacuity.baseImage,
- id: vacuityIds[legacyLeaseVacuity.baseImage],
- },
- },
- }
-}
-
-test('vacuity images must come from the pre-fix base and this HEAD, for this profile', () => {
- // Witness: the well-formed manifest is proven.
- assert.deepEqual(proveVacuityImages(vacuityManifest(), vacuityIds, head, profile), {
- head,
- profile,
- baseRevision: '74e0d81d9b70bbc0e123ed2bad89f08d3e13e99e',
- images: vacuityIds,
- })
- for (const [label, mutate, pattern] of [
- ['other kind', value => (value.kind = 'image-manifest'), /ownership/],
- ['other profile', value => (value.profile = 'clerum-other-12345678'), /ownership/],
- ['built from HEAD', value => (value.baseRevision = head), /pre-fix base/],
- ['stale fixture layer', value => (value.fixtureLayerRevision = 'd'.repeat(40)), /this HEAD/],
- [
- 'other base id',
- value => (value.images[legacyLeaseVacuity.baseImage] = `sha256:${'1'.repeat(64)}`),
- /identity/,
- ],
- [
- 'fixture from another base',
- value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].id = `sha256:${'2'.repeat(64)}`),
- /derived-base/,
- ],
- [
- 'fixture from the HEAD Host',
- value => (value.derivedFrom[legacyLeaseVacuity.fixtureImage].ref = baseImage),
- /derived-base/,
- ],
- ]) {
- const changed = vacuityManifest()
- mutate(changed)
- assert.throws(() => proveVacuityImages(changed, vacuityIds, head, profile), pattern, label)
- }
- // The node holds a different image than the manifest recorded.
- assert.throws(
- () =>
- proveVacuityImages(
- vacuityManifest(),
- { ...vacuityIds, [legacyLeaseVacuity.fixtureImage]: `sha256:${'3'.repeat(64)}` },
- head,
- profile
- ),
- /identity/
- )
- // A HEAD image manifest is never accepted as a vacuity proof.
- assert.throws(() => proveVacuityImages(manifest(), ids, head, profile), /ownership/)
-})
-
-test('restoration scales the Host before HCC and never accepts zero as restored', () => {
- const recorded = { host: 1, hcc: 1 }
- // Witness: both at zero yields a plan, Host first.
- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 0, hcc: 0 }), [
- { key: 'host', replicas: 1 },
- { key: 'hcc', replicas: 1 },
- ])
- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 0 }), [
- { key: 'hcc', replicas: 1 },
- ])
- assert.deepEqual(legacyLeaseReplicaPlan(recorded, { host: 1, hcc: 1 }), [])
- for (const [label, bad] of [
- ['zero host', { host: 0, hcc: 1 }],
- ['zero hcc', { host: 1, hcc: 0 }],
- ['missing', {}],
- ['fractional', { host: 1.5, hcc: 1 }],
- ]) {
- assert.throws(() => legacyLeaseReplicaPlan(bad, { host: 1, hcc: 1 }), /Recorded/, label)
- }
- assert.throws(() => legacyLeaseReplicaPlan(recorded, { host: undefined, hcc: 1 }), /unreadable/)
-})
diff --git a/tests/e2e/fixtures/image-capabilities/README.md b/tests/e2e/fixtures/image-capabilities/README.md
index 624c1216b..b3a207519 100644
--- a/tests/e2e/fixtures/image-capabilities/README.md
+++ b/tests/e2e/fixtures/image-capabilities/README.md
@@ -55,39 +55,6 @@ file chip, sends, and requires the digest of the attached bytes in the answer,
one `document-read-requested` row, one `document-answer` row carrying that
digest, no image row and no `documentFailures`.
-## Legacy processing-lease restart journey (issue #1022)
-
-`make minikube-run-legacy-lease-restart` is the same runner with
-`IMAGE_CAPABILITIES_LANE=legacy-lease-restart`; the config is
-`desktop-app/test/e2e-playwright/playwright.legacy-lease-restart.config.ts` and
-the cluster side lives in `helpers/legacyLeaseRestart.ts`. The user message
-carries one line:
-
-```
-LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>
-```
-
-The fixture then scripts the turns from the message list alone: it asks for
-`shell_exec` with a command whose output is `<marker>-shell-ok` (a token absent
-from the command), then for `clerum__gfs_download` of the named resource once
-the shell result echoes the token, and answers
-`LEGACY-LEASE-FIXTURE-OK marker=… sha256=… bytes=…` from the download receipt.
-A shell or download result without what the next step needs is answered with
-`LEGACY-LEASE-FIXTURE-SHELL-FAILED code=<code>` or
-`LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=<code>`, where the code is a GFS
-download store error code (for example `download_busy`) or `unrecognized`.
-`download_busy` is what the pre-fix Host returns for a legacy lease. The ledger
-rows are `legacy-lease-shell-requested`, `legacy-lease-download-requested`,
-`legacy-lease-answer` (with `downloadSha256` and `downloadBytes`),
-`legacy-lease-shell-failed` and `legacy-lease-download-failed` (with
-`failureCode`). It never records command output.
-
-The vacuity lane, `make minikube-run-legacy-lease-restart-vacuity`, runs the same
-fixture layer on a Host built from the pre-fix base by
-`make minikube-build-legacy-lease-vacuity-fixture`. The procedure and the
-signals checked after each scenario are in
-`docs/testing/minikube-t2-runbook.md` (Legacy processing-lease restart lane).
-
## Local execution
Use Node 24 and Bash 5, a clean development branch based on current `origin/dev`,
diff --git a/tests/e2e/fixtures/image-capabilities/provider.mjs b/tests/e2e/fixtures/image-capabilities/provider.mjs
index 1ecd30441..85bf71b60 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.mjs
@@ -160,12 +160,6 @@ export const FIXTURE_REJECTION_REASONS = Object.freeze([
'document-tool-output-unreadable',
'document-page-range-malformed',
'document-stream-unsupported',
- 'legacy-lease-shell-not-offered',
- 'legacy-lease-download-not-offered',
- 'legacy-lease-tool-result-unexpected',
- 'legacy-lease-tool-result-count',
- 'legacy-lease-download-result-malformed',
- 'legacy-lease-stream-unsupported',
])
/** A refusal with a stable machine-readable reason code for the evidence tally. */
@@ -518,187 +512,6 @@ function classifyDocumentTurn(body) {
}
}
-/*
- * Legacy processing-lease restart journey (issue #1022, `make
- * minikube-run-legacy-lease-restart`). The user message carries one line
- *
- * LEGACY_LEASE_JOURNEY marker=legacy-lease-<16 hex> drive=<drive> resourceId=<32 hex>
- *
- * and the fixture scripts two tool calls from the message list alone:
- *
- * 1. no tool result yet: call `shell_exec` with a command whose output is
- * `<marker>-shell-ok`. That token does not appear in the command, so only
- * a command that really ran can echo it back;
- * 2. one shell result carrying that token: call `clerum__gfs_download` for the
- * named resource;
- * 3. a shell result and a download result: answer with the sha256 and byte
- * count the Host's download receipt reports.
- *
- * A shell or download result that does not carry what step 2 or 3 needs is
- * answered with a failure line naming a closed-set code (a store error code such
- * as `download_busy`, or `unrecognized`), so the journey fails on screen at the
- * step that broke. The ledger records only the step, the code, and for the
- * answer the receipt's digest and byte count, never the command output.
- */
-const SHELL_TOOL = 'shell_exec'
-const GFS_DOWNLOAD_TOOL = 'clerum__gfs_download'
-const LEGACY_LEASE_JOURNEY_LINE =
- /LEGACY_LEASE_JOURNEY marker=(legacy-lease-[a-f0-9]{16}) drive=([a-z0-9][a-z0-9-]{0,62}) resourceId=([a-f0-9]{32})/
-const LEGACY_LEASE_SHELL_TOKEN_SUFFIX = '-shell-ok'
-export const LEGACY_LEASE_ANSWER_PREFIX = 'LEGACY-LEASE-FIXTURE-OK'
-export const LEGACY_LEASE_SHELL_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-SHELL-FAILED'
-export const LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX = 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED'
-
-/**
- * Codes a failed shell or download step may report: the GFS download store's
- * error codes (`GfsDownloadStoreErrorCode`), plus `unrecognized` for a result
- * that names none of them. A closed set of fixed strings, never tool output.
- */
-export const LEGACY_LEASE_FAILURE_CODES = Object.freeze([
- 'caller_mismatch',
- 'caller_quota_exceeded',
- 'corrupt_store_ledger',
- 'download_expired',
- 'download_missing',
- 'download_busy',
- 'host_quota_exceeded',
- 'publication_cancelled',
- 'storage_write_failed',
- 'unsupported_store_schema',
- 'workspace_unavailable',
- 'writer_locked',
- 'unrecognized',
-])
-
-/** The store error text (`GfsDownloadStoreError`): `GFS download store failed (<code>)`. */
-const STORE_FAILURE = /GFS download store failed \(([a-z_]+)\)/
-
-function toolOutputPattern(name) {
- return new RegExp(
- `^<tool_output name="${name}" sanitized="(?:true|false)">\\n([\\s\\S]*)\\n</tool_output>$`
- )
-}
-const SHELL_TOOL_OUTPUT = toolOutputPattern(SHELL_TOOL)
-const GFS_DOWNLOAD_TOOL_OUTPUT = toolOutputPattern(GFS_DOWNLOAD_TOOL)
-
-class FixtureLegacyLeaseError extends Error {
- constructor(code, message) {
- super(message)
- this.name = 'FixtureLegacyLeaseError'
- this.code = code
- }
-}
-
-function failureCode(content) {
- const code = STORE_FAILURE.exec(content)?.[1]
- return code && LEGACY_LEASE_FAILURE_CODES.includes(code) ? code : 'unrecognized'
-}
-
-function offeredTool(body, name) {
- const tools = Array.isArray(body.tools) ? body.tools : []
- return tools.some(tool => tool?.function?.name === name)
-}
-
-/**
- * Classifies a legacy-lease journey turn.
- *
- * - `null`: the last user message carries no journey line, or no tool is offered
- * (a title or summary request), so the request is not part of the journey.
- * - `{ stage: 'shell', marker }`: ask for the shell command.
- * - `{ stage: 'download', drive, resourceId }`: the shell echoed the token.
- * - `{ stage: 'shell-failed', code }`: the shell result lacks the token.
- * - `{ stage: 'answer', marker, sha256, sizeBytes }`: the download receipt.
- * - `{ stage: 'download-failed', code }`: the download result is an error.
- */
-function classifyLegacyLeaseTurn(body) {
- if (!Array.isArray(body.tools) || body.tools.length === 0) return null
- const userIndex = lastUserIndex(body.messages)
- if (userIndex < 0) return null
- const user = body.messages[userIndex]
- if (typeof user.content !== 'string') return null
- const journey = LEGACY_LEASE_JOURNEY_LINE.exec(user.content)
- if (!journey) return null
- const [, marker, drive, resourceId] = journey
-
- const toolMessages = body.messages
- .slice(userIndex + 1)
- .filter(message => message?.role === 'tool')
- if (toolMessages.length > 2)
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-tool-result-count',
- `the journey makes two tool calls, received ${toolMessages.length} results`
- )
- const contents = toolMessages.map(message => message.content)
- if (contents.some(content => typeof content !== 'string'))
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-tool-result-unexpected',
- 'a tool result is not a string'
- )
-
- if (contents.length === 0) {
- if (!offeredTool(body, SHELL_TOOL))
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-shell-not-offered',
- 'the Host did not offer shell_exec for the journey'
- )
- return { stage: 'shell', marker }
- }
-
- const shell = SHELL_TOOL_OUTPUT.exec(contents[0])
- if (!shell)
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-tool-result-unexpected',
- 'the first tool result is not a shell_exec result'
- )
- const token = `${marker}${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}`
- const echoed = shell[1].split('\n').some(line => line.trim() === token)
- if (contents.length === 1) {
- if (!echoed) return { stage: 'shell-failed', code: failureCode(shell[1]) }
- if (!offeredTool(body, GFS_DOWNLOAD_TOOL))
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-download-not-offered',
- 'the Host did not offer clerum__gfs_download after the shell step'
- )
- return { stage: 'download', drive, resourceId }
- }
- if (!echoed)
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-tool-result-unexpected',
- 'a download result follows a shell result without the token'
- )
-
- const download = GFS_DOWNLOAD_TOOL_OUTPUT.exec(contents[1])
- if (!download)
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-tool-result-unexpected',
- 'the second tool result is not a clerum__gfs_download result'
- )
- let receipt
- try {
- receipt = JSON.parse(download[1])
- } catch {
- return { stage: 'download-failed', code: failureCode(download[1]) }
- }
- if (
- !receipt ||
- typeof receipt !== 'object' ||
- receipt.delivery !== 'workspace_file' ||
- typeof receipt.sha256 !== 'string' ||
- !SHA256_HEX.test(receipt.sha256) ||
- !isByteCount(receipt.sizeBytes)
- )
- throw new FixtureLegacyLeaseError(
- 'legacy-lease-download-result-malformed',
- 'the download result is not a workspace_file receipt with sha256 and sizeBytes'
- )
- return {
- stage: 'answer',
- marker,
- sha256: receipt.sha256.toLowerCase(),
- sizeBytes: receipt.sizeBytes,
- }
-}
-
const isByteCount = value => Number.isSafeInteger(value) && value >= 0
const INTERNAL_HOST_SUFFIXES = [
@@ -945,10 +758,6 @@ export function createImageFixtureFetch(
documentReadRequests: 0,
documentAnswers: 0,
documentFailures: 0,
- legacyLeaseShellRequests: 0,
- legacyLeaseDownloadRequests: 0,
- legacyLeaseAnswers: 0,
- legacyLeaseFailures: 0,
},
attempts: [],
}
@@ -1102,74 +911,6 @@ export function createImageFixtureFetch(
}
)
}
-
- let legacyTurn
- try {
- legacyTurn = classifyLegacyLeaseTurn(body)
- } catch (error) {
- if (!(error instanceof FixtureLegacyLeaseError)) throw error
- state.counters.legacyLeaseFailures += 1
- return refuse(model, error.code, { code: error.code })
- }
- if (legacyTurn) {
- if (stream) {
- // A tool call cannot be answered in this fixture's SSE, exactly as for
- // the document read.
- state.counters.legacyLeaseFailures += 1
- return refuse(model, 'legacy-lease-stream-unsupported', {
- code: 'legacy_lease_stream_unsupported',
- })
- }
- sequence += 1
- const completionId = `${COMPLETION_ID_PREFIX}${runId.slice(-12)}-${sequence}`
- const reply = (responseKind, content, fields = {}) => {
- recordCall(model, null, responseKind, fields)
- publish()
- return jsonResponse(completionBody(completionId, model, content))
- }
- const call = (responseKind, name, args) => {
- recordCall(model, null, responseKind)
- publish()
- return jsonResponse(toolCallCompletionBody(completionId, model, name, args))
- }
- switch (legacyTurn.stage) {
- case 'shell':
- state.counters.legacyLeaseShellRequests += 1
- return call('legacy-lease-shell-requested', SHELL_TOOL, {
- command: `printf '%s${LEGACY_LEASE_SHELL_TOKEN_SUFFIX}\\n' ${legacyTurn.marker}`,
- })
- case 'download':
- state.counters.legacyLeaseDownloadRequests += 1
- return call('legacy-lease-download-requested', GFS_DOWNLOAD_TOOL, {
- drive: legacyTurn.drive,
- resourceId: legacyTurn.resourceId,
- })
- case 'shell-failed':
- state.counters.legacyLeaseFailures += 1
- return reply(
- 'legacy-lease-shell-failed',
- `${LEGACY_LEASE_SHELL_FAILED_PREFIX} code=${legacyTurn.code}`,
- { failureCode: legacyTurn.code }
- )
- case 'download-failed':
- state.counters.legacyLeaseFailures += 1
- return reply(
- 'legacy-lease-download-failed',
- `${LEGACY_LEASE_DOWNLOAD_FAILED_PREFIX} code=${legacyTurn.code}`,
- { failureCode: legacyTurn.code }
- )
- case 'answer':
- state.counters.legacyLeaseAnswers += 1
- return reply(
- 'legacy-lease-answer',
- `${LEGACY_LEASE_ANSWER_PREFIX} marker=${legacyTurn.marker} ` +
- `sha256=${legacyTurn.sha256} bytes=${legacyTurn.sizeBytes}`,
- { downloadSha256: legacyTurn.sha256, downloadBytes: legacyTurn.sizeBytes }
- )
- default:
- throw new Error(`Unhandled legacy-lease stage ${String(legacyTurn.stage)}`)
- }
- }
state.counters.textOnlyResponses += 1
return answer(model, TEXT_ONLY_CONTENT, 'text-only', { stream })
}
diff --git a/tests/e2e/fixtures/image-capabilities/provider.test.mjs b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
index 3beca6da5..a527950d2 100644
--- a/tests/e2e/fixtures/image-capabilities/provider.test.mjs
+++ b/tests/e2e/fixtures/image-capabilities/provider.test.mjs
@@ -1091,12 +1091,6 @@ const EXPECTED_REJECTION_REASONS = [
'image-png-malformed',
'image-png-unsupported-form',
'image-tile-grid-mismatch',
- 'legacy-lease-download-not-offered',
- 'legacy-lease-download-result-malformed',
- 'legacy-lease-shell-not-offered',
- 'legacy-lease-stream-unsupported',
- 'legacy-lease-tool-result-count',
- 'legacy-lease-tool-result-unexpected',
'provider-auth-mismatch',
'provider-body-invalid',
'provider-body-unsupported',
@@ -1233,308 +1227,3 @@ test('every rejected row records its closed-set reason and no request content',
assert.equal(accepted.getEvidence().attempts[0].responseKind, 'text-only')
assert.equal(Object.hasOwn(accepted.getEvidence().attempts[0], 'reason'), false)
})
-
-// ---------------------------------------------------------------------------
-// Legacy processing-lease restart journey (issue #1022): the fixture scripts
-// shell_exec, then clerum__gfs_download, then answers from the receipt.
-// ---------------------------------------------------------------------------
-
-const LEGACY_MARKER = 'legacy-lease-0123456789abcdef'
-const LEGACY_RESOURCE = 'a'.repeat(32)
-const LEGACY_SHA = 'b'.repeat(64)
-const LEGACY_TOKEN = `${LEGACY_MARKER}-shell-ok`
-
-const toolDefinition = name => ({
- type: 'function',
- function: { name, description: name, parameters: { type: 'object' } },
-})
-const LEGACY_TOOLS = [toolDefinition('shell_exec'), toolDefinition('clerum__gfs_download')]
-
-function legacyUserMessage(marker = LEGACY_MARKER) {
- return {
- role: 'user',
- content:
- '<turn-context>\ndate: 2026-10-08\n</turn-context>\n\n' +
- `Run the journey. LEGACY_LEASE_JOURNEY marker=${marker} drive=main resourceId=${LEGACY_RESOURCE}`,
- }
-}
-
-function legacyMessages(results) {
- const messages = [legacyUserMessage()]
- for (const [index, [name, content]] of results.entries()) {
- messages.push({
- role: 'assistant',
- content: null,
- tool_calls: [{ id: `call-${index}`, type: 'function', function: { name, arguments: '{}' } }],
- })
- messages.push({ role: 'tool', tool_call_id: `call-${index}`, content })
- }
- return messages
-}
-
-const wrapped = (name, content) =>
- `<tool_output name="${name}" sanitized="false">\n${content}\n</tool_output>`
-const shellResult = content => ['shell_exec', wrapped('shell_exec', content)]
-const downloadResult = content => ['clerum__gfs_download', wrapped('clerum__gfs_download', content)]
-const receipt = (overrides = {}) =>
- JSON.stringify({
- delivery: 'workspace_file',
- id: '11111111-2222-4333-8444-555555555555',
- path: '.gfs-downloads/input-1/report.pdf',
- sizeBytes: 42,
- sha256: LEGACY_SHA,
- ...overrides,
- })
-
-test('legacy-lease step 1: asks for a shell command whose output token is not in the command', async () => {
- const h = harness()
- const response = await call(
- h,
- chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS })
- )
- assert.equal(response.status, 200)
- const body = await response.json()
- assert.equal(body.choices[0].finish_reason, 'tool_calls')
- const [toolCall] = body.choices[0].message.tool_calls
- assert.equal(toolCall.function.name, 'shell_exec')
- const { command } = JSON.parse(toolCall.function.arguments)
- // The approval preview shows the marker, but only an executed command can
- // produce the joined token.
- assert.ok(command.includes(LEGACY_MARKER))
- assert.equal(command.includes(LEGACY_TOKEN), false)
- assert.equal(command, `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`)
- const evidence = h.getEvidence()
- assert.equal(evidence.counters.legacyLeaseShellRequests, 1)
- assert.deepEqual(evidence.attempts, [
- { model: VISUAL_MODEL, imageSha256: null, responseKind: 'legacy-lease-shell-requested' },
- ])
-})
-
-test('legacy-lease step 2: a shell result with the token asks for the GFS download', async () => {
- const h = harness()
- const response = await call(
- h,
- chatBody(VISUAL_MODEL, legacyMessages([shellResult(`${LEGACY_TOKEN}\n`)]), {
- tools: LEGACY_TOOLS,
- })
- )
- const body = await response.json()
- const [toolCall] = body.choices[0].message.tool_calls
- assert.equal(toolCall.function.name, 'clerum__gfs_download')
- assert.deepEqual(JSON.parse(toolCall.function.arguments), {
- drive: 'main',
- resourceId: LEGACY_RESOURCE,
- })
- assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 1)
- assert.equal(h.getEvidence().attempts[0].responseKind, 'legacy-lease-download-requested')
-})
-
-test('legacy-lease step 2: a lease failure before command start is answered with its store code', async () => {
- const h = harness()
- const response = await call(
- h,
- chatBody(
- VISUAL_MODEL,
- legacyMessages([
- shellResult(
- 'Processing lease failed before command start: GFS download store failed (download_busy)'
- ),
- ]),
- { tools: LEGACY_TOOLS }
- )
- )
- const body = await response.json()
- assert.equal(body.choices[0].finish_reason, 'stop')
- assert.equal(
- body.choices[0].message.content,
- 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=download_busy'
- )
- const evidence = h.getEvidence()
- assert.equal(evidence.counters.legacyLeaseFailures, 1)
- assert.equal(evidence.counters.legacyLeaseDownloadRequests, 0)
- assert.deepEqual(evidence.attempts, [
- {
- model: VISUAL_MODEL,
- imageSha256: null,
- responseKind: 'legacy-lease-shell-failed',
- failureCode: 'download_busy',
- },
- ])
-})
-
-test('legacy-lease step 2: output without the token or a known code is unrecognized, never a download', async () => {
- for (const content of [
- // The command echoed back is not the token.
- `printf '%s-shell-ok\\n' ${LEGACY_MARKER}`,
- 'GFS download store failed (not_a_store_code)',
- `${LEGACY_TOKEN}-extra`,
- ]) {
- const h = harness()
- const response = await call(
- h,
- chatBody(VISUAL_MODEL, legacyMessages([shellResult(content)]), { tools: LEGACY_TOOLS })
- )
- const body = await response.json()
- assert.equal(
- body.choices[0].message.content,
- 'LEGACY-LEASE-FIXTURE-SHELL-FAILED code=unrecognized',
- content
- )
- assert.equal(h.getEvidence().counters.legacyLeaseDownloadRequests, 0, content)
- }
-})
-
-test('legacy-lease step 3: the answer carries the receipt sha256 and byte count', async () => {
- const h = harness()
- const response = await call(
- h,
- chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt())]), {
- tools: LEGACY_TOOLS,
- })
- )
- const body = await response.json()
- assert.equal(
- body.choices[0].message.content,
- `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${LEGACY_SHA} bytes=42`
- )
- const evidence = h.getEvidence()
- assert.equal(evidence.counters.legacyLeaseAnswers, 1)
- assert.deepEqual(evidence.attempts, [
- {
- model: VISUAL_MODEL,
- imageSha256: null,
- responseKind: 'legacy-lease-answer',
- downloadSha256: LEGACY_SHA,
- downloadBytes: 42,
- },
- ])
-})
-
-test('legacy-lease step 3: a different receipt gives a different answer (not a constant)', async () => {
- const other = 'c'.repeat(64)
- const h = harness()
- const response = await call(
- h,
- chatBody(
- VISUAL_MODEL,
- legacyMessages([
- shellResult(LEGACY_TOKEN),
- downloadResult(receipt({ sha256: other, sizeBytes: 7 })),
- ]),
- { tools: LEGACY_TOOLS }
- )
- )
- const body = await response.json()
- assert.equal(
- body.choices[0].message.content,
- `LEGACY-LEASE-FIXTURE-OK marker=${LEGACY_MARKER} sha256=${other} bytes=7`
- )
-})
-
-test('legacy-lease step 3: a download error is answered with its store code', async () => {
- const h = harness()
- const response = await call(
- h,
- chatBody(
- VISUAL_MODEL,
- legacyMessages([
- shellResult(LEGACY_TOKEN),
- downloadResult('Error: GFS download store failed (workspace_unavailable)'),
- ]),
- { tools: LEGACY_TOOLS }
- )
- )
- const body = await response.json()
- assert.equal(
- body.choices[0].message.content,
- 'LEGACY-LEASE-FIXTURE-DOWNLOAD-FAILED code=workspace_unavailable'
- )
- assert.deepEqual(h.getEvidence().attempts, [
- {
- model: VISUAL_MODEL,
- imageSha256: null,
- responseKind: 'legacy-lease-download-failed',
- failureCode: 'workspace_unavailable',
- },
- ])
-})
-
-test('legacy-lease turns the fixture cannot use are refused with a closed-set reason', async () => {
- const cases = [
- [
- 'shell not offered',
- chatBody(VISUAL_MODEL, [legacyUserMessage()], {
- tools: [toolDefinition('clerum__gfs_download')],
- }),
- 'legacy-lease-shell-not-offered',
- ],
- [
- 'download not offered',
- chatBody(VISUAL_MODEL, legacyMessages([shellResult(LEGACY_TOKEN)]), {
- tools: [toolDefinition('shell_exec')],
- }),
- 'legacy-lease-download-not-offered',
- ],
- [
- 'first result from another tool',
- chatBody(VISUAL_MODEL, legacyMessages([downloadResult(receipt())]), { tools: LEGACY_TOOLS }),
- 'legacy-lease-tool-result-unexpected',
- ],
- [
- 'three results',
- chatBody(
- VISUAL_MODEL,
- legacyMessages([
- shellResult(LEGACY_TOKEN),
- downloadResult(receipt()),
- downloadResult(receipt()),
- ]),
- { tools: LEGACY_TOOLS }
- ),
- 'legacy-lease-tool-result-count',
- ],
- [
- 'receipt without sha256',
- chatBody(
- VISUAL_MODEL,
- legacyMessages([shellResult(LEGACY_TOKEN), downloadResult(receipt({ sha256: 'x' }))]),
- { tools: LEGACY_TOOLS }
- ),
- 'legacy-lease-download-result-malformed',
- ],
- [
- 'streamed turn',
- chatBody(VISUAL_MODEL, [legacyUserMessage()], { tools: LEGACY_TOOLS, stream: true }),
- 'legacy-lease-stream-unsupported',
- ],
- ]
- for (const [label, body, reason] of cases) {
- const h = harness()
- const response = await call(h, body)
- assert.ok(response.status >= 400, label)
- const evidence = h.getEvidence()
- assert.equal(evidence.counters.legacyLeaseFailures, 1, label)
- assert.deepEqual(
- evidence.attempts,
- [{ model: VISUAL_MODEL, imageSha256: null, responseKind: 'rejected', reason }],
- label
- )
- }
-})
-
-test('legacy-lease: a journey line without offered tools is an ordinary text-only request', async () => {
- const h = harness()
- const response = await call(h, chatBody(VISUAL_MODEL, [legacyUserMessage()]))
- const body = await response.json()
- assert.equal(body.choices[0].message.content, TEXT_ONLY_CONTENT)
- // Witness: the request reached the fixture and was answered as text.
- assert.equal(h.getEvidence().counters.textOnlyResponses, 1)
- assert.equal(h.getEvidence().counters.legacyLeaseShellRequests, 0)
-})
-
-test('legacy-lease failure codes are a frozen closed set that includes download_busy', () => {
- const codes = providerModule.LEGACY_LEASE_FAILURE_CODES
- assert.equal(Object.isFrozen(codes), true)
- assert.ok(codes.includes('download_busy'))
- assert.ok(codes.includes('unrecognized'))
-})
diff --git a/tests/e2e/fixtures/package-lock.json b/tests/e2e/fixtures/package-lock.json
index 5153d8811..86c865e52 100644
--- a/tests/e2e/fixtures/package-lock.json
+++ b/tests/e2e/fixtures/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.108",
+ "version": "1.0.107",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.108",
+ "version": "1.0.107",
"dependencies": {
"@kubernetes/client-node": "^1.0.0"
},
diff --git a/tests/e2e/fixtures/package.json b/tests/e2e/fixtures/package.json
index a1819b160..18d0e2843 100644
--- a/tests/e2e/fixtures/package.json
+++ b/tests/e2e/fixtures/package.json
@@ -1,6 +1,6 @@
{
"name": "@clerum/e2e-test-fixtures",
- "version": "1.0.108",
+ "version": "1.0.107",
"description": "E2E test fixtures and utilities for Clerum testing",
"type": "module",
"main": "index.ts",