Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript

fix(e2e): r4-m6 bind recovery to the immutable lease

Josuée7fdcfa3b1e7
diff --git a/tests/e2e/gfsUploadProductMutation.test.ts b/tests/e2e/gfsUploadProductMutation.test.ts
new file mode 100644
index 000000000..0a98288ab
--- /dev/null
+++ b/tests/e2e/gfsUploadProductMutation.test.ts
@@ -0,0 +1,237 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import {
+ type GfsUploadProductEnvPair,
+ type GfsUploadProductMutationAdapter,
+ type GfsUploadProductMutationLease,
+ applyGfsUploadProductMutation,
+ beginGfsUploadProductMutation,
+ recoverGfsUploadProductMutation,
+ restoreGfsUploadProductMutation,
+} from './gfsUploadProductMutation'
+
+const baseline: GfsUploadProductEnvPair = {
+ canonical: { kind: 'explicit', value: 100 },
+ alias: { kind: 'absent' },
+}
+const changed: GfsUploadProductEnvPair = {
+ canonical: { kind: 'explicit', value: 300 },
+ alias: { kind: 'explicit', value: 300 },
+}
+
+function fixture(initial: GfsUploadProductEnvPair = baseline) {
+ let marker: GfsUploadProductMutationLease | undefined
+ let current = structuredClone(initial)
+ let createObservedPair: GfsUploadProductEnvPair | undefined
+ const authorizeNewMutation = vi.fn(async () => ({
+ context: 'clerum-issue-300-deadbeef',
+ branch: 'feat/issue-300',
+ worktreeId: 'worktree',
+ gitHead: 'head',
+ clusterFingerprint: 'fingerprint',
+ preGateMarkerUid: 'pre-gate-uid',
+ }))
+ const authorizeRecovery = vi.fn(async () => undefined)
+ const applyPair = vi.fn(async (pair: GfsUploadProductEnvPair) => {
+ current = structuredClone(pair)
+ })
+ const verifyPair = vi.fn(async (pair: GfsUploadProductEnvPair) => {
+ expect(current).toEqual(pair)
+ })
+ const adapter: GfsUploadProductMutationAdapter = {
+ authorizeNewMutation,
+ authorizeRecovery,
+ readBaseline: vi.fn(async () => structuredClone(baseline)),
+ readMarker: vi.fn(async () => (marker ? structuredClone(marker) : undefined)),
+ createMarker: vi.fn(async value => {
+ createObservedPair = structuredClone(current)
+ marker = { uid: 'marker-uid', marker: structuredClone(value) }
+ return { uid: marker.uid }
+ }),
+ applyPair,
+ verifyPair,
+ deleteMarker: vi.fn(async lease => {
+ expect(marker?.uid).toBe(lease.uid)
+ marker = undefined
+ }),
+ newHolder: () => 'holder',
+ now: () => '2026-08-24T00:00:00.000Z',
+ waitBeforeRetry: vi.fn(async () => undefined),
+ }
+ return {
+ adapter,
+ authorizeNewMutation,
+ authorizeRecovery,
+ applyPair,
+ verifyPair,
+ getMarker: () => marker,
+ setMarker: (value: GfsUploadProductMutationLease | undefined) => {
+ marker = value
+ },
+ getCurrent: () => current,
+ getCreateObservedPair: () => createObservedPair,
+ }
+}
+
+describe('GFS Upload v2 product-limit mutation transaction', () => {
+ beforeEach(() => vi.restoreAllMocks())
+
+ it('persists the exact two-input baseline before the first mutation', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+
+ expect(runtime.getCreateObservedPair()).toEqual(baseline)
+ expect(lease.marker.baseline).toEqual(baseline)
+
+ await applyGfsUploadProductMutation(runtime.adapter, lease, changed)
+ expect(runtime.getCurrent()).toEqual(changed)
+ })
+
+ it('blocks a new mutation while a recovery marker exists', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+
+ await expect(beginGfsUploadProductMutation(runtime.adapter)).rejects.toThrow(
+ 'recovery is required'
+ )
+ expect(runtime.getMarker()).toEqual(lease)
+ })
+
+ it('restores an owned exact baseline without repeating new-mutation authorization', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ await applyGfsUploadProductMutation(runtime.adapter, lease, changed)
+
+ runtime.authorizeNewMutation.mockRejectedValueOnce(new Error('worktree became dirty'))
+ await restoreGfsUploadProductMutation(runtime.adapter, lease)
+
+ expect(runtime.authorizeNewMutation).toHaveBeenCalledTimes(1)
+ expect(runtime.authorizeRecovery).toHaveBeenCalledWith(lease.marker)
+ expect(runtime.getCurrent()).toEqual(baseline)
+ expect(runtime.getMarker()).toBeUndefined()
+ })
+
+ it('lets another recovery process restore from the durable marker', async () => {
+ const first = fixture()
+ const lease = await beginGfsUploadProductMutation(first.adapter)
+ await applyGfsUploadProductMutation(first.adapter, lease, changed)
+
+ const recovery = fixture(changed)
+ recovery.setMarker(lease)
+ await recoverGfsUploadProductMutation(recovery.adapter, {
+ uid: lease.uid,
+ holder: lease.marker.holder,
+ })
+
+ expect(recovery.authorizeNewMutation).not.toHaveBeenCalled()
+ expect(recovery.authorizeRecovery).toHaveBeenCalledWith(lease.marker)
+ expect(recovery.getCurrent()).toEqual(baseline)
+ expect(recovery.getMarker()).toBeUndefined()
+ })
+
+ it('retries restoration and retains the marker when convergence never verifies', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ runtime.verifyPair.mockRejectedValue(new Error('transient rollout failure'))
+
+ await expect(restoreGfsUploadProductMutation(runtime.adapter, lease)).rejects.toThrow(
+ 'recovery remains required after 3 attempts'
+ )
+ expect(runtime.applyPair).toHaveBeenCalledTimes(3)
+ expect(runtime.adapter.waitBeforeRetry).toHaveBeenCalledTimes(2)
+ expect(runtime.getMarker()).toEqual(lease)
+ expect(runtime.adapter.deleteMarker).not.toHaveBeenCalled()
+ })
+
+ it('fails closed when immutable lease evidence changes', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ runtime.setMarker({ ...lease, uid: 'replacement-uid' })
+
+ await expect(applyGfsUploadProductMutation(runtime.adapter, lease, changed)).rejects.toThrow(
+ 'recovery marker changed'
+ )
+ expect(runtime.applyPair).not.toHaveBeenCalled()
+ })
+
+ it('does not let an unconfirmed second process recover an active lease', async () => {
+ const active = fixture(changed)
+ const lease = await beginGfsUploadProductMutation(active.adapter)
+ const second = fixture(changed)
+ second.setMarker(lease)
+
+ await expect(
+ recoverGfsUploadProductMutation(second.adapter, {
+ uid: lease.uid,
+ holder: 'different-holder',
+ })
+ ).rejects.toThrow('takeover evidence does not match')
+ expect(second.applyPair).not.toHaveBeenCalled()
+ expect(second.getMarker()).toEqual(lease)
+ expect(second.getCurrent()).toEqual(changed)
+ })
+
+ it('retains recovery state when the marker is replaced during a long apply', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ const replacement = {
+ ...lease,
+ uid: 'replacement-uid',
+ marker: { ...lease.marker, holder: 'replacement-holder' },
+ }
+ runtime.applyPair.mockImplementationOnce(async () => {
+ runtime.setMarker(replacement)
+ })
+ runtime.verifyPair.mockResolvedValueOnce(undefined)
+
+ await expect(applyGfsUploadProductMutation(runtime.adapter, lease, changed)).rejects.toThrow(
+ 'recovery marker changed'
+ )
+ expect(runtime.getMarker()).toEqual(replacement)
+ expect(runtime.getMarker()?.marker.baseline).toEqual(baseline)
+ await expect(beginGfsUploadProductMutation(runtime.adapter)).rejects.toThrow(
+ 'recovery is required'
+ )
+ })
+
+ it('retains recovery state when cluster ownership changes during a long apply', async () => {
+ const runtime = fixture()
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ runtime.authorizeRecovery
+ .mockResolvedValueOnce(undefined)
+ .mockRejectedValueOnce(new Error('cluster ownership marker was replaced'))
+
+ await expect(applyGfsUploadProductMutation(runtime.adapter, lease, changed)).rejects.toThrow(
+ 'cluster ownership marker was replaced'
+ )
+ expect(runtime.getMarker()).toEqual(lease)
+ expect(runtime.getMarker()?.marker.baseline).toEqual(baseline)
+ await expect(beginGfsUploadProductMutation(runtime.adapter)).rejects.toThrow(
+ 'recovery is required'
+ )
+ })
+
+ it('cannot delete a replacement marker after the final lease check', async () => {
+ const runtime = fixture(changed)
+ const lease = await beginGfsUploadProductMutation(runtime.adapter)
+ const replacement = {
+ ...lease,
+ uid: 'replacement-uid',
+ marker: { ...lease.marker, holder: 'replacement-holder' },
+ }
+ runtime.adapter.deleteMarker = vi.fn(async expected => {
+ runtime.setMarker(replacement)
+ if (runtime.getMarker()?.uid !== expected.uid) {
+ throw new Error('Kubernetes UID precondition failed')
+ }
+ })
+
+ await expect(restoreGfsUploadProductMutation(runtime.adapter, lease)).rejects.toThrow(
+ 'recovery remains required'
+ )
+ expect(runtime.getMarker()).toEqual(replacement)
+ expect(runtime.getMarker()?.marker.baseline).toEqual(baseline)
+ await expect(beginGfsUploadProductMutation(runtime.adapter)).rejects.toThrow(
+ 'recovery is required'
+ )
+ })
+})