Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
44.1 KB
Raw Blame
import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'
import fc from 'fast-check'
import {
  SANDBOX_UI_COOKIE_TTL_MS,
  SANDBOX_UI_RATE_LIMIT_MS,
  SANDBOX_UI_REFRESH_INTERVAL_MS,
  _isPinnedForTests,
  _resetSandboxUiRefreshForTests,
  cancelSandboxUiRefresh,
  handleEmbedRefreshRequest,
  handleSandboxUiUnauthorizedNavigation,
  notifySandboxUiHostResumed,
  startSandboxUiRefresh,
} from '../sandboxUiSessionRefresh.js'
import type {
  SandboxUiInstallCookieFn,
  SandboxUiRefreshFn,
  StartSandboxUiRefreshArgs,
} from '../sandboxUiSessionRefresh.js'

type Listener = () => void

class FakeBrowserWindow {
  destroyed = false
  visible = true
  minimized = false
  private listeners = new Map<string, Set<Listener>>()
  isDestroyed(): boolean {
    return this.destroyed
  }
  isVisible(): boolean {
    return this.visible
  }
  isMinimized(): boolean {
    return this.minimized
  }
  on(event: string, handler: Listener): this {
    if (!this.listeners.has(event)) this.listeners.set(event, new Set())
    this.listeners.get(event)!.add(handler)
    return this
  }
  removeListener(event: string, handler: Listener): this {
    this.listeners.get(event)?.delete(handler)
    return this
  }
  emit(event: string): void {
    for (const h of this.listeners.get(event) ?? []) h()
  }
  listenerCount(event: string): number {
    return this.listeners.get(event)?.size ?? 0
  }
}

// Two independent clocks: the wall clock the module reads through `now()`, and
// the fake timers that drive `setTimeout`. Awake, the wall clock follows the
// timer clock (`advance`, firing each timer at its own instant); asleep, only
// the wall clock moves (`sleep`) — the case a timer-only design gets wrong.
// `performance` is faked only so the test can read the timer clock; the module
// never touches it, and Date stays real.
let sleptMs = 0
const wallNow = (): number => 1_800_000_000_000 + performance.now() + sleptMs

async function advance(ms: number): Promise<void> {
  await vi.advanceTimersByTimeAsync(ms)
}

function sleep(ms: number): void {
  sleptMs += ms
}

/** Lets already-started promise chains (mint → install → reschedule) settle. */
async function flush(): Promise<void> {
  await vi.advanceTimersByTimeAsync(0)
}

beforeEach(() => {
  // Date is deliberately left real: the module must take its wall clock from
  // the injected `now()`, never from whatever the timer clock says.
  vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] })
  sleptMs = 0
  _resetSandboxUiRefreshForTests()
})

afterEach(() => {
  _resetSandboxUiRefreshForTests()
  vi.useRealTimers()
})

function makeArgs(
  overrides: {
    refresh?: SandboxUiRefreshFn
    installCookie?: SandboxUiInstallCookieFn
    onError?: (error: Error) => void
    onRecovered?: () => void
    reload?: () => void
    parentWindow?: FakeBrowserWindow
    webContentsId?: number
  } = {}
): StartSandboxUiRefreshArgs {
  const parentWindow = overrides.parentWindow ?? new FakeBrowserWindow()
  return {
    recipeNs: 'sandbox-recipes',
    recipeName: 'r1',
    webContentsId: overrides.webContentsId ?? 42,
    // Cast through unknown — the consumer only calls a small subset of
    // BrowserWindow's interface and we don't depend on Electron's
    // internals in the test.
    parentWindow: parentWindow as unknown as Parameters<
      typeof startSandboxUiRefresh
    >[0]['parentWindow'],
    refresh:
      overrides.refresh ??
      vi.fn<SandboxUiRefreshFn>(async () => ({ setCookie: 'clerum_sandbox_ui_session=NEW' })),
    installCookie:
      overrides.installCookie ?? vi.fn<SandboxUiInstallCookieFn>(async () => undefined),
    onError: overrides.onError ?? vi.fn(),
    onRecovered: overrides.onRecovered ?? vi.fn(),
    reload: overrides.reload ?? vi.fn(),
    now: wallNow,
  }
}

const mintOk = (): Promise<{ setCookie: string }> =>
  Promise.resolve({ setCookie: 'clerum_sandbox_ui_session=X' })
const mintDown = (): Promise<never> => Promise.reject(new Error('control-api 503'))

describe('startSandboxUiRefresh', () => {
  it('pins the sender and arms the visibility-driven timer', () => {
    const args = makeArgs()
    startSandboxUiRefresh(args)
    expect(_isPinnedForTests(args.webContentsId)).toBe(true)
  })

  it('replaces an earlier active refresh when a new view mounts', () => {
    const a = makeArgs({ webContentsId: 1 })
    const b = makeArgs({ webContentsId: 2 })
    startSandboxUiRefresh(a)
    startSandboxUiRefresh(b)
    expect(_isPinnedForTests(1)).toBe(false)
    expect(_isPinnedForTests(2)).toBe(true)
  })

  it('does NOT arm the timer when the parent window is hidden at mount', async () => {
    const window = new FakeBrowserWindow()
    window.visible = false
    const refresh = vi.fn<SandboxUiRefreshFn>(async () => ({
      setCookie: 'clerum_sandbox_ui_session=X',
    }))
    const args = makeArgs({ parentWindow: window, refresh })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1000)
    expect(refresh).not.toHaveBeenCalled()
  })
})

describe('refresh timer', () => {
  it('fires at SANDBOX_UI_REFRESH_INTERVAL_MS and reschedules itself', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(async () => ({
      setCookie: 'clerum_sandbox_ui_session=X',
    }))
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const args = makeArgs({ refresh, installCookie })
    startSandboxUiRefresh(args)

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS - 1)
    expect(refresh).not.toHaveBeenCalled()

    await advance(2)
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(installCookie).toHaveBeenCalledWith('clerum_sandbox_ui_session=X')

    // After the first refresh, the next tick is 270 s later.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS)
    expect(refresh).toHaveBeenCalledTimes(2)
  })

  it('cancels on hide and refreshes at once on a restore after the cookie expired', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const args = makeArgs({ parentWindow: window, refresh })
    startSandboxUiRefresh(args)

    // Halfway to the next refresh, the user minimizes — timer should be
    // cleared and no refresh should fire even if we wait the full interval.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS / 2)
    window.minimized = true
    window.emit('minimize')
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS * 2)
    expect(refresh).not.toHaveBeenCalled()

    // The cookie died while minimized: restoring re-mints immediately.
    window.minimized = false
    window.emit('restore')
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS - 1)
    expect(refresh).toHaveBeenCalledTimes(1)
    await advance(2)
    expect(refresh).toHaveBeenCalledTimes(2)
  })
})

describe('cancelSandboxUiRefresh', () => {
  it('unpins the sender, clears the timer, and removes parent listeners', () => {
    const window = new FakeBrowserWindow()
    const args = makeArgs({ parentWindow: window })
    startSandboxUiRefresh(args)

    expect(_isPinnedForTests(args.webContentsId)).toBe(true)
    expect(window.listenerCount('hide')).toBe(1)
    expect(window.listenerCount('show')).toBe(1)

    cancelSandboxUiRefresh()

    expect(_isPinnedForTests(args.webContentsId)).toBe(false)
    expect(window.listenerCount('hide')).toBe(0)
    expect(window.listenerCount('show')).toBe(0)
    expect(window.listenerCount('minimize')).toBe(0)
    expect(window.listenerCount('restore')).toBe(0)
    expect(window.listenerCount('focus')).toBe(0)
  })

  it('stays terminal: no retry, return, embed request or 401 revives a cancelled loop', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const reload = vi.fn()
    const args = makeArgs({ parentWindow: window, refresh, reload })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(1)

    // Cancelled mid-backoff, with a retry already scheduled.
    cancelSandboxUiRefresh()
    sleep(SANDBOX_UI_COOKIE_TTL_MS)
    window.emit('show')
    notifySandboxUiHostResumed()
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await expect(handleEmbedRefreshRequest(args.webContentsId)).rejects.toThrow(/unpinned sender/)
    await advance(SANDBOX_UI_COOKIE_TTL_MS * 2)

    expect(refresh).toHaveBeenCalledTimes(1)
    expect(reload).not.toHaveBeenCalled()
    expect(_isPinnedForTests(args.webContentsId)).toBe(false)
  })
})

describe('handleEmbedRefreshRequest (Decision 11 — IPC pinning)', () => {
  it('rejects refresh requests from senders not in the pinning map', async () => {
    await expect(handleEmbedRefreshRequest(999)).rejects.toThrow(/unpinned sender/)
  })

  it('rejects refresh requests within the 30 s rate-limit window', async () => {
    const args = makeArgs()
    startSandboxUiRefresh(args)
    // Mint counts as t=0 — an immediate IPC must be rate-limited.
    await expect(handleEmbedRefreshRequest(args.webContentsId)).rejects.toThrow(/rate-limited/)
  })

  it('allows IPC refresh once the rate-limit window has passed', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(async () => ({
      setCookie: 'clerum_sandbox_ui_session=NEW',
    }))
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const args = makeArgs({ refresh, installCookie })
    startSandboxUiRefresh(args)

    // Advance past the rate limit but well before the scheduled timer.
    await advance(SANDBOX_UI_RATE_LIMIT_MS + 100)
    await handleEmbedRefreshRequest(args.webContentsId)

    expect(refresh).toHaveBeenCalledTimes(1)
    expect(installCookie).toHaveBeenCalledWith('clerum_sandbox_ui_session=NEW')
  })

  it('counts an accepted request against the rate limit even when its mint fails', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const args = makeArgs({ refresh })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_RATE_LIMIT_MS + 100)

    await handleEmbedRefreshRequest(args.webContentsId)
    expect(refresh).toHaveBeenCalledTimes(1)
    await expect(handleEmbedRefreshRequest(args.webContentsId)).rejects.toThrow(/rate-limited/)
    // The first request's mint failed, yet the second is still rate-limited and mints nothing.
    expect(refresh).toHaveBeenCalledTimes(1)
  })
})

describe('session expiry by wall clock', () => {
  it('a show after the cookie expired while the timers were frozen refreshes at once', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh }))

    // Machine asleep for 10 min: the wall clock moves, the timer does not.
    sleep(10 * 60 * 1000)
    window.emit('show')
    await flush()

    expect(refresh).toHaveBeenCalledTimes(1)
  })

  it('a host resume after the cookie expired while asleep refreshes at once', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    startSandboxUiRefresh(makeArgs({ refresh }))

    sleep(10 * 60 * 1000)
    notifySandboxUiHostResumed()
    await flush()

    expect(refresh).toHaveBeenCalledTimes(1)
  })

  it('a restore before expiry re-arms the timer to the remaining time, not a full interval', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh }))

    await advance(100_000)
    window.minimized = true
    window.emit('minimize')
    await advance(50_000)
    window.minimized = false
    window.emit('restore')
    await flush()
    expect(refresh).not.toHaveBeenCalled()

    const remaining = SANDBOX_UI_REFRESH_INTERVAL_MS - 150_000
    await advance(remaining - 1)
    expect(refresh).not.toHaveBeenCalled()
    await advance(2)
    expect(refresh).toHaveBeenCalledTimes(1)
  })

  it('a focus after the timer clock lagged behind the wall clock re-arms to the wall-clock remainder', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh }))

    await advance(100_000)
    sleep(100_000)
    window.emit('focus')
    await flush()
    expect(refresh).not.toHaveBeenCalled()

    // 70 s of wall time remain; the timer armed at mount still had 170 s.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS - 200_000 + 1)
    expect(refresh).toHaveBeenCalledTimes(1)
  })
})

describe('mint failure backoff', () => {
  it('retries a failed mint instead of stopping, keeping the sender pinned', async () => {
    const refresh = vi
      .fn<SandboxUiRefreshFn>()
      .mockImplementationOnce(mintDown)
      .mockImplementationOnce(mintDown)
      .mockImplementation(mintOk)
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const onError = vi.fn()
    const args = makeArgs({ refresh, installCookie, onError })
    startSandboxUiRefresh(args)

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(_isPinnedForTests(args.webContentsId)).toBe(true)

    await advance(1_000)
    expect(refresh).toHaveBeenCalledTimes(2)
    await advance(2_000)
    expect(refresh).toHaveBeenCalledTimes(3)
    expect(installCookie).toHaveBeenCalledWith('clerum_sandbox_ui_session=X')
    expect(onError).not.toHaveBeenCalled()

    // Back on the regular schedule, counted from the mint that succeeded
    // (requested 1 ms before the clock now reads).
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS - 2)
    expect(refresh).toHaveBeenCalledTimes(3)
    await advance(2)
    expect(refresh).toHaveBeenCalledTimes(4)
  })

  it('reports once the cookie has been dead for the grace period, and stays pinned', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onError = vi.fn()
    const args = makeArgs({ refresh, onError })
    startSandboxUiRefresh(args)

    // First failure at 270 s; the cookie expires at 300 s, retries continue
    // to 360 s: attempts at 271, 273, 278, 293, 323, 353 — the next (383)
    // would land past the deadline.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(100_000)
    expect(refresh).toHaveBeenCalledTimes(7)
    expect(onError).toHaveBeenCalledTimes(1)
    expect(onError.mock.calls[0]![0].message).toContain('503')
    expect(_isPinnedForTests(args.webContentsId)).toBe(true)

    await advance(SANDBOX_UI_COOKIE_TTL_MS * 2)
    expect(refresh).toHaveBeenCalledTimes(7)
  })

  it('gives a mint that fails right after waking a full retry window', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onError = vi.fn()
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, onError }))

    sleep(60 * 60 * 1000)
    window.emit('show')
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(onError).not.toHaveBeenCalled()

    await advance(1_000)
    expect(refresh).toHaveBeenCalledTimes(2)
  })

  it('an embed request after the retries ran out resumes the loop', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const onError = vi.fn()
    const args = makeArgs({ refresh, installCookie, onError })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(100_000)
    expect(onError).toHaveBeenCalledTimes(1)
    const callsWhenExhausted = refresh.mock.calls.length

    refresh.mockImplementation(mintOk)
    await handleEmbedRefreshRequest(args.webContentsId)
    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)
    expect(installCookie).toHaveBeenCalledWith('clerum_sandbox_ui_session=X')
    // The 30 s rate limit still applies to the resumed loop.
    await expect(handleEmbedRefreshRequest(args.webContentsId)).rejects.toThrow(/rate-limited/)

    // And the timer is back.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 2)
  })
})

describe('top-level 401 recovery', () => {
  it('re-mints and reloads exactly once per mint', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const reload = vi.fn()
    const args = makeArgs({ refresh, installCookie, reload })
    startSandboxUiRefresh(args)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(installCookie).toHaveBeenCalledTimes(1)
    expect(reload).toHaveBeenCalledTimes(1)

    // The reload under the recovery mint 401s again: no loop.
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(reload).toHaveBeenCalledTimes(1)

    // A later scheduled mint earns a fresh recovery.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(2)
    expect(reload).toHaveBeenCalledTimes(1)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(3)
    expect(reload).toHaveBeenCalledTimes(2)
  })

  it('reloads only once the recovery mint succeeds, after backoff', async () => {
    const refresh = vi
      .fn<SandboxUiRefreshFn>()
      .mockImplementationOnce(mintDown)
      .mockImplementation(mintOk)
    const reload = vi.fn()
    const args = makeArgs({ refresh, reload })
    startSandboxUiRefresh(args)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(reload).not.toHaveBeenCalled()
    // Further 401s while the recovery is pending add no mints.
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)

    await advance(1_000)
    expect(refresh).toHaveBeenCalledTimes(2)
    expect(reload).toHaveBeenCalledTimes(1)
  })

  it('ignores a 401 from a webContents that is not the pinned embed', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const reload = vi.fn()
    startSandboxUiRefresh(makeArgs({ refresh, reload, webContentsId: 42 }))

    handleSandboxUiUnauthorizedNavigation(7)
    await flush()

    expect(refresh).not.toHaveBeenCalled()
    expect(reload).not.toHaveBeenCalled()
  })
})

describe('recovery after retries ran out or went stale', () => {
  it('a 401 after its recovery mint exhausted its retries reopens them and reloads on success', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const reload = vi.fn()
    const onError = vi.fn()
    const args = makeArgs({ refresh, reload, onError })
    startSandboxUiRefresh(args)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await advance(400_000)
    expect(onError).toHaveBeenCalledTimes(1)
    const callsWhenExhausted = refresh.mock.calls.length

    // rpc-proxy is back; the user reloads and the page 401s again.
    refresh.mockImplementation(mintOk)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()

    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)
    expect(reload).toHaveBeenCalledTimes(1)
  })

  it('a 401 once the recovery cookie expired is recovered, even while the timer mint is retrying', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const reload = vi.fn()
    const args = makeArgs({ refresh, reload })
    startSandboxUiRefresh(args)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(reload).toHaveBeenCalledTimes(1)

    // The timer mint fails and keeps retrying; the recovery cookie dies.
    refresh.mockImplementation(mintDown)
    await advance(SANDBOX_UI_COOKIE_TTL_MS + 5_000)
    // The user navigates: the dead cookie 401s. Its own mint fails too, but
    // the retry that lands must still reload the dead page.
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    refresh.mockImplementation(mintOk)
    await advance(30_000)

    expect(reload).toHaveBeenCalledTimes(2)
  })

  it('a 401 once the recovery cookie expired reopens retries that ran out', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const reload = vi.fn()
    const onError = vi.fn()
    const args = makeArgs({ refresh, reload, onError })
    startSandboxUiRefresh(args)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(reload).toHaveBeenCalledTimes(1)

    refresh.mockImplementation(mintDown)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 100_000)
    expect(onError).toHaveBeenCalledTimes(1)
    const callsWhenExhausted = refresh.mock.calls.length

    refresh.mockImplementation(mintOk)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()

    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)
    expect(reload).toHaveBeenCalledTimes(2)
  })

  it('a restore after sleeping through a failure gets a fresh retry window instead of reporting', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onError = vi.fn()
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, onError }))

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(1)
    window.minimized = true
    window.emit('minimize')
    sleep(10 * 60 * 1000)
    window.minimized = false
    window.emit('restore')
    await flush()

    expect(refresh).toHaveBeenCalledTimes(2)
    expect(onError).not.toHaveBeenCalled()
    await advance(1_000)
    expect(refresh).toHaveBeenCalledTimes(3)
    expect(onError).not.toHaveBeenCalled()
  })

  it('a host resume after the retries ran out re-mints the session', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const onError = vi.fn()
    startSandboxUiRefresh(makeArgs({ refresh, installCookie, onError }))
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(100_000)
    expect(onError).toHaveBeenCalledTimes(1)
    const callsWhenExhausted = refresh.mock.calls.length

    // The machine sleeps; it wakes with a healthy network.
    sleep(10 * 60 * 1000)
    refresh.mockImplementation(mintOk)
    notifySandboxUiHostResumed()
    await flush()

    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)
    expect(installCookie).toHaveBeenCalledWith('clerum_sandbox_ui_session=X')
  })

  it('a focus burst on an exhausted loop resumes it once per rate-limit window', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh }))
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(100_000)
    const callsWhenExhausted = refresh.mock.calls.length

    for (let i = 0; i < 5; i += 1) window.emit('focus')
    await flush()
    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)

    // Once that resumed episode runs out too, the next burst (well past the
    // window) resumes once more — and only once.
    await advance(120_000)
    const callsWhenExhaustedAgain = refresh.mock.calls.length
    for (let i = 0; i < 5; i += 1) window.emit('focus')
    await flush()
    expect(refresh).toHaveBeenCalledTimes(callsWhenExhaustedAgain + 1)
  })
})

describe('concurrent triggers share one mint', () => {
  function heldMint(): { refresh: ReturnType<typeof vi.fn<SandboxUiRefreshFn>>; land: () => void } {
    let land: () => void = () => undefined
    const refresh = vi.fn<SandboxUiRefreshFn>(
      () =>
        new Promise(resolve => {
          land = () => resolve({ setCookie: 'clerum_sandbox_ui_session=X' })
        })
    )
    return { refresh, land: () => land() }
  }

  it('an embed request during a timer mint joins it instead of minting again', async () => {
    const { refresh, land } = heldMint()
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
    const args = makeArgs({ refresh, installCookie })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(1)

    const request = handleEmbedRefreshRequest(args.webContentsId)
    land()
    await request

    expect(refresh).toHaveBeenCalledTimes(1)
    expect(installCookie).toHaveBeenCalledTimes(1)
  })

  it('a 401 during a timer mint reloads once that mint lands, without minting again', async () => {
    const { refresh, land } = heldMint()
    const reload = vi.fn()
    const args = makeArgs({ refresh, reload })
    startSandboxUiRefresh(args)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(reload).not.toHaveBeenCalled()

    land()
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
    expect(reload).toHaveBeenCalledTimes(1)
  })

  it('a 401 during the mint of a restore after the recovery cookie expired reloads once it lands', async () => {
    const window = new FakeBrowserWindow()
    const { refresh, land } = heldMint()
    const reload = vi.fn()
    const args = makeArgs({ parentWindow: window, refresh, reload })
    startSandboxUiRefresh(args)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    land()
    await flush()
    expect(reload).toHaveBeenCalledTimes(1)

    window.minimized = true
    window.emit('minimize')
    await advance(SANDBOX_UI_COOKIE_TTL_MS + 60_000)
    window.minimized = false
    window.emit('restore')
    await flush()
    expect(refresh).toHaveBeenCalledTimes(2)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    land()
    await flush()
    expect(refresh).toHaveBeenCalledTimes(2)
    expect(reload).toHaveBeenCalledTimes(2)
  })

  it('a 401 under the recovery cookie while a newer mint is in flight reloads once it lands', async () => {
    const { refresh, land } = heldMint()
    const reload = vi.fn()
    const args = makeArgs({ refresh, reload })
    startSandboxUiRefresh(args)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    land()
    await flush()
    expect(reload).toHaveBeenCalledTimes(1)

    // The timer mint starts while the recovery cookie is still valid.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(2)

    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    land()
    await flush()
    expect(refresh).toHaveBeenCalledTimes(2)
    expect(reload).toHaveBeenCalledTimes(2)
  })

  it('a focus burst neither duplicates nor advances the mint', async () => {
    const window = new FakeBrowserWindow()
    const { refresh, land } = heldMint()
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh }))

    await advance(100_000)
    for (let i = 0; i < 10; i += 1) window.emit('focus')
    await flush()
    expect(refresh).not.toHaveBeenCalled()

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS - 100_000 - 1)
    expect(refresh).not.toHaveBeenCalled()
    await advance(2)
    expect(refresh).toHaveBeenCalledTimes(1)

    // While that mint is in flight, more focus adds nothing.
    for (let i = 0; i < 10; i += 1) window.emit('focus')
    await flush()
    land()
    await flush()
    expect(refresh).toHaveBeenCalledTimes(1)
  })
})

describe('recovery signal after retries ran out', () => {
  /** From mount to the point where the loop gives up on a mint that keeps failing. */
  async function runOutOfRetries(): Promise<void> {
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(100_000)
  }

  it('is never signalled for mints that succeed, nor for a failure retried within the grace period', async () => {
    const refresh = vi.fn<SandboxUiRefreshFn>(mintOk)
    const onError = vi.fn()
    const onRecovered = vi.fn()
    startSandboxUiRefresh(makeArgs({ refresh, onError, onRecovered }))

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(refresh).toHaveBeenCalledTimes(1)

    refresh.mockImplementationOnce(mintDown)
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    await advance(1_000)
    expect(refresh).toHaveBeenCalledTimes(3)
    expect(onError).not.toHaveBeenCalled()
    expect(onRecovered).not.toHaveBeenCalled()
  })

  it('is signalled once, after the cookie is installed, when a return resumes the loop and the mint succeeds', async () => {
    const window = new FakeBrowserWindow()
    const log: string[] = []
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => {
      log.push('installCookie')
    })
    const onError = vi.fn(() => log.push('onError'))
    const onRecovered = vi.fn(() => log.push('onRecovered'))
    startSandboxUiRefresh(
      makeArgs({ parentWindow: window, refresh, installCookie, onError, onRecovered })
    )
    await runOutOfRetries()
    expect(log).toEqual(['onError'])

    refresh.mockImplementation(mintOk)
    window.emit('focus')
    await flush()
    expect(log).toEqual(['onError', 'installCookie', 'onRecovered'])

    // The next timer mint has nothing left to recover from.
    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(log).toEqual(['onError', 'installCookie', 'onRecovered', 'installCookie'])
  })

  it.each([
    ['a parent window focus', (window: FakeBrowserWindow) => window.emit('focus')],
    ['a host resume', () => notifySandboxUiHostResumed()],
    [
      'an embed refresh request',
      (_window: FakeBrowserWindow, webContentsId: number) =>
        void handleEmbedRefreshRequest(webContentsId),
    ],
  ])('is signalled when %s resumes the loop and the mint succeeds', async (_name, resume) => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onRecovered = vi.fn()
    const args = makeArgs({ parentWindow: window, refresh, onRecovered })
    startSandboxUiRefresh(args)
    await runOutOfRetries()
    const callsWhenExhausted = refresh.mock.calls.length

    refresh.mockImplementation(mintOk)
    resume(window, args.webContentsId)
    await flush()
    expect(refresh).toHaveBeenCalledTimes(callsWhenExhausted + 1)
    expect(onRecovered).toHaveBeenCalledTimes(1)
  })

  it('is signalled once for a streak of failures reported twice', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onError = vi.fn()
    const onRecovered = vi.fn()
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, onError, onRecovered }))
    await runOutOfRetries()

    // The resumed episode fails all the way through as well.
    window.emit('focus')
    await advance(120_000)
    expect(onError).toHaveBeenCalledTimes(2)

    refresh.mockImplementation(mintOk)
    window.emit('focus')
    await flush()
    expect(onRecovered).toHaveBeenCalledTimes(1)
  })

  it('is signalled again for a failure reported after an earlier recovery', async () => {
    const window = new FakeBrowserWindow()
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const onError = vi.fn()
    const onRecovered = vi.fn()
    startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, onError, onRecovered }))
    await runOutOfRetries()
    refresh.mockImplementation(mintOk)
    window.emit('focus')
    await flush()
    expect(onRecovered).toHaveBeenCalledTimes(1)

    refresh.mockImplementation(mintDown)
    await runOutOfRetries()
    expect(onError).toHaveBeenCalledTimes(2)

    refresh.mockImplementation(mintOk)
    window.emit('focus')
    await flush()
    expect(onRecovered).toHaveBeenCalledTimes(2)
  })

  it.each([
    ['the mint', 'refresh'],
    ['the cookie install', 'installCookie'],
  ] as const)(
    'is not signalled when the loop is cancelled during %s that would recover',
    async (_name, held) => {
      const window = new FakeBrowserWindow()
      let land: () => void = () => undefined
      const hold = <T>(value: T): Promise<T> =>
        new Promise(resolve => {
          land = () => resolve(value)
        })
      const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
      const installCookie = vi.fn<SandboxUiInstallCookieFn>(async () => undefined)
      const onRecovered = vi.fn()
      startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, installCookie, onRecovered }))
      await runOutOfRetries()

      if (held === 'refresh') {
        refresh.mockImplementation(() => hold({ setCookie: 'clerum_sandbox_ui_session=X' }))
      } else {
        refresh.mockImplementation(mintOk)
        installCookie.mockImplementation(() => hold(undefined))
      }
      window.emit('focus')
      await flush()
      cancelSandboxUiRefresh()
      land()
      await flush()
      expect(onRecovered).not.toHaveBeenCalled()
    }
  )

  it('a top-level 401 after the retries ran out reloads and then signals the recovery once', async () => {
    const log: string[] = []
    const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
    const reload = vi.fn(() => log.push('reload'))
    const onRecovered = vi.fn(() => log.push('onRecovered'))
    const args = makeArgs({ refresh, reload, onRecovered })
    startSandboxUiRefresh(args)
    await runOutOfRetries()

    refresh.mockImplementation(mintOk)
    handleSandboxUiUnauthorizedNavigation(args.webContentsId)
    await flush()
    expect(log).toEqual(['reload', 'onRecovered'])

    await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
    expect(onRecovered).toHaveBeenCalledTimes(1)
  })

  it('a recovery listener that throws leaves the loop armed and rejects nothing', async () => {
    const unhandled: unknown[] = []
    const onUnhandled = (reason: unknown): void => {
      unhandled.push(reason)
    }
    process.on('unhandledRejection', onUnhandled)
    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    try {
      const window = new FakeBrowserWindow()
      const refresh = vi.fn<SandboxUiRefreshFn>(mintDown)
      const onRecovered = vi.fn(() => {
        throw new Error('renderer gone')
      })
      startSandboxUiRefresh(makeArgs({ parentWindow: window, refresh, onRecovered }))
      await runOutOfRetries()
      refresh.mockImplementation(mintOk)
      window.emit('focus')
      await flush()
      expect(onRecovered).toHaveBeenCalledTimes(1)
      const callsAfterRecovery = refresh.mock.calls.length

      await advance(SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
      expect(refresh).toHaveBeenCalledTimes(callsAfterRecovery + 1)
      await new Promise(resolve => setImmediate(resolve))
      expect(unhandled).toEqual([])
    } finally {
      process.off('unhandledRejection', onUnhandled)
      consoleError.mockRestore()
    }
  })
})

/**
 * Model-based check of the error/recovery signals against the real loop. Mints
 * and cookie installs are released by a scheduler, so an event can land while
 * either one is in flight. At rest, the renderer's banner (whether the last
 * signal was an error) must match the last mint that either gave up or
 * succeeded; failures that still had retries left say nothing about it.
 */
describe('recovery signal model', () => {
  type MintResult = 'gaveUp' | 'succeeded'

  type Scheduler = {
    schedule<T>(task: Promise<T>, label: string): Promise<T>
    count(): number
    waitOne(): Promise<void>
  }

  type Harness = {
    window: FakeBrowserWindow
    mintMode: 'ok' | 'fail'
    mints: number
    signals: Array<'error' | 'recovered'>
    results: MintResult[]
    cancelledAtSignal: number | null
    cancelled: boolean
  }

  type Model = Record<string, never>

  abstract class Step implements fc.AsyncCommand<Model, Harness> {
    check(): boolean {
      return true
    }
    abstract run(model: Model, harness: Harness): Promise<void>
    abstract toString(): string
  }

  class MintMode extends Step {
    constructor(readonly mode: 'ok' | 'fail') {
      super()
    }
    async run(_m: Model, h: Harness): Promise<void> {
      h.mintMode = this.mode
    }
    toString(): string {
      return `MintMode(${this.mode})`
    }
  }
  class Advance extends Step {
    constructor(readonly ms: number) {
      super()
    }
    async run(): Promise<void> {
      await advance(this.ms)
    }
    toString(): string {
      return `Advance(${this.ms})`
    }
  }
  /** Time passes with the network answering promptly: every mint lands within a second. */
  class Elapse extends Step {
    constructor(
      readonly ms: number,
      private readonly scheduler: () => Scheduler
    ) {
      super()
    }
    async run(): Promise<void> {
      for (let elapsed = 0; elapsed < this.ms; elapsed += 1_000) {
        await advance(1_000)
        await settle(this.scheduler())
      }
    }
    toString(): string {
      return `Elapse(${this.ms})`
    }
  }
  class Focus extends Step {
    async run(_m: Model, h: Harness): Promise<void> {
      h.window.emit('focus')
      await flush()
    }
    toString(): string {
      return 'Focus'
    }
  }
  class HostResumed extends Step {
    async run(): Promise<void> {
      notifySandboxUiHostResumed()
      await flush()
    }
    toString(): string {
      return 'HostResumed'
    }
  }
  class EmbedRequest extends Step {
    async run(): Promise<void> {
      void handleEmbedRefreshRequest(MODEL_SENDER).catch(() => undefined)
      await flush()
    }
    toString(): string {
      return 'EmbedRequest'
    }
  }
  class Unauthorized extends Step {
    async run(): Promise<void> {
      handleSandboxUiUnauthorizedNavigation(MODEL_SENDER)
      await flush()
    }
    toString(): string {
      return 'Unauthorized'
    }
  }
  class Cancel extends Step {
    async run(_m: Model, h: Harness): Promise<void> {
      if (!h.cancelled) h.cancelledAtSignal = h.signals.length
      h.cancelled = true
      cancelSandboxUiRefresh()
    }
    toString(): string {
      return 'Cancel'
    }
  }
  class Release extends Step {
    constructor(private readonly scheduler: () => Scheduler) {
      super()
    }
    async run(): Promise<void> {
      if (this.scheduler().count() === 0) return
      await this.scheduler().waitOne()
      await flush()
    }
    toString(): string {
      return 'Release'
    }
  }

  const MODEL_SENDER = 77

  /** `observeRecovery: false` wires a no-op `onRecovered`, as if nobody listened. */
  function startHarness(scheduler: Scheduler, observeRecovery: boolean): Harness {
    _resetSandboxUiRefreshForTests()
    const harness: Harness = {
      window: new FakeBrowserWindow(),
      // Starts failing: recoveries only exist after the loop gave up once.
      mintMode: 'fail',
      mints: 0,
      signals: [],
      results: [],
      cancelledAtSignal: null,
      cancelled: false,
    }
    let installs = 0
    startSandboxUiRefresh(
      makeArgs({
        parentWindow: harness.window,
        webContentsId: MODEL_SENDER,
        refresh: () => {
          const label = `mint#${harness.mints}`
          harness.mints += 1
          const fails = harness.mintMode === 'fail'
          return scheduler.schedule(Promise.resolve(), label).then(() => {
            if (fails) throw new Error('control-api 503')
            return { setCookie: 'clerum_sandbox_ui_session=X' }
          })
        },
        installCookie: () => {
          const label = `install#${installs}`
          installs += 1
          return scheduler.schedule(Promise.resolve(), label).then(() => {
            // A cancelled loop stops before acting on the install.
            if (!harness.cancelled) harness.results.push('succeeded')
          })
        },
        onError: () => {
          harness.signals.push('error')
          harness.results.push('gaveUp')
        },
        onRecovered: observeRecovery ? () => void harness.signals.push('recovered') : () => {},
      })
    )
    return harness
  }

  async function settle(scheduler: Scheduler): Promise<void> {
    while (scheduler.count() > 0) {
      await scheduler.waitOne()
      await flush()
    }
  }

  /** Releases scheduled tasks in the order a previous run released them. */
  function replayScheduler(order: readonly string[]): Scheduler {
    const pending = new Map<string, () => void>()
    let next = 0
    return {
      schedule: <T>(task: Promise<T>, label: string) =>
        new Promise<T>((resolve, reject) => {
          pending.set(label, () => void task.then(resolve, reject))
        }),
      count: () => pending.size,
      async waitOne() {
        const label = order[next]
        next += 1
        const release = label === undefined ? undefined : pending.get(label)
        if (!release) throw new Error(`replay diverged at release #${next - 1} (${label})`)
        pending.delete(label!)
        release()
      },
    }
  }

  function recordingScheduler(s: fc.Scheduler, order: string[]): Scheduler {
    return {
      schedule: <T>(task: Promise<T>, label: string) =>
        s.schedule(task, label).then(
          value => {
            order.push(label)
            return value
          },
          error => {
            order.push(label)
            throw error
          }
        ),
      count: () => s.count(),
      waitOne: () => s.waitOne(),
    }
  }

  it('keeps the banner in step with the last mint that gave up or succeeded', async () => {
    const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
    onTestFinished(() => consoleError.mockRestore())
    let current: Scheduler | null = null
    const currentScheduler = (): Scheduler => current!
    const mode = fc.constantFrom('ok', 'fail').map(m => new MintMode(m as 'ok' | 'fail'))
    // Weighted so that runs regularly give up (a failing streak needs several
    // minutes of mints) and then come back, which is where the signals live.
    const step = fc.oneof(
      { weight: 3, arbitrary: mode },
      {
        weight: 2,
        arbitrary: fc
          .constantFrom(1_000, 5_000, 30_000, 100_000, SANDBOX_UI_REFRESH_INTERVAL_MS + 1)
          .map(ms => new Advance(ms)),
      },
      {
        weight: 4,
        arbitrary: fc
          .constantFrom(30_000, 100_000, SANDBOX_UI_REFRESH_INTERVAL_MS + 1_000)
          .map(ms => new Elapse(ms, currentScheduler)),
      },
      { weight: 2, arbitrary: fc.constant(new Focus()) },
      { weight: 1, arbitrary: fc.constant(new HostResumed()) },
      { weight: 1, arbitrary: fc.constant(new EmbedRequest()) },
      { weight: 1, arbitrary: fc.constant(new Unauthorized()) },
      { weight: 1, arbitrary: fc.constant(new Cancel()) },
      { weight: 3, arbitrary: fc.constant(new Release(currentScheduler)) }
    )

    await fc.assert(
      fc.asyncProperty(
        fc.scheduler(),
        fc.commands([step], { maxCommands: 50, size: '+1' }),
        async (s, cmds) => {
          const order: string[] = []
          const executed: Step[] = []
          current = recordingScheduler(s, order)
          const scheduler = current
          const harness = startHarness(scheduler, true)
          await fc.asyncModelRun(
            () => ({ model: {}, real: harness }),
            [...cmds].map(cmd => {
              const step = cmd as Step
              return {
                check: () => true,
                run: async (m: Model, h: Harness) => {
                  executed.push(step)
                  await step.run(m, h)
                },
                toString: () => step.toString(),
              }
            })
          )
          await settle(scheduler)

          const banner = harness.signals.at(-1) === 'error'
          const lastDecisive = harness.results.at(-1)
          // V1 / S1: the banner shows exactly when the last decisive mint gave up.
          expect(banner).toBe(lastDecisive === 'gaveUp')
          // C1 (per loop): never more recoveries than reported failures.
          const errors = harness.signals.filter(signal => signal === 'error').length
          const recoveries = harness.signals.filter(signal => signal === 'recovered').length
          expect(recoveries).toBeLessThanOrEqual(errors)
          // S2: nothing is signalled once the loop was cancelled.
          if (harness.cancelledAtSignal !== null) {
            expect(harness.signals.length).toBe(harness.cancelledAtSignal)
          }

          // C1: the signal itself never changes how many mints the loop makes.
          current = replayScheduler(order)
          const replay = startHarness(current, false)
          for (const step of executed) await step.run({}, replay)
          await settle(current)
          expect(replay.mints).toBe(harness.mints)
        }
      ),
      { numRuns: 300 }
    )
  })
})