Skip to content

armgit / fx0

Public Imported from GitHub
0

A lightweight design agent built with libfx. Describe a page and an agent writes it as one HTML file.

TypeScript

Commit turn bookkeeping with its transcript lines and retry history writes

Pranit00f02da12ead
diff --git a/README.md b/README.md
index 421a14c..96273e6 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ fx0 keeps the transcript, designs and settings in Redis. Here is one message fro
**3. libfx runs the turn.** One worker holds a session at a time, and libfx fences out any other. The prompt request runs the turn first; it and each delivery to `/api/libfx` can run for 800 seconds, their `maxDuration`. libfx stops a turn 30 seconds before that limit, keeping what it has done so far, and a delivery to `/api/libfx` continues it, so a long turn spans several invocations.
-**4. Copy the stream into the transcript.** Any route following the chat, the prompt's own response or an open tab, copies libfx's stream into the transcript. A cursor check makes each line land once, however many follow. When a turn ends, the copy records its history and its design version.
+**4. Copy the stream into the transcript.** Any route following the chat, the prompt's own response or an open tab, copies libfx's stream into the transcript. A cursor check makes each line land once, however many follow. The same Redis script marks a turn running when its start lands, and when its end lands it leaves the turn's history to write, with the design the turn left. The copy then writes that history and design version to Neon, and a write that fails waits in Redis for the next copy to retry.
**5. Pick up where it stopped.** On Vercel, libfx can't tell a crashed function from a slow one, so the request running a turn holds its session until its time limit. The prompt's backstop comes due 5 seconds after that limit: if the request died, the backstop takes the session over and the turn continues where its records end. In a delivery to `/api/libfx`, a process that exits has its delivery run again about a second later, and a function that stops without exiting holds its message until the queue's hold runs out, up to five minutes. The model is told the turn was interrupted, so it can check what happened before going on. A function that only stalled and wakes after that stops at its next write, so two never write at once.
@@ -172,6 +172,7 @@ npm run test:journal:integration # needs KV_REST_API_URL/TOKEN, writes under it
npm run test:trial:integration # needs local redis-server/redis-cli; no remote services
npm run test:artifact:browser # needs local Chrome (or CHROME_PATH); checks design CSP enforcement
npm run test:budget:integration # needs local redis-server/redis-cli; no remote services
+npm run test:turns:integration # needs local redis-server/redis-cli; no remote services
```
The auth and route tests mock Vercel, AI Gateway and both databases, so check a real sign-in and real billing on a deployment.
diff --git a/lib/history.ts b/lib/history.ts
index 5c750e6..6beeea5 100644
--- a/lib/history.ts
+++ b/lib/history.ts
@@ -72,9 +72,10 @@ export function createHistoryStore(query: AccountQuery) {
}
}
-// History is written after Redis commits the turn, so a failed write has no later
-// chance. The inserts are idempotent (ON CONFLICT DO NOTHING), so retrying is safe;
-// each attempt is bounded so a stalled database cannot hold the worker.
+// History is written from the work a turn's end leaves in Redis, which stays until
+// the write succeeds, so a later follower retries a failed one. The inserts are
+// idempotent (ON CONFLICT DO NOTHING), so retrying is safe; each attempt is bounded
+// so a stalled database cannot hold the worker.
const WRITE_ATTEMPTS = 3
const WRITE_TIMEOUT_MS = 5_000
diff --git a/lib/journal.ts b/lib/journal.ts
index 7b350de..85ff60a 100644
--- a/lib/journal.ts
+++ b/lib/journal.ts
@@ -432,6 +432,8 @@ const SESSION_PARTS = [
'question-seq',
'state',
'transcript',
+ 'turn-designs',
+ 'turn-work',
] as const
const CLAIM_CLEANUP = `
@@ -546,12 +548,24 @@ local revision = tonumber(redis.call('HGET', KEYS[2], 'artifactRevision') or '0'
if redis.call('GET', KEYS[3]) == ARGV[1] then
return cjson.encode({status = 'unchanged', revision = revision})
end
+-- The design another turn published last is that turn's history version. A turn
+-- whose end is still to be projected would find it replaced, so keep it now.
+local previous = redis.call('HGET', KEYS[2], 'artifactTurn')
+if previous and previous ~= '' and previous ~= ARGV[3] then
+ local html = redis.call('GET', KEYS[3])
+ if html then
+ redis.call('HSET', KEYS[8], previous, cjson.encode({revision = revision, html = html}))
+ redis.call('EXPIRE', KEYS[8], ARGV[4])
+ end
+end
redis.call('SET', KEYS[3], ARGV[1])
revision = redis.call('HINCRBY', KEYS[2], 'artifactRevision', 1)
local event = {type = 'artifact', artifactRevision = revision, at = tonumber(ARGV[2])}
if ARGV[3] ~= '' then
event.turnId = tonumber(ARGV[3])
redis.call('HSET', KEYS[2], 'artifactTurn', ARGV[3])
+else
+ redis.call('HDEL', KEYS[2], 'artifactTurn')
end
local seq = redis.call('INCR', KEYS[4])
event.seq = seq
@@ -581,10 +595,11 @@ export async function publishDesignArtifact(
key(sessionId, 'events'),
key(sessionId, 'draft'),
key(sessionId, 'draft-chunks'),
+ key(sessionId, 'turn-designs'),
],
// The stored design is the normalized one validation passed, never the raw input.
// The turn lets the transcript group a turn's revisions under its activity.
- [validation.html, String(Date.now()), turnId == null ? '' : String(turnId)],
+ [validation.html, String(Date.now()), turnId == null ? '' : String(turnId), String(TURN_WORK_SECONDS)],
)
const result = parseJson<{
status: 'published' | 'unchanged' | 'fenced'
@@ -602,29 +617,15 @@ export async function publishDesignArtifact(
}
}
-// The projector marks a turn active at its turn_start and clears it at its
-// turn_end. The TTL bounds a turn whose end the projector never sees.
+// projectEvents marks a turn active at its turn_start and clears it at its end,
+// in the same script as those lines. The TTL bounds a turn whose end is never seen.
const ACTIVE_TURN_MS = 30 * 60_000
-export async function setActiveTurn(sessionId: string, turn: ActiveTurn): Promise<void> {
- await redis().set(key(sessionId, 'active'), JSON.stringify(turn), { px: ACTIVE_TURN_MS })
-}
-
export async function getActiveTurn(sessionId: string): Promise<ActiveTurn | null> {
const value = await redis().get<unknown>(key(sessionId, 'active'))
return value == null ? null : parseJson<ActiveTurn>(value)
}
-// Clears only the turn that ended, never a later one already marked.
-const CLEAR_ACTIVE_TURN = `
-local value = redis.call('GET', KEYS[1])
-if value and cjson.decode(value).id == tonumber(ARGV[1]) then redis.call('DEL', KEYS[1]) end
-return 1
-`
-export async function clearActiveTurn(sessionId: string, turnId: number): Promise<void> {
- await redis().eval<unknown[], number>(CLEAR_ACTIVE_TURN, [key(sessionId, 'active')], [String(turnId)])
-}
-
export interface ActiveQuestion {
turnId: number
qid: number
@@ -715,16 +716,44 @@ export async function projectedCursor(sessionId: string): Promise<number> {
return Number((await redis().get<number>(key(sessionId, 'projected'))) ?? 0)
}
+// A turn's running record, as its start is projected: its startSeq is the seq the
+// turn_start line gets in the same script.
+export type TurnStart = Omit<ActiveTurn, 'startSeq'>
+
+// What a finished turn still owes its history, kept in Redis until it is written:
+// its end, the running record that end found (absent if the start was never
+// recorded), and the design the turn left. Work for a design alone comes from a
+// publish that replaced it before the turn's end was projected.
+export interface TurnWork {
+ turnId: number
+ type?: 'done' | 'error'
+ stopReason?: string
+ usage?: { inputTokens?: number; outputTokens?: number }
+ at?: number
+ turn?: ActiveTurn
+ design?: { revision: number; html: string }
+}
+
+// Work left by a turn whose history cannot be written is retried by later
+// followers of the chat, for this long.
+const TURN_WORK_SECONDS = 7 * 24 * 60 * 60
+
// Appends the items copied from libfx's stream between two cursors, only if no
// other projector moved the cursor first: each libfx line reaches the transcript
-// once however many projectors follow the chat.
+// once however many projectors follow the chat. The turn bookkeeping those lines
+// imply commits with them: a turn_start marks the turn running, and a turn's end
+// clears it and leaves the turn's history work, with the design it left if it
+// published last. So a crash, or a projector on another instance, can neither skip
+// that bookkeeping nor apply it out of order.
const PROJECT_EVENTS = INIT_STATE + `
if redis.call('EXISTS', KEYS[4]) == 1 then return false end
if tonumber(redis.call('GET', KEYS[5]) or '0') ~= tonumber(ARGV[1]) then return false end
local items = cjson.decode(ARGV[3])
+local starts = cjson.decode(ARGV[6])
local out = {}
for i, item in ipairs(items) do
- if item.type == 'done' or item.type == 'error' then
+ local ended = item.type == 'done' or item.type == 'error'
+ if ended then
local turn = item.turnId or item.id or 0
local closed = tonumber(redis.call('HGET', KEYS[3], 'draftClosedTurn') or '0')
redis.call('HSET', KEYS[3], 'draftClosedTurn', math.max(closed, turn))
@@ -737,6 +766,32 @@ for i, item in ipairs(items) do
local encoded = cjson.encode(item)
redis.call('RPUSH', KEYS[2], encoded)
out[i] = encoded
+ if item.type == 'turn_start' and item.id and starts[tostring(item.id)] then
+ local start = starts[tostring(item.id)]
+ start.startSeq = item.seq
+ redis.call('SET', KEYS[8], cjson.encode(start), 'PX', ARGV[7])
+ elseif ended and item.id then
+ local work = {turnId = item.id, type = item.type, stopReason = item.stopReason, usage = item.usage, at = item.at}
+ local active = redis.call('GET', KEYS[8])
+ if active then
+ local running = cjson.decode(active)
+ if running.id == item.id then
+ work.turn = running
+ redis.call('DEL', KEYS[8])
+ end
+ end
+ redis.call('HSET', KEYS[9], tostring(item.id), cjson.encode(work))
+ redis.call('EXPIRE', KEYS[9], ARGV[8])
+ if tonumber(redis.call('HGET', KEYS[3], 'artifactTurn') or '0') == item.id then
+ local html = redis.call('GET', KEYS[11])
+ if html then
+ local revision = tonumber(redis.call('HGET', KEYS[3], 'artifactRevision') or '0')
+ redis.call('HSET', KEYS[10], tostring(item.id), cjson.encode({revision = revision, html = html}))
+ redis.call('EXPIRE', KEYS[10], ARGV[8])
+ end
+ redis.call('HDEL', KEYS[3], 'artifactTurn')
+ end
+ end
end
redis.call('SET', KEYS[5], ARGV[2])
init_state(KEYS[3], ARGV[4], ARGV[5])
@@ -747,14 +802,46 @@ export async function projectEvents(
fromCursor: number,
toCursor: number,
entries: (Omit<StreamItem, 'seq'>)[],
+ starts: Record<number, TurnStart> = {},
): Promise<StreamItem[] | null> {
const raw = await redis().eval<unknown[], unknown>(PROJECT_EVENTS,
[key(sessionId, 'event-seq'), key(sessionId, 'events'), key(sessionId, 'state'), key(sessionId, 'deleting'),
- key(sessionId, 'projected'), key(sessionId, 'draft'), key(sessionId, 'draft-chunks')],
- [String(fromCursor), String(toCursor), JSON.stringify(entries), sessionId, String(Date.now())])
+ key(sessionId, 'projected'), key(sessionId, 'draft'), key(sessionId, 'draft-chunks'),
+ key(sessionId, 'active'), key(sessionId, 'turn-work'), key(sessionId, 'turn-designs'), key(sessionId, 'design-artifact')],
+ [String(fromCursor), String(toCursor), JSON.stringify(entries), sessionId, String(Date.now()),
+ JSON.stringify(starts), String(ACTIVE_TURN_MS), String(TURN_WORK_SECONDS)])
if (raw === null || raw === false) return null
// An empty Lua table comes back as an empty array or an empty object.
const items = Array.isArray(raw) ? raw.map((item) => parseJson<StreamItem>(item)) : []
for (const item of items) announce(sessionId, item)
return items
}
+
+// Every finished turn's history work still waiting, oldest turn first.
+export async function readTurnWork(sessionId: string): Promise<TurnWork[]> {
+ const [work, designs] = await Promise.all([
+ redis().hgetall<Record<string, unknown>>(key(sessionId, 'turn-work')),
+ redis().hgetall<Record<string, unknown>>(key(sessionId, 'turn-designs')),
+ ])
+ const byTurn = new Map<number, TurnWork>()
+ for (const value of Object.values(work ?? {})) {
+ const item = parseJson<TurnWork>(value)
+ byTurn.set(item.turnId, item)
+ }
+ for (const [turn, value] of Object.entries(designs ?? {})) {
+ const turnId = Number(turn)
+ byTurn.set(turnId, { ...(byTurn.get(turnId) ?? { turnId }), design: parseJson<{ revision: number; html: string }>(value) })
+ }
+ return [...byTurn.values()].sort((a, b) => a.turnId - b.turnId)
+}
+
+// A turn's history is written: its work and its design go together.
+const FINISH_TURN_WORK = `
+redis.call('HDEL', KEYS[1], ARGV[1])
+redis.call('HDEL', KEYS[2], ARGV[1])
+return 1
+`
+export async function finishTurnWork(sessionId: string, turnId: number): Promise<void> {
+ await redis().eval<unknown[], number>(FINISH_TURN_WORK,
+ [key(sessionId, 'turn-work'), key(sessionId, 'turn-designs')], [String(turnId)])
+}
diff --git a/lib/projector.ts b/lib/projector.ts
index bb9fed5..1472c80 100644
--- a/lib/projector.ts
+++ b/lib/projector.ts
@@ -1,19 +1,18 @@
import type { FxAgent, FxStreamLine } from 'libfx'
import { tail } from './tools'
import {
- clearActiveTurn,
+ finishTurnWork,
getActiveTurn,
projectEvents,
projectedCursor,
- readDesignArtifact,
readEvents,
readSession,
- readSessionView,
- setActiveTurn,
- type ActiveTurn,
+ readTurnWork,
type TurnSettings,
+ type TurnStart,
+ type TurnWork,
} from './journal'
-import { recordGeneration, recordTurn } from './history'
+import { recordGeneration, recordTurn, type TurnRecord } from './history'
import { readExecutionFunding } from './turn-credential'
import { effectiveEffort, effectiveFast, effectiveModel, INSTANCE, log, type SessionState } from './session'
import type { StreamItem } from './events'
@@ -126,57 +125,77 @@ function turnSettings(host: SessionState, sponsored: boolean): TurnSettings {
: { model: effectiveModel(host), effort: effectiveEffort(host), fast: effectiveFast(host) }
}
-async function startTurn(host: SessionState, id: number, input: string | undefined, startSeq: number): Promise<void> {
+// A turn's running record, read before its turn_start commits so that the commit
+// can mark it running in the same script (see projectEvents). If these reads
+// fail, nothing commits and the next follower projects the lines again.
+async function readTurnStart(host: SessionState, id: number, input: string | undefined): Promise<TurnStart> {
const [funding] = await Promise.all([readExecutionFunding(host.id, id), readSettings(host)])
const credential = funding.ok ? funding.credential : null
const sponsored = !!credential && 'kind' in credential
- const turn: ActiveTurn = {
+ return {
id,
startedAt: Date.now(),
billingTeamId: credential?.teamId ?? null,
...(sponsored ? { sponsored: true as const } : {}),
prompt: promptText(input),
- startSeq,
settings: turnSettings(host, sponsored),
}
- await setActiveTurn(host.id, turn)
}
-async function endTurn(host: SessionState, terminal: StreamItem): Promise<void> {
- const id = terminal.id!
- const active = await getActiveTurn(host.id)
- await clearActiveTurn(host.id, id)
- const turn = active?.id === id ? active : null
- const stored = await readSettings(host)
+// A finished turn's history row, from the work its end left and the transcript
+// since it started. `host` carries the chat's settings, for a turn whose start
+// was never recorded.
+export function turnHistoryRow(host: SessionState, work: TurnWork, events: StreamItem[]): TurnRecord {
+ const turn = work.turn ?? null
const settings = turn?.settings ?? turnSettings(host, !!turn?.sponsored)
- const events = turn ? await readEvents(host.id, Math.max(0, turn.startSeq - 1)) : []
- const response = events.filter((item) => item.type === 'text' && item.turnId === id).map((item) => item.delta ?? '').join('')
- // One history version per turn: the design as the turn left it, if it
- // published one and no later turn has published since.
- if (stored?.artifactTurn === id) {
- const [view, html] = await Promise.all([readSessionView(host.id), readDesignArtifact(host.id)])
- if (html != null) {
- await recordGeneration(host.id, view.artifactRevision, id, html).catch(() =>
- log(host.id, 'history:generation:error', { revision: view.artifactRevision }))
- }
- }
- await recordTurn({
+ return {
sessionId: host.id,
- turnId: id,
+ turnId: work.turnId,
prompt: turn?.prompt ?? '',
- response,
+ response: events.filter((item) => item.type === 'text' && item.turnId === work.turnId).map((item) => item.delta ?? '').join(''),
model: settings.model,
effort: settings.effort,
fast: settings.fast,
- status: terminal.type === 'done' ? 'done' : 'error',
- stopReason: terminal.type === 'done' ? terminal.stopReason ?? null : null,
+ status: work.type === 'done' ? 'done' : 'error',
+ stopReason: work.type === 'done' ? work.stopReason ?? null : null,
billingTeamId: turn?.billingTeamId ?? null,
- inputTokens: terminal.usage?.inputTokens ?? null,
- outputTokens: terminal.usage?.outputTokens ?? null,
- startedAt: turn?.startedAt ?? Date.now(),
- }).catch(() => log(host.id, 'history:turn:error', { id }))
+ inputTokens: work.usage?.inputTokens ?? null,
+ outputTokens: work.usage?.outputTokens ?? null,
+ startedAt: turn?.startedAt ?? work.at ?? 0,
+ }
}
+// Writes the history of every finished turn still waiting: one row per turn and
+// one design version, the design as the turn left it. Both inserts are idempotent,
+// and a turn's work is removed only once they succeed, so a crash, a database
+// error or a second follower can neither lose nor duplicate a row. A failure
+// leaves the rest for the next follower of the chat.
+export async function finishTurns(host: SessionState): Promise<void> {
+ let work: TurnWork[]
+ try {
+ work = await readTurnWork(host.id)
+ if (work.length) await readSettings(host)
+ } catch (error) {
+ log(host.id, 'history:read:error', { error: errorText(error) })
+ return
+ }
+ for (const item of work) {
+ try {
+ if (item.design) await recordGeneration(host.id, item.design.revision, item.turnId, item.design.html)
+ if (item.type) {
+ const events = item.turn ? await readEvents(host.id, Math.max(0, item.turn.startSeq - 1)) : []
+ await recordTurn(turnHistoryRow(host, item, events))
+ }
+ await finishTurnWork(host.id, item.turnId)
+ } catch (error) {
+ log(host.id, 'history:turn:error', { id: item.turnId, error: errorText(error) })
+ return
+ }
+ }
+}
+
+const errorText = (error: unknown) => String((error as Error)?.message ?? error).slice(0, 200)
+
const BATCH_MS = 40
export interface FollowOptions {
@@ -190,6 +209,8 @@ export interface FollowOptions {
// chat stays idle for idleMs, when the signal aborts, or when another projector
// moved the cursor first (that one carries on).
export async function project(agent: FxAgent, host: SessionState, libfxSessionId: string, options: FollowOptions): Promise<void> {
+ // History an earlier follower could not finish, before following on.
+ await finishTurns(host)
let cursor = await projectedCursor(host.id)
const reader = agent.session(libfxSessionId).stream(cursor).getReader()
const decoder = new TextDecoder()
@@ -215,19 +236,23 @@ export async function project(agent: FxAgent, host: SessionState, libfxSessionId
if (!batch.length) return true
const toCursor = batch[batch.length - 1].line.cursor ?? cursor
const entries = mergeDeltas(batch.flatMap((part) => part.entries))
- const items = await projectEvents(host.id, cursor, toCursor, entries)
+ const starts: Record<number, TurnStart> = {}
+ for (const { line } of batch) {
+ const id = line.type === 'turn_start' ? messageIdOf(line) : null
+ if (id != null && line.type === 'turn_start') starts[id] = await readTurnStart(host, id, line.input)
+ }
+ const items = await projectEvents(host.id, cursor, toCursor, entries, starts)
if (!items) return false
cursor = toCursor
+ let ended = false
for (const item of items) {
- if (item.type === 'turn_start') {
- running = true
- const line = batch.find((part) => part.line.type === 'turn_start' && messageIdOf(part.line) === item.id)?.line
- await startTurn(host, item.id!, line?.type === 'turn_start' ? line.input : undefined, item.seq ?? 0)
- } else if (item.type === 'done' || item.type === 'error') {
+ if (item.type === 'turn_start') running = true
+ else if (item.type === 'done' || item.type === 'error') {
running = false
- await endTurn(host, item)
+ ended = true
}
}
+ if (ended) await finishTurns(host)
armIdle()
return true
}
@@ -237,7 +262,11 @@ export async function project(agent: FxAgent, host: SessionState, libfxSessionId
const scheduleCommit = () => {
flushTimer ??= setTimeout(() => {
flushTimer = null
- committing = committing.then((ok) => (ok ? commit() : false)).then((ok) => {
+ committing = committing.then((ok) => (ok ? commit() : false)).catch((error) => {
+ // Nothing committed: the next follower projects these lines again.
+ log(host.id, 'projector:error', { error: errorText(error) })
+ return false
+ }).then((ok) => {
if (!ok) stop()
return ok
})
diff --git a/package.json b/package.json
index c70d9d0..d922ba6 100644
--- a/package.json
+++ b/package.json
@@ -16,6 +16,7 @@
"db:migrate": "node --env-file-if-exists=.env.local scripts/migrate-database.mjs",
"test:artifact:browser": "node --experimental-strip-types --test tests/design-artifact.browser.mjs",
"test:budget:integration": "node --experimental-strip-types --test tests/turn-budget.integration.mjs",
+ "test:turns:integration": "node --experimental-strip-types --test tests/turn-work.integration.mjs",
"test:trial:integration": "node --experimental-strip-types --test tests/trial.integration.mjs",
"test:credential:integration": "node --experimental-strip-types --test tests/turn-credential.integration.mjs"
},
diff --git a/tests/fixtures/redis-rest.mjs b/tests/fixtures/redis-rest.mjs
new file mode 100644
index 0000000..2d46e40
--- /dev/null
+++ b/tests/fixtures/redis-rest.mjs
@@ -0,0 +1,84 @@
+import { once } from 'node:events'
+import { createServer } from 'node:http'
+import { createConnection } from 'node:net'
+
+// A stand-in for Upstash's REST API in front of a local redis-server, so a test can
+// run the real journal code, its Lua included, against real Redis. It answers the
+// way @upstash/redis expects: `{ result }` per command, base64 strings when asked.
+export async function redisRest(socketPath) {
+ const connection = createConnection(socketPath)
+ await once(connection, 'connect')
+ let buffer = Buffer.alloc(0)
+ const waiting = []
+ connection.on('data', (chunk) => {
+ buffer = Buffer.concat([buffer, chunk])
+ for (let parsed = parse(buffer, 0); parsed && waiting.length; parsed = parse(buffer, 0)) {
+ buffer = buffer.subarray(parsed.next)
+ waiting.shift()(parsed.value)
+ }
+ })
+ const send = (args) => new Promise((resolve) => {
+ waiting.push(resolve)
+ const parts = args.map((arg) => Buffer.from(String(arg)))
+ connection.write(Buffer.concat([
+ Buffer.from(`*${parts.length}\r\n`),
+ ...parts.flatMap((part) => [Buffer.from(`$${part.length}\r\n`), part, Buffer.from('\r\n')]),
+ ]))
+ })
+
+ const server = createServer(async (request, response) => {
+ let body = ''
+ for await (const chunk of request) body += chunk
+ const base64 = request.headers['upstash-encoding'] === 'base64'
+ const reply = (value) => value?.error ? { error: value.error } : { result: base64 ? encode(value) : value }
+ const commands = JSON.parse(body)
+ const pipelined = request.url.endsWith('/pipeline') || request.url.endsWith('/multi-exec')
+ const results = []
+ for (const command of pipelined ? commands : [commands]) results.push(reply(await send(command)))
+ response.setHeader('content-type', 'application/json')
+ response.end(JSON.stringify(pipelined ? results : results[0]))
+ })
+ server.listen(0, '127.0.0.1')
+ await once(server, 'listening')
+ return {
+ url: `http://127.0.0.1:${server.address().port}`,
+ async close() {
+ connection.end()
+ await new Promise((resolve) => server.close(resolve))
+ },
+ }
+}
+
+const encode = (value) => typeof value === 'string' ? Buffer.from(value).toString('base64')
+ : Array.isArray(value) ? value.map(encode) : value
+
+// One RESP2 reply from `buffer` at `offset`, or null until it has all arrived.
+function parse(buffer, offset) {
+ const end = buffer.indexOf('\r\n', offset)
+ if (end < 0) return null
+ const kind = String.fromCharCode(buffer[offset])
+ const line = buffer.toString('utf8', offset + 1, end)
+ if (kind === '+') return { value: line, next: end + 2 }
+ if (kind === '-') return { value: { error: line }, next: end + 2 }
+ if (kind === ':') return { value: Number(line), next: end + 2 }
+ if (kind === '$') {
+ const length = Number(line)
+ if (length < 0) return { value: null, next: end + 2 }
+ if (buffer.length < end + 2 + length + 2) return null
+ return { value: buffer.toString('utf8', end + 2, end + 2 + length), next: end + 2 + length + 2 }
+ }
+ if (kind === '*') {
+ const count = Number(line)
+ if (count < 0) return { value: null, next: end + 2 }
+ const values = []
+ let next = end + 2
+ for (let index = 0; index < count; index++) {
+ const item = parse(buffer, next)
+ if (!item) return null
+ values.push(item.value)
+ next = item.next
+ }
+ return { value: values, next }
+ }
+ throw new Error(`unexpected RESP reply ${JSON.stringify(kind)}`)
+}
diff --git a/tests/journal.integration.mjs b/tests/journal.integration.mjs
index 9026066..ca7de87 100644
--- a/tests/journal.integration.mjs
+++ b/tests/journal.integration.mjs
@@ -219,11 +219,11 @@ test('journal Redis integration uses isolated durable session state', async (t)
await t.test('a turn end clears only its own turn', async () => {
const sessionId = 'active-turn'
- const turn = { id: 2, startedAt: 1, billingTeamId: 'team_a', prompt: 'p', startSeq: 3 }
- await journal.setActiveTurn(sessionId, turn)
- await journal.clearActiveTurn(sessionId, 1)
- assert.deepEqual(await journal.getActiveTurn(sessionId), turn)
- await journal.clearActiveTurn(sessionId, 2)
+ const start = { id: 2, startedAt: 1, billingTeamId: 'team_a', prompt: 'p' }
+ const [item] = await journal.projectEvents(sessionId, 0, 1, [{ type: 'turn_start', id: 2, turnId: 2, at: 1 }], { 2: start })
+ await journal.projectEvents(sessionId, 1, 2, [{ type: 'done', id: 1, turnId: 1, at: 2 }])
+ assert.deepEqual(await journal.getActiveTurn(sessionId), { ...start, startSeq: item.seq })
+ await journal.projectEvents(sessionId, 2, 3, [{ type: 'done', id: 2, turnId: 2, at: 3 }])
assert.equal(await journal.getActiveTurn(sessionId), null)
})
@@ -232,9 +232,9 @@ test('journal Redis integration uses isolated durable session state', async (t)
await journal.updateSession(id, { preferredEffort: null })
await journal.appendEvent(id, { type: 'status', text: 'before deletion' })
await journal.bindLibfxSession(id, 'wrun_deleted')
- await journal.setActiveTurn(id, { id: 1, startedAt: 1, billingTeamId: null, prompt: '', startSeq: 1 })
+ await journal.projectEvents(id, 0, 1, [{ type: 'turn_start', id: 1, turnId: 1, at: 1 }], { 1: { id: 1, startedAt: 1, billingTeamId: null, prompt: '' } })
assert.equal(await journal.claimSessionCleanup(id, randomUUID()), 'busy')
- await journal.clearActiveTurn(id, 1)
+ await journal.projectEvents(id, 1, 2, [{ type: 'done', id: 1, turnId: 1, at: 2 }])
const token = randomUUID()
assert.equal(await journal.claimSessionCleanup(id, token), 'claimed')
await journal.releaseSessionCleanup(id, token)
diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs
index b37586b..94b8d43 100644
--- a/tests/runtime.test.mjs
+++ b/tests/runtime.test.mjs
@@ -7,10 +7,10 @@ import test from 'node:test'
// own integration test; the agent's fetch has tests/libfx-agent.test.mjs.
const journalNames = [
'answerActiveQuestion', 'appendEvent', 'bindLibfxSession', 'chatOfLibfxSession', 'claimSessionCleanup',
- 'clearActiveQuestion', 'clearActiveTurn', 'currentSeq', 'emitLive', 'finishSessionCleanup', 'getActiveQuestion',
+ 'clearActiveQuestion', 'currentSeq', 'emitLive', 'finishSessionCleanup', 'finishTurnWork', 'getActiveQuestion',
'getActiveTurn', 'nextQuestionId', 'projectEvents', 'projectedCursor', 'publishDesignArtifact', 'readDesignArtifact',
- 'readDesignMd', 'readEvents', 'readSession', 'readSessionView', 'releaseSessionCleanup', 'renewSessionCleanup',
- 'setActiveQuestion', 'setActiveTurn', 'submitPrompt', 'takeAnswer', 'updateSession',
+ 'readDesignMd', 'readEvents', 'readSession', 'readSessionView', 'readTurnWork', 'releaseSessionCleanup', 'renewSessionCleanup',
+ 'setActiveQuestion', 'submitPrompt', 'takeAnswer', 'updateSession',
]
const dataUrl = (source) => `data:text/javascript,${encodeURIComponent(source)}`
const forward = (root, names) => names.map((name) =>
@@ -80,6 +80,9 @@ function fixture(t, stored = { preferredModel: null, preferredEffort: null, pref
// `${chatId}:${turnId}` -> who funded the turn; sign-out keeps these.
const fundings = new Map()
let active = null
+ // Each finished turn's history work, and the design it left, until written.
+ const work = new Map()
+ const designs = new Map()
let deleting = false
let question = null
let answer = null
@@ -122,8 +125,10 @@ function fixture(t, stored = { preferredModel: null, preferredEffort: null, pref
appendEvent: async (_id, entry) => { const item = { ...entry, seq: events.length + 1 }; events.push(item); return item },
publishDesignArtifact: async (...args) => { calls.push(['publish', ...args]); return { ok: true, revision: 2, changed: true } },
getActiveTurn: async () => active,
- setActiveTurn: async (_id, turn) => { active = turn },
- clearActiveTurn: async (_id, id) => { if (active?.id === id) active = null },
+ readTurnWork: async () => [...work.values()].map((item) => ({ ...item, ...(designs.has(item.turnId) ? { design: designs.get(item.turnId) } : {}) }))
+ .concat([...designs].filter(([turnId]) => !work.has(turnId)).map(([turnId, design]) => ({ turnId, design })))
+ .sort((a, b) => a.turnId - b.turnId),
+ finishTurnWork: async (_id, turnId) => { work.delete(turnId); designs.delete(turnId) },
nextQuestionId: async () => 7,
setActiveQuestion: async (_id, value) => { question = value },
getActiveQuestion: async () => question,
@@ -141,11 +146,26 @@ function fixture(t, stored = { preferredModel: null, preferredEffort: null, pref
},
clearActiveQuestion: async (_id, qid) => { if (question?.qid !== qid) return false; question = null; return true },
projectedCursor: async () => projected,
- // The same cursor check as the Lua script.
- projectEvents: async (_id, from, to, entries) => {
+ // The same cursor check and turn bookkeeping as the Lua script; its own
+ // behavior against real Redis is tests/turn-work.integration.mjs.
+ projectEvents: async (_id, from, to, entries, starts = {}) => {
if (deleting || projected !== from) return null
projected = to
- return entries.map((entry) => { const item = { ...entry, seq: events.length + 1 }; events.push(item); return item })
+ return entries.map((entry) => {
+ const item = { ...entry, seq: events.length + 1 }
+ events.push(item)
+ if (item.type === 'turn_start' && starts[item.id]) active = { ...starts[item.id], startSeq: item.seq }
+ else if ((item.type === 'done' || item.type === 'error') && item.id) {
+ const turn = active?.id === item.id ? active : undefined
+ if (turn) active = null
+ work.set(item.id, { turnId: item.id, type: item.type, stopReason: item.stopReason, usage: item.usage, at: item.at, ...(turn ? { turn } : {}) })
+ if (state.stored?.artifactTurn === item.id) {
+ designs.set(item.id, { revision: artifactRevision, html: '<!doctype html><html><head></head><body>old</body></html>' })
+ state.stored = { ...state.stored, artifactTurn: null }
+ }
+ }
+ return item
+ })
},
readEvents: async (_id, after = 0) => events.filter((item) => item.seq > after),
updateSession: async (_id, fields) => {
@@ -543,13 +563,13 @@ test('a settings change while a turn runs is accepted and says it applies from t
test('a turn\'s history and subagents keep the settings it started with after a mid-turn change', async (t) => {
const f = fixture(t, { libfxSessionId: 'wrun_paid', preferredModel: 'anthropic/start-model', preferredEffort: 'high', preferredFast: false })
const host = boundHost('paid-history')
- let started = null
- const setActiveTurn = f.journal.setActiveTurn
- f.journal.setActiveTurn = async (id, turn) => { started = turn; return setActiveTurn(id, turn) }
const log = f.logOf('wrun_paid')
const following = projector.project(runtime.agent(), host, 'wrun_paid', { idleMs: 60 })
log.push({ type: 'turn_start', messageId: 'm1', sessionId: 'wrun_paid', input: '[Ada] Go' })
- await until(() => started, 'turn start')
+ // The turn is marked running in the same commit that projects its turn_start.
+ await until(() => f.events.some((item) => item.type === 'turn_start'), 'turn start')
+ const started = await f.journal.getActiveTurn('paid-history')
+ assert.equal(started.startSeq, f.events.find((item) => item.type === 'turn_start').seq)
assert.deepEqual(started.settings, { model: 'anthropic/start-model', effort: 'high', fast: false })
assert.deepEqual(await runtime.setSessionGeneration(host, { model: 'anthropic/next-model', effort: 'low' }), { ok: true })
@@ -598,6 +618,39 @@ test('the projector copies a turn once however many follow it, then records its
assert.deepEqual(f.calls.filter((call) => call[0] === 'generation').map((call) => call.slice(1, 4)), [['project-chat', 3, 4]])
})
+test('a turn\'s history waits when the database fails, and a later follower writes it once', async (t) => {
+ const f = fixture(t, { libfxSessionId: 'wrun_retry', preferredModel: null, preferredEffort: null, preferredFast: null })
+ const host = boundHost('retry-chat')
+ f.credentials.set('retry-chat', credential())
+ let failing = true
+ const recordTurn = f.history.recordTurn
+ f.history.recordTurn = async (turn) => { if (failing) throw new Error('database unavailable'); return recordTurn(turn) }
+ const log = f.logOf('wrun_retry')
+ const following = projector.project(runtime.agent(), host, 'wrun_retry', { idleMs: 60 })
+ log.push({ type: 'turn_start', messageId: 'm2', sessionId: 'wrun_retry', input: '[Ada] Keep this' })
+ log.push({ type: 'text_delta', messageId: 'm2', delta: 'Kept.' })
+ f.setArtifactRevision(5)
+ f.stored = { ...f.stored, artifactTurn: 2 }
+ log.push({ type: 'turn_end', messageId: 'm2', stopReason: 'end_turn' })
+ await following
+ // The transcript and the turn's end committed; its history did not, so its work stays.
+ assert.equal(await f.journal.getActiveTurn('retry-chat'), null)
+ const waiting = await f.journal.readTurnWork('retry-chat')
+ assert.deepEqual(waiting.map((item) => [item.turnId, item.turn?.prompt, item.design?.revision]), [[2, 'Keep this', 5]])
+ assert.deepEqual(f.calls.filter((call) => call[0] === 'history'), [])
+
+ failing = false
+ await projector.finishTurns(host)
+ await projector.finishTurns(host)
+ const history = f.calls.filter((call) => call[0] === 'history')
+ assert.equal(history.length, 1)
+ assert.equal(history[0][1].prompt, 'Keep this')
+ assert.equal(history[0][1].response, 'Kept.')
+ // The design is written again with its retry; the insert ignores a repeat.
+ assert.ok(f.calls.filter((call) => call[0] === 'generation').every((call) => call[1] === 'retry-chat' && call[2] === 5 && call[3] === 2))
+ assert.deepEqual(await f.journal.readTurnWork('retry-chat'), [])
+})
+
test('sponsored history keeps the actual free model even after later settings change', async (t) => {
const f = fixture(t, { libfxSessionId: 'wrun_trial', preferredModel: 'paid/other-model', preferredEffort: 'high', preferredFast: false })
const host = boundHost('trial-history')
diff --git a/tests/turn-work.integration.mjs b/tests/turn-work.integration.mjs
new file mode 100644
index 0000000..68eb57c
--- /dev/null
+++ b/tests/turn-work.integration.mjs
@@ -0,0 +1,95 @@
+import assert from 'node:assert/strict'
+import { execFile, spawn } from 'node:child_process'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { promisify } from 'node:util'
+import test from 'node:test'
+import { redisRest } from './fixtures/redis-rest.mjs'
+
+// The journal's turn bookkeeping, Lua included, against a local redis-server behind a
+// stand-in for Upstash's REST API. No remote services.
+const run = promisify(execFile)
+const dir = await mkdtemp(join(tmpdir(), 'fx0-turn-work-'))
+const socket = join(dir, 'redis.sock')
+const server = spawn('redis-server', ['--port', '0', '--unixsocket', socket, '--save', '', '--appendonly', 'no'], { stdio: 'ignore' })
+for (let i = 0; i < 100; i++) {
+ try { await run('redis-cli', ['-s', socket, 'PING']); break } catch { await new Promise(resolve => setTimeout(resolve, 10)) }
+}
+const rest = await redisRest(socket)
+process.env.KV_REST_API_URL = rest.url
+process.env.KV_REST_API_TOKEN = 'local-test-token'
+process.env.FX0_REDIS_PREFIX = 'turn-work-test'
+const journal = await import('../lib/journal.ts')
+test.after(async () => {
+ await rest.close()
+ server.kill()
+ await new Promise(resolve => server.once('exit', resolve))
+ await rm(dir, { recursive: true, force: true })
+})
+
+const design = (text) => `<!doctype html><html><head></head><body><p>${text}</p></body></html>`
+const start = (id, prompt) => ({ id, startedAt: 1_000 + id, billingTeamId: 'team_a', prompt, settings: { model: 'test/model', effort: 'low', fast: true } })
+const ended = (id) => ({ type: 'done', id, turnId: id, stopReason: 'end_turn', usage: { inputTokens: 3, outputTokens: 2 }, at: 2_000 + id })
+
+test('a turn is marked running with its start, and its end leaves its history and design in the same commit', async () => {
+ const chat = 'chat-commit'
+ const [started] = await journal.projectEvents(chat, 0, 1, [{ type: 'turn_start', id: 7, turnId: 7, at: 1 }], { 7: start(7, 'Make it teal') })
+ assert.deepEqual(await journal.getActiveTurn(chat), { ...start(7, 'Make it teal'), startSeq: started.seq })
+ // A second projector still at the old cursor changes nothing.
+ assert.equal(await journal.projectEvents(chat, 0, 1, [{ type: 'turn_start', id: 7, turnId: 7, at: 1 }], { 7: start(7, 'other') }), null)
+ assert.equal((await journal.getActiveTurn(chat)).prompt, 'Make it teal')
+
+ assert.equal((await journal.publishDesignArtifact(chat, design('teal'), 7)).revision, 1)
+ await journal.projectEvents(chat, 1, 2, [ended(7)])
+ assert.equal(await journal.getActiveTurn(chat), null)
+ const [work] = await journal.readTurnWork(chat)
+ assert.deepEqual(work, {
+ turnId: 7, type: 'done', stopReason: 'end_turn', usage: { inputTokens: 3, outputTokens: 2 }, at: 2_007,
+ turn: { ...start(7, 'Make it teal'), startSeq: started.seq },
+ design: { revision: 1, html: design('teal') },
+ })
+ // That design is the turn's history version now: a later turn replacing it keeps no second copy.
+ await journal.publishDesignArtifact(chat, design('coral'), 8)
+ assert.deepEqual((await journal.readTurnWork(chat)).map((item) => [item.turnId, item.design?.html]), [[7, design('teal')]])
+
+ await journal.finishTurnWork(chat, 7)
+ assert.deepEqual(await journal.readTurnWork(chat), [])
+})
+
+test('a turn keeps the design it left when the next turn replaces it before its end is projected', async () => {
+ const chat = 'chat-lag'
+ await journal.projectEvents(chat, 0, 1, [{ type: 'turn_start', id: 1, turnId: 1, at: 1 }], { 1: start(1, 'first') })
+ await journal.publishDesignArtifact(chat, design('first'), 1)
+ await journal.publishDesignArtifact(chat, design('first, edited'), 1)
+ // libfx runs the next turn before the projector reaches the first one's end.
+ await journal.publishDesignArtifact(chat, design('second'), 2)
+ await journal.projectEvents(chat, 1, 2, [ended(1)])
+ const [work] = await journal.readTurnWork(chat)
+ assert.equal(work.turnId, 1)
+ assert.deepEqual(work.design, { revision: 2, html: design('first, edited') })
+})
+
+test('several turns in one batch each get their bookkeeping, in order', async () => {
+ const chat = 'chat-batch'
+ await journal.projectEvents(chat, 0, 3, [
+ { type: 'turn_start', id: 4, turnId: 4, at: 1 }, ended(4), { type: 'turn_start', id: 5, turnId: 5, at: 3 },
+ ], { 4: start(4, 'four'), 5: start(5, 'five') })
+ assert.equal((await journal.getActiveTurn(chat)).id, 5)
+ assert.deepEqual((await journal.readTurnWork(chat)).map((item) => [item.turnId, item.turn?.prompt]), [[4, 'four']])
+ // An end whose start was never recorded still leaves its work, without a running record.
+ await journal.projectEvents(chat, 3, 4, [ended(6)])
+ assert.deepEqual((await journal.readTurnWork(chat)).map((item) => [item.turnId, item.turn?.id ?? null]), [[4, 4], [6, null]])
+ assert.equal((await journal.getActiveTurn(chat)).id, 5, 'another turn\'s end leaves the running one alone')
+})
+
+test('deleting a chat removes its waiting history work and designs', async () => {
+ const chat = 'chat-deleted'
+ await journal.projectEvents(chat, 0, 2, [{ type: 'turn_start', id: 1, turnId: 1, at: 1 }, ended(1)], { 1: start(1, 'gone') })
+ await journal.publishDesignArtifact(chat, design('kept for turn 1'), 1)
+ await journal.publishDesignArtifact(chat, design('turn 2'), 2)
+ assert.equal((await journal.readTurnWork(chat)).length, 1)
+ assert.equal(await journal.claimSessionCleanup(chat, 'token'), 'claimed')
+ assert.equal(await journal.finishSessionCleanup(chat, 'token', null), true)
+ assert.deepEqual(await journal.readTurnWork(chat), [])
+})