Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm polls send the machine and agent summary only when it changed (#3011)

harness-for-autonomous[bot]2b164811e255
diff --git a/backend/integration/bench/derivedEtagCost.ts b/backend/integration/bench/derivedEtagCost.ts
new file mode 100644
index 000000000..d921819be
--- /dev/null
+++ b/backend/integration/bench/derivedEtagCost.ts
@@ -0,0 +1,126 @@
+import { setTimeout as sleep } from 'node:timers/promises'
+import type { AuthUser } from '../../src/lib/ssoAuth.js'
+import type { Changes } from '../../src/superTabs/read/contract.js'
+import { readDelta } from '../../src/superTabs/cachedReads.js'
+import { listGroupIndexes } from '../../src/superTabs/parts/indexed.js'
+import { parseArgs } from './config.js'
+import { partsDriver } from './drivers/parts.js'
+import type { Reader, Reply } from './driver.js'
+import { machineId, seedAccounts } from './fixtures.js'
+import { storeRoutes } from './legacyDriver.js'
+import { localDatabase } from './mongo.js'
+import { plan } from './plan.js'
+import { seedSwarm } from './seed.js'
+import { benchStamps } from './stamps.js'
+
+/** Past the change reads' 30 s overlap, so the polls after it resend nothing they sent before. */
+const QUIET_MS = 35_000
+/** Windows of each kind, the same at every size, so a window's cost compares across sizes. */
+const READERS = 10
+
+/**
+ * What an app window's change poll costs with and without the summary etag (task #2906), on Wren13's machine-count
+ * harness (machineCapCost.ts, PR #2909): a Swarm of `--machines` machines, each its own person's, and READERS windows of
+ * each kind polling changes. `old` sends no etag, as every released client; `etag` echoes the `derivedEtag` of its last
+ * read, as this release's Desktop, web and phone do. Rounds as there: after nothing, after every machine's reading, after
+ * one post. Per round, reply KB per poll, the `derived` KB in it, and how many replies left the summary out. From
+ * `backend/`:
+ *
+ * NODE_ENV=test HARNESS_CREDENTIAL_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= \
+ * DATABASE_URL=mongodb://127.0.0.1:1/bench-never-used npx tsx integration/bench/derivedEtagCost.ts --machines 36 [--rounds 3]
+ */
+async function main() {
+ const argv = process.argv.slice(2)
+ const at = argv.indexOf('--rounds'), rounds = at < 0 ? 3 : Number(argv[at + 1])
+ const { config } = parseArgs(argv.filter((arg, i) => arg !== '--rounds' && argv[i - 1] !== '--rounds'))
+ config.desktopReaders = Math.min(config.desktopReaders, config.machines)
+ config.hostReaders = Math.min(config.hostReaders, config.machines)
+ const database = await localDatabase()
+ await listGroupIndexes(database.db)
+ const stamps = await benchStamps('memory', readDelta(database.db))
+ let clock = Date.now()
+ try {
+ const people = await seedAccounts(database.db, config.machines)
+ const driver = await partsDriver(database.db, () => people, { now: () => clock })
+ const { call, close } = await storeRoutes(database.db, () => people)
+ const { swarm } = await seedSwarm(driver, config, plan(config), people)
+ const derivedBytes = new Map<Reader, number>(), unchanged = new Map<Reader, boolean>()
+ const window = (user: AuthUser, echoes: boolean): Reader => {
+ let cursor: string | null = null, etag: string | null = null
+ const reader: Reader = { label: echoes ? 'etag' : 'old', async poll() {
+ const reply = cursor === null
+ ? await call<{ cursor: string; derivedEtag?: string; derived: unknown }>(user, 'GET', `/api/super-tabs/${swarm.tabId}/parts`)
+ : await call<Changes>(user, 'GET', `/api/super-tabs/${swarm.tabId}/changes?cursor=${cursor}${echoes && etag ? `&derivedEtag=${etag}` : ''}`)
+ const data = reply.data as { reload?: boolean; cursor: string; derivedEtag?: string; derived: unknown; derivedUnchanged?: true } | null
+ if (data?.reload) throw new Error('A window was told to reload')
+ if (data) {
+ cursor = data.cursor
+ etag = data.derivedEtag ?? null
+ derivedBytes.set(reader, JSON.stringify(data.derived ?? null).length)
+ unchanged.set(reader, data.derivedUnchanged === true)
+ }
+ return reply
+ } }
+ return reader
+ }
+ const kinds: Record<string, Reader[]> = {
+ old: Array.from({ length: READERS }, (_, i) => window(people[i % people.length]!, false)),
+ etag: Array.from({ length: READERS }, (_, i) => window(people[i % people.length]!, true)),
+ }
+ const all = Object.values(kinds).flat()
+ let n = 0
+ const reading = (machine: number): Promise<unknown> => driver.observeMachine(swarm.tabId, people[machine]!, machineId(machine), {
+ generation: swarm.generations[machine]!, quotas: [], tools: {}, runningHarnessIds: machine === 0 ? ['bench-lead'] : [], stoppedHarnessIds: [],
+ resources: { cpuPercent: 10 + (n++ % 50), memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,
+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Bench', os: 'macOS', gpuPercent: null } })
+ const post = () => driver.execute(swarm.tabId, people[0]!, `etag-cost-${n++}`,
+ { type: 'message.send', conversationId: swarm.workers[0]!.group, text: 'One post.', replyTo: null, recipientIds: [] })
+ for (const r of all) await r.poll()
+ // A first reading after the seed changes more than the reading: done before the quiet wait, as machineCapCost.ts does.
+ await Promise.all(people.map((_, m) => reading(m)))
+ await sleep(QUIET_MS)
+ clock = Date.now()
+ for (const r of all) await r.poll()
+
+ process.stdout.write(`${config.machines} machines (${people.length} people), ${config.groups} groups; ${READERS} windows of each kind\n\n`)
+ process.stdout.write('| Before each round | Window | Polls | Reply KB/poll | `derived` KB/poll | Summary left out |\n|---|---|---|---|---|---|\n')
+ const phases: Array<[string, () => Promise<unknown>]> = [['nothing', async () => {}],
+ ['every machine reading', () => Promise.all(people.map((_, m) => reading(m)))], ['one post', post]]
+ for (const [label, before] of phases) {
+ const sums = Object.fromEntries(Object.keys(kinds).map(k => [k, { polls: 0, reply: 0, derived: 0, unchanged: 0 }]))
+ for (let r = 0; r < rounds; r++) {
+ await before()
+ await sleep(200)
+ for (const [kind, readers] of Object.entries(kinds)) {
+ const replies = await Promise.all(readers.map(p => p.poll()))
+ check(kind, replies)
+ const s = sums[kind]!
+ s.polls += readers.length
+ s.reply += replies.reduce((sum, reply) => sum + reply.bytes, 0)
+ s.derived += readers.reduce((sum, p) => sum + (derivedBytes.get(p) ?? 0), 0)
+ s.unchanged += readers.filter(p => unchanged.get(p)).length
+ }
+ await sleep(3_000)
+ }
+ for (const [kind, s] of Object.entries(sums)) {
+ process.stdout.write(`| ${label} | ${kind} | ${s.polls} | ${(s.reply / s.polls / 1024).toFixed(2)} | ${(s.derived / s.polls / 1024).toFixed(2)} | ` +
+ `${s.unchanged} of ${s.polls} |\n`)
+ }
+ }
+ await close()
+ await driver.close()
+ } finally {
+ await stamps.close()
+ await database.stop()
+ }
+}
+
+function check(kind: string, replies: Reply[]) {
+ const failed = replies.find(reply => reply.code)
+ if (failed) throw new Error(`${kind}: ${failed.code}`)
+}
+
+main().catch(error => {
+ process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`)
+ process.exitCode = 1
+})
diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.ts
index 7680589cc..7b2aa6bf2 100644
--- a/backend/integration/superTabRead.test.ts
+++ b/backend/integration/superTabRead.test.ts
@@ -127,7 +127,7 @@ describe('first load (design 4.3)', () => {
clock = NOW
const { status, body } = await get(`/api/super-tabs/${s.tabId}/parts`, peer)
expect(status).toBe(200)
- expect(Object.keys(body.data).sort()).toEqual(['agents', 'cursor', 'derived', 'epoch', 'format', 'groups', 'head', 'lastChangedAt',
+ expect(Object.keys(body.data).sort()).toEqual(['agents', 'cursor', 'derived', 'derivedEtag', 'epoch', 'format', 'groups', 'head', 'lastChangedAt',
'machines', 'members', 'progress', 'serverTime', 'starts', 'steps', 'tabId'].sort())
expect(body.data).toMatchObject({ format: 2, tabId: s.tabId, serverTime: NOW, epoch: 1, lastChangedAt: MIGRATED, steps: true,
head: { key: 'head', version: R, changedAt: MIGRATED, data: { id: s.tabId, name: 'Pilot' } } })
@@ -1240,6 +1240,74 @@ describe('decisions asked of one person (v2)', () => {
})
})
+// Task #2906: an unchanged poll leaves out the machine and agent summary the reader already holds (12 KB a poll at 36
+// machines, 92 KB at 360). A released client, which echoes no etag, still gets it whole.
+describe('the summary only when it changed (derivedEtag)', () => {
+ const read = async (url: string, as: AuthUser) => { clock = Date.now(); return (await get(url, as)).body.data }
+ const poll = (tabId: string, cursor: string, as: AuthUser, etag?: string) =>
+ read(`/api/super-tabs/${tabId}/changes?cursor=${cursor}${etag ? `&derivedEtag=${etag}` : ''}`, as)
+ /** An open decision asked of peer, written as a row (as `askedBefore` does): a command would leave its notices and wakes
+ * running into the next tests, whose database-byte counts (group 3b) then count them. */
+ const ask = async (tabId: string, conversationId: string, lead: string) => {
+ await db.superTabDecision.create({ data: { id: `etag-${randomUUID()}`, tabId, batchId: `${conversationId}-etag`, projectId: 'lead',
+ conversationId, assignmentId: null, personId: peer.sub, askerAgentId: lead, question: 'Ship how?', options: ['Behind a flag', 'To everyone'],
+ recommended: 0, unblocks: 'The release', createdAt: new Date() } })
+ }
+
+ it('stamps a first load and a change read alike, and an echoed etag gets no summary', async () => {
+ const s = await scene()
+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)
+ expect(first.derivedEtag).toMatch(/^[A-Za-z0-9_-]{22}$/)
+ const unchanged = await poll(s.tabId, first.cursor, peer, first.derivedEtag)
+ expect(unchanged).toMatchObject({ reload: false, derivedEtag: first.derivedEtag, derivedUnchanged: true })
+ expect(Object.keys(unchanged.derived).filter(k => !['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'].includes(k))).toEqual([])
+ })
+
+ it('sends the whole summary to a client that echoes none, or another etag', async () => {
+ const s = await scene()
+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)
+ for (const etag of [undefined, 'not-the-etag']) {
+ const reply = await poll(s.tabId, first.cursor, peer, etag)
+ expect(reply.derivedUnchanged, String(etag)).toBeUndefined()
+ expect(reply.derivedEtag, String(etag)).toBe(first.derivedEtag)
+ expect(reply.derived, String(etag)).toEqual(first.derived)
+ }
+ })
+
+ it('a summary change after an unchanged poll sends it whole, with a new etag', async () => {
+ const s = await scene()
+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)
+ expect(first.derived.decisionsWaiting).toBe(0)
+ const quiet = await poll(s.tabId, first.cursor, peer, first.derivedEtag)
+ expect(quiet.derivedUnchanged).toBe(true)
+ await ask(s.tabId, s.web, s.lead)
+ const moved = await poll(s.tabId, quiet.cursor, peer, quiet.derivedEtag)
+ expect(moved.derivedUnchanged).toBeUndefined()
+ expect(moved.derivedEtag).not.toBe(first.derivedEtag)
+ expect(moved.derived).toMatchObject({ decisionsWaiting: 1, agents: first.derived.agents })
+ expect((await poll(s.tabId, moved.cursor, peer, moved.derivedEtag)).derivedUnchanged).toBe(true)
+ })
+
+ it('an unchanged poll still sends the reader\'s moved marks', async () => {
+ const s = await scene()
+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)
+ await writeReadMark(db, { tabId: s.tabId, personId: peer.sub, conversationId: s.web, sequence: 3, now: Date.now() })
+ const reply = await poll(s.tabId, first.cursor, peer, first.derivedEtag)
+ expect(reply.derivedUnchanged).toBe(true)
+ expect(reply.derived).toEqual({ readMarks: { [s.web]: 3 } })
+ })
+
+ it('an etag is the reader\'s own: another reader\'s summary never matches it', async () => {
+ const s = await scene()
+ await ask(s.tabId, s.web, s.lead)
+ const [mine, theirs] = [await read(`/api/super-tabs/${s.tabId}/parts`, peer), await read(`/api/super-tabs/${s.tabId}/parts`, owner)]
+ expect(mine.derivedEtag).not.toBe(theirs.derivedEtag)
+ const reply = await poll(s.tabId, mine.cursor, peer, theirs.derivedEtag)
+ expect(reply.derivedUnchanged).toBeUndefined()
+ expect(reply.derived).toMatchObject({ decisionsWaiting: 1 })
+ })
+})
+
describe('routes given only a store', () => {
it('read from that store\'s database, not the app-wide client', async () => {
const s = await scene()
diff --git a/backend/src/routes/superTabs.test.ts b/backend/src/routes/superTabs.test.ts
index 1fee25446..7256a9809 100644
--- a/backend/src/routes/superTabs.test.ts
+++ b/backend/src/routes/superTabs.test.ts
@@ -334,6 +334,23 @@ describe('the machine lines of a change read', () => {
expect(changes).not.toHaveBeenCalled()
})
+ // Task #2906: a client that holds the summary echoes its etag; a released client sends none and is asked nothing new.
+ it('passes the summary etag a client echoes, with lines or without', async () => {
+ expect((await get('/api/super-tabs/tab1/changes?cursor=abc&derivedEtag=e1_-Z')).statusCode).toBe(200)
+ expect(changes.mock.calls[0]!.slice(2)).toEqual(['abc', undefined, { derivedEtag: 'e1_-Z' }])
+ expect((await get('/api/super-tabs/tab1/changes?cursor=abc&machines=light&derivedEtag=e2')).statusCode).toBe(200)
+ expect(changes.mock.calls[1]!.slice(2)).toEqual(['abc', undefined, { lightMachines: true, derivedEtag: 'e2' }])
+ })
+
+ it('refuses a malformed summary etag', async () => {
+ for (const etag of ['bad!', '', 'x'.repeat(65)]) {
+ const res = await get(`/api/super-tabs/tab1/changes?cursor=abc&derivedEtag=${etag}`)
+ expect(res.statusCode, etag).toBe(400)
+ expect(res.json().error.code).toBe('VALIDATION_ERROR')
+ }
+ expect(changes).not.toHaveBeenCalled()
+ })
+
it('serves the whole readings on demand, for an agent too', async () => {
expect((await get('/api/super-tabs/tab1/machine-readings?asAgentId=agent1')).statusCode).toBe(200)
expect(machineReadings.mock.calls[0]!.slice(1)).toEqual(['tab1', 'agent1'])
diff --git a/backend/src/routes/superTabs.ts b/backend/src/routes/superTabs.ts
index fd79a3c43..3592f2caf 100644
--- a/backend/src/routes/superTabs.ts
+++ b/backend/src/routes/superTabs.ts
@@ -81,7 +81,9 @@ const workspaceJoin = z.object({ operationId: id }).strict()
const personOnly = (user: AuthUser) => { if (isTeamSession(user)) fail(TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE, 403) }
const listQuery = z.object({ after: id.optional(), mentions: z.literal('1').optional() }).strict()
const cursor = z.string().min(1).max(512).regex(/^[A-Za-z0-9_-]+$/)
-const changesQuery = agentQuery.extend({ cursor, machines: z.literal('light').optional() })
+// `derivedEtag`: the summary's etag the client holds (read/derivedEtag.ts, task #2906).
+const changesQuery = agentQuery.extend({ cursor, machines: z.literal('light').optional(),
+ derivedEtag: z.string().min(1).max(64).regex(/^[A-Za-z0-9_-]+$/).optional() })
const costsQuery = z.object({ days: z.coerce.number().int().min(1).max(MAX_DAYS).default(DEFAULT_DAYS), asAgentId: id.optional() }).strict()
const leaderboardQuery = z.object({ projectId: id, week: z.coerce.number().int().nonnegative().safe().optional(), period: z.enum(PERIODS).optional(), scope: z.literal('all').optional() }).strict()
.refine(query => query.week === undefined || query.period === undefined, 'Ask for a week or a period, not both.')
@@ -521,7 +523,8 @@ export async function superTabRoutes(app: FastifyInstance, options: { store?: Su
app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof changesQuery> }>('/api/super-tabs/:tabId/changes',
{ preHandler: [validateParams(tabParams), validateQuery(changesQuery)] },
(req, reply) => respond(reply, noting(req, () => reads.changes(req.user!, req.params.tabId, req.query.cursor, req.query.asAgentId,
- req.query.machines === 'light' ? { lightMachines: true } : {}), nothingChanged)))
+ { ...(req.query.machines === 'light' ? { lightMachines: true } : {}), ...(req.query.derivedEtag ? { derivedEtag: req.query.derivedEtag } : {}) }),
+ nothingChanged)))
// The whole readings on demand, for a client that takes lines in its changes (read/changes.ts).
app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof agentQuery> }>('/api/super-tabs/:tabId/machine-readings',
{ preHandler: [validateParams(tabParams), validateQuery(agentQuery)] },
diff --git a/backend/src/superTabs/read/contract.ts b/backend/src/superTabs/read/contract.ts
index 755fa1d29..226ca0ae8 100644
--- a/backend/src/superTabs/read/contract.ts
+++ b/backend/src/superTabs/read/contract.ts
@@ -153,9 +153,15 @@ export interface FirstLoad extends ReadMeta {
starts: LivePart<StartView>[]
progress: LivePart<ProgressView>[]
derived: Derived
+ /** Opaque: the reader's summary (`derived` without its marks) as a hash (derivedEtag.ts). Send it back as `derivedEtag` on
+ * the next change read. Absent from an older backend. */
+ derivedEtag: string
}
-/** GET /api/super-tabs/:tabId/changes?cursor=[&asAgentId=] */
+/** The reader's own marks of `derived`, which a change read sends only as they move. */
+export type ReaderMarks = Pick<Derived, 'readMarks' | 'unreadMarks' | 'threadMarks' | 'chatSettings'>
+
+/** GET /api/super-tabs/:tabId/changes?cursor=[&asAgentId=][&machines=light][&derivedEtag=] */
export type Changes = ChangesReload | ChangesDelta
/** Drop the store and do a first load: bad or old cursor, another epoch, or older than tombstones are kept. */
export interface ChangesReload { reload: true }
@@ -175,7 +181,13 @@ export interface ChangesDelta extends ReadMeta {
machineReadings?: MachineReading[]
starts: PartChange<StartView>[]
progress: PartChange<ProgressView>[]
- derived: Derived
+ /** The whole summary, or with `derivedUnchanged` only the reader's moved marks: the client keeps the summary it holds. */
+ derived: Derived | ReaderMarks
+ /** As a first load's: the summary this reply's reader has now, sent whether or not `derived` carries it. */
+ derivedEtag: string
+ /** Only when the client sent the `derivedEtag` its summary still hashes to (task #2906): `derived` then holds only the
+ * reader's moved marks. A client that sends no etag, as every released one, always gets the whole summary. */
+ derivedUnchanged?: true
}
/** GET /api/super-tabs/:tabId/groups?dormant=1[&projectId=][&before=][&asAgentId=]: newest activity first, 50 a page. */
diff --git a/backend/src/superTabs/read/derivedEtag.test.ts b/backend/src/superTabs/read/derivedEtag.test.ts
new file mode 100644
index 000000000..27983e6cb
--- /dev/null
+++ b/backend/src/superTabs/read/derivedEtag.test.ts
@@ -0,0 +1,48 @@
+import { describe, expect, it } from 'vitest'
+import type { Derived } from './contract.js'
+import { derivedEtag, MARK_FIELDS } from './derivedEtag.js'
+
+const summary = (): Derived => ({
+ agents: { a1: { downSince: null, limitedUntil: null, workDecision: null, roleId: null, roleName: 'Engineer' } },
+ machines: { m1: { availableSlots: 2, teamWork: { allowed: true } } },
+ mentionsWaiting: 1, mentionsDigest: 'd1', avatars: { 'person:p1': 'https://example.test/a?sig=1' },
+})
+
+// Task #2906: the etag a client echoes so an unchanged poll leaves the summary out.
+describe('derivedEtag', () => {
+ it('is short and URL-safe', () => {
+ expect(derivedEtag(summary())).toMatch(/^[A-Za-z0-9_-]{22}$/)
+ })
+
+ it('is the same for the same summary built in another key order', () => {
+ const s = summary()
+ const reordered: Derived = { avatars: s.avatars, mentionsDigest: s.mentionsDigest, mentionsWaiting: s.mentionsWaiting,
+ machines: { m1: { teamWork: { allowed: true }, availableSlots: 2 } }, agents: s.agents }
+ expect(derivedEtag(reordered)).toBe(derivedEtag(s))
+ })
+
+ it('leaves out the reader\'s marks, which a change read sends only as they move', () => {
+ expect(MARK_FIELDS).toEqual(['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'])
+ const marked: Derived = { ...summary(), readMarks: { g1: 4 }, unreadMarks: { g1: 3 }, threadMarks: { g1: { r1: 2 } }, chatSettings: { g1: 'all' } }
+ expect(derivedEtag(marked)).toBe(derivedEtag(summary()))
+ })
+
+ it('moves with any other value', () => {
+ const base = derivedEtag(summary())
+ const changes: Array<(d: Derived) => void> = [
+ d => { d.agents.a1!.downSince = '2026-11-04T00:00:00.000Z' },
+ d => { d.agents.a1!.roleName = 'Reviewer' },
+ d => { d.machines.m1!.availableSlots = 1 },
+ d => { d.machines.m1!.teamWork = { allowed: false, reason: 'full' } },
+ d => { d.avatars = { 'person:p1': 'https://example.test/a?sig=2' } },
+ d => { d.mentionsWaiting = 2 },
+ d => { d.decisionsDigest = 'x' },
+ d => { delete d.avatars },
+ ]
+ for (const change of changes) {
+ const d = summary()
+ change(d)
+ expect(derivedEtag(d), change.toString()).not.toBe(base)
+ }
+ })
+})
diff --git a/backend/src/superTabs/read/derivedEtag.ts b/backend/src/superTabs/read/derivedEtag.ts
new file mode 100644
index 000000000..6a6f225b7
--- /dev/null
+++ b/backend/src/superTabs/read/derivedEtag.ts
@@ -0,0 +1,23 @@
+import { createHash } from 'node:crypto'
+import type { Derived } from './contract.js'
+import { sorted } from './hostEtag.js'
+
+/** The reader's own marks: a change read sends only those moved since its cursor, so they are no part of the summary. */
+export const MARK_FIELDS = ['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'] as const
+export type MarkField = typeof MARK_FIELDS[number]
+
+/**
+ * A reader's summary (`derived` without its marks) as a short hash (task #2906): equal summaries, equal etag, whatever
+ * order their records were built in. A client echoes it as `derivedEtag` on its next change read, and while it still
+ * matches the reply leaves the summary out (read/service.ts changes), as a host's `held` etag does (hostEtag.ts).
+ */
+export function derivedEtag(derived: Derived): string {
+ const summary: Record<string, unknown> = { ...derived }
+ for (const field of MARK_FIELDS) delete summary[field]
+ return createHash('sha256').update(sorted(summary)).digest('base64url').slice(0, 22)
+}
+
+/** Only the reader's marks of `derived`, the fields it has. */
+export function marksOf(derived: Derived): Pick<Derived, MarkField> {
+ return Object.fromEntries(MARK_FIELDS.filter(field => derived[field] !== undefined).map(field => [field, derived[field]]))
+}
diff --git a/backend/src/superTabs/read/hostEtag.ts b/backend/src/superTabs/read/hostEtag.ts
index c6fc32a2a..3869025f9 100644
--- a/backend/src/superTabs/read/hostEtag.ts
+++ b/backend/src/superTabs/read/hostEtag.ts
@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'
import type { HostTabRows } from './contract.js'
/** JSON with sorted keys, so equal rows hash equal whatever order their records were built in. */
-const sorted = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === 'object' && !Array.isArray(item)
+export const sorted = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === 'object' && !Array.isArray(item)
? Object.fromEntries(Object.entries(item as Record<string, unknown>).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item)
/**
diff --git a/backend/src/superTabs/read/service.ts b/backend/src/superTabs/read/service.ts
index 21364a3ca..e9103e6cd 100644
--- a/backend/src/superTabs/read/service.ts
+++ b/backend/src/superTabs/read/service.ts
@@ -33,7 +33,8 @@ import { LEDGER_WEEKS } from '../leaderboard/record.js'
import type { PeriodName } from '../leaderboard/rolling.js'
import { leaderboard } from '../leaderboard/stats.js'
import { WEEK_MS } from '../leaderboard/week.js'
-import { READ_FORMAT, SWARM_MIGRATING, type Changes, type Derived, type MachineReadings, type DormantGroups, type FirstLoad, type OneGroup, type SwarmList } from './contract.js'
+import { derivedEtag, marksOf } from './derivedEtag.js'
+import { READ_FORMAT, SWARM_MIGRATING, type Changes, type Derived, type ReaderMarks, type MachineReadings, type DormantGroups, type FirstLoad, type OneGroup, type SwarmList } from './contract.js'
import { AvatarLinks } from '../avatarLinks.js'
import { cursorFor, readCursor } from './cursor.js'
import { changesFor } from './changes.js'
@@ -73,6 +74,14 @@ function readableMarks(read: Read, marks: ReadMarkRow[], threads: Map<string, Ma
...(Object.keys(threadMarks).length ? { threadMarks } : {}), ...(Object.keys(chatSettings).length ? { chatSettings } : {}) }
}
+/** A reply's `derived` with its etag, or only the reader's marks when `held` is still the etag of the summary (task #2906). */
+function stamped(derived: Derived): { derived: Derived; derivedEtag: string }
+function stamped(derived: Derived, held: string | undefined): { derived: Derived | ReaderMarks; derivedEtag: string; derivedUnchanged?: true }
+function stamped(derived: Derived, held?: string) {
+ const etag = derivedEtag(derived)
+ return held === etag ? { derived: marksOf(derived), derivedEtag: etag, derivedUnchanged: true as const } : { derived, derivedEtag: etag }
+}
+
/** A change poll's light base (parts/light.ts): the same for every reader of a Swarm, so one load per stamp serves them all. */
const LIGHT = { groups: 'working' } as const
@@ -114,8 +123,8 @@ export class SwarmReads {
return { format: READ_FORMAT, tabId, cursor: cursorFor(t0, loaded.visibilityEpoch), serverTime: t0, lastChangedAt, epoch: loaded.visibilityEpoch,
// This backend opens steps (task steps design): a newer CLI sends `step` only to a backend that says so.
...(this.options.githubApp ? { githubApp: true as const } : {}), steps: true as const, ...view.parts,
- derived: { ...await this.withAvatars(view, t0), ...readableMarks(read, await this.marksOf(tx, tabId, read.actor),
- await this.threadMarksOf(tx, tabId, read.actor), await this.settingsOf(tx, tabId, read.actor)) } }
+ ...stamped({ ...await this.withAvatars(view, t0), ...readableMarks(read, await this.marksOf(tx, tabId, read.actor),
+ await this.threadMarksOf(tx, tabId, read.actor), await this.settingsOf(tx, tabId, read.actor)) }) }
})
}
@@ -163,7 +172,10 @@ export class SwarmReads {
})
}
- async changes(user: AuthUser, tabId: string, cursor: string, asAgentId?: string, options: { lightMachines?: boolean } = {}): Promise<Changes> {
+ /** `derivedEtag`: the etag of the summary the client holds (derivedEtag.ts); while it still matches, the reply leaves the
+ * summary out and sends only the reader's moved marks. */
+ async changes(user: AuthUser, tabId: string, cursor: string, asAgentId?: string,
+ options: { lightMachines?: boolean; derivedEtag?: string } = {}): Promise<Changes> {
const t0 = this.now()
return this.read(user, tabId, async (tx, cached) => {
// Light (parts/light.ts): changed parts come whole from the change feed; the rest is read only to check visibility and
@@ -197,7 +209,7 @@ export class SwarmReads {
if (!asAgentId) sent.groups = await this.servedTops.served(tabId, read.view.state, sent.groups)
return { format: READ_FORMAT, reload: false, cursor: cursorFor(t0, read.parts.visibilityEpoch), serverTime: t0, lastChangedAt: read.lastChangedAt,
epoch: read.parts.visibilityEpoch, ...sent,
- derived: { ...await this.withAvatars(read.view, t0), ...readableMarks(read, marks, threadMarks, settings) } }
+ ...stamped({ ...await this.withAvatars(read.view, t0), ...readableMarks(read, marks, threadMarks, settings) }, options.derivedEtag) }
})
}
diff --git a/desktop/lib/super_tabs/controller.dart b/desktop/lib/super_tabs/controller.dart
index 4f9795826..830935435 100644
--- a/desktop/lib/super_tabs/controller.dart
+++ b/desktop/lib/super_tabs/controller.dart
@@ -4038,7 +4038,7 @@ class SuperTabController extends ChangeNotifier {
try {
final answer = await request(
'GET',
- '$_path/changes?cursor=${Uri.encodeQueryComponent(held.cursor)}',
+ '$_path/changes?${held.changesQuery}',
);
if (answer['reload'] != true) {
final SuperTabPartsChange change;
@@ -4059,7 +4059,14 @@ class SuperTabController extends ChangeNotifier {
);
}
} on SuperTabFailure catch (failure) {
- if (!const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) rethrow;
+ // A backend rolled back to one without the summary etag refuses it
+ // (#2906): read in full, which holds none.
+ final etagRefused =
+ failure.code == 'VALIDATION_ERROR' && held.derivedEtag != null;
+ if (!etagRefused &&
+ !const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) {
+ rethrow;
+ }
}
_parts = null;
}
diff --git a/desktop/lib/super_tabs/parts.dart b/desktop/lib/super_tabs/parts.dart
index 06a186b3f..acf823696 100644
--- a/desktop/lib/super_tabs/parts.dart
+++ b/desktop/lib/super_tabs/parts.dart
@@ -55,6 +55,7 @@ class SuperTabParts {
required this.epoch,
required this.lastChangedAt,
required this.githubApp,
+ this.derivedEtag,
});
/// A first load (section 4.3). Throws a [FormatException] for a response of another format.
@@ -76,6 +77,7 @@ class SuperTabParts {
// Null for a Swarm with no part changed yet (contract.ts `ReadMeta`).
lastChangedAt: _int(answer['lastChangedAt'] ?? 0),
githubApp: answer['githubApp'] == true,
+ derivedEtag: _etag(answer),
);
parts._seen['head'] = _int(parts._head['version']);
for (final kind in _lists) {
@@ -131,6 +133,26 @@ class SuperTabParts {
/// Whether the backend has the GitHub app, as the first load says.
final bool githubApp;
+ /// The etag of the summary held (`derived` without the marks, backend read/derivedEtag.ts, #2906), as the last read
+ /// sent it: sent back with the cursor, so a change read leaves the summary out while it still matches. Null from a
+ /// backend without it, and then none is sent.
+ String? derivedEtag;
+
+ /// What a change read asks after `?`: the cursor, and the summary's etag when one is held.
+ String get changesQuery {
+ final etag = derivedEtag;
+ final query = 'cursor=${Uri.encodeQueryComponent(cursor)}';
+ return etag == null
+ ? query
+ : '$query&derivedEtag=${Uri.encodeQueryComponent(etag)}';
+ }
+
+ static String? _etag(Map<String, dynamic> answer) =>
+ switch (answer['derivedEtag']) {
+ final String etag when etag.isNotEmpty => etag,
+ _ => null,
+ };
+
Map<String, dynamic> _head;
Map<String, dynamic> _derived;
final _live = <String, Map<String, Map<String, dynamic>>>{};
@@ -269,6 +291,14 @@ class SuperTabParts {
/// Merges one change read (section 4.4): a higher version wins, a tombstone removes the part, a dormant stub moves
/// the group out of the active ones, `derived` is replaced, and the cursor moves. A reload answer is the caller's.
SuperTabPartsChange apply(Map<String, dynamic> changes) {
+ // Without the summary, a reply is only right for the one held: checked before anything is merged.
+ final unchanged = changes['derivedUnchanged'] == true;
+ final etag = _etag(changes);
+ if (unchanged && (etag == null || etag != derivedEtag)) {
+ throw const FormatException(
+ 'A Swarm read left out a summary this reader does not hold.',
+ );
+ }
var changed = false;
// Anything but a machine's own report moved.
var beyondMachines = false;
@@ -333,11 +363,14 @@ class SuperTabParts {
_takeThreadMarks(derived.remove('threadMarks')) |
_takeChatSettings(derived.remove('chatSettings')) |
readMarksMoved;
- if (!const DeepCollectionEquality().equals(derived, _derived)) {
+ // Unchanged: `derived` held only the marks, and the summary held stands.
+ if (!unchanged &&
+ !const DeepCollectionEquality().equals(derived, _derived)) {
if (!_sameBeyondMachines(derived, _derived)) beyondMachines = true;
_derived = derived;
changed = true;
}
+ derivedEtag = etag;
cursor = _cursor(changes);
epoch = _int(changes['epoch'] ?? epoch);
lastChangedAt = _int(changes['lastChangedAt'] ?? lastChangedAt);
@@ -494,6 +527,7 @@ class SuperTabParts {
for (final kind in _lists)
kind: (_live[kind] ?? const {}).values.toList(),
'derived': _derived,
+ if (derivedEtag != null) 'derivedEtag': derivedEtag,
},
'dormant': _dormant.values.toList(),
'seen': Map<String, int>.of(_seen),
diff --git a/desktop/test/super_tab_parts_reading_test.dart b/desktop/test/super_tab_parts_reading_test.dart
index d3dfaa373..25f4cd8f3 100644
--- a/desktop/test/super_tab_parts_reading_test.dart
+++ b/desktop/test/super_tab_parts_reading_test.dart
@@ -123,6 +123,65 @@ void main() {
},
);
+ // #2906: the machine and agent summary is sent again only when it changed.
+ test(
+ 'an unchanged poll echoes the summary etag and keeps the summary it holds',
+ () async {
+ server.state['mentionsWaiting'] = 2;
+ final c = open();
+ await c.refresh();
+ await c.refresh();
+ final echoed = Uri.parse(reads.last);
+ expect(echoed.path, '/api/super-tabs/tab/changes');
+ expect(echoed.queryParameters['derivedEtag'], isNotEmpty);
+ expect(server.parts.unchangedSummaries, 1);
+ expect(c.snapshot!.mentionsWaiting, 2);
+
+ // The summary moves: the next read sends it whole, and it replaces the one held.
+ server.state['mentionsWaiting'] = 3;
+ await c.refresh();
+ expect(server.parts.unchangedSummaries, 1);
+ expect(c.snapshot!.mentionsWaiting, 3);
+ await c.refresh();
+ expect(server.parts.unchangedSummaries, 2);
+ expect(c.snapshot!.mentionsWaiting, 3);
+ expect(c.error, isNull);
+ },
+ );
+
+ test('a refused etag (a backend rolled back) reads in full once, then polls without it', () async {
+ final c = open();
+ await c.refresh();
+ await c.refresh();
+ expect(Uri.parse(reads.last).queryParameters['derivedEtag'], isNotNull);
+ server.parts.stampsDerived = false;
+ await c.refresh();
+ await c.refresh();
+ expect(reads.skip(2).map((p) => Uri.parse(p).path), [
+ '/api/super-tabs/tab/changes',
+ '/api/super-tabs/tab/parts',
+ '/api/super-tabs/tab/changes',
+ ]);
+ expect(
+ Uri.parse(reads.last).queryParameters,
+ isNot(contains('derivedEtag')),
+ );
+ expect(c.error, isNull);
+ expect(c.accessRevoked, isFalse);
+ });
+
+ test('a backend without the etag is polled as before', () async {
+ server.parts.stampsDerived = false;
+ final c = open();
+ await c.refresh();
+ await c.refresh();
+ await c.refresh();
+ for (final path in reads.skip(1)) {
+ expect(Uri.parse(path).queryParameters.keys, ['cursor']);
+ }
+ expect(c.error, isNull);
+ });
+
test(
'a removed group leaves the snapshot and its local state goes',
() async {
diff --git a/desktop/test/super_tab_parts_test.dart b/desktop/test/super_tab_parts_test.dart
index 34679f13a..48bde0831 100644
--- a/desktop/test/super_tab_parts_test.dart
+++ b/desktop/test/super_tab_parts_test.dart
@@ -287,6 +287,100 @@ void main() {
expect(snapshot.people.map((p) => p.id), ['6a86', '6a87']);
});
+ // #2906: a change read leaves out the summary the reader holds while its etag still matches.
+ {
+ Map<String, dynamic> load() => {
+ ...recorded('first_load'),
+ 'derivedEtag': 'e1',
+ };
+ // Nothing else moved either: no head, no parts.
+ Map<String, dynamic> unchanged(String etag, [Map? marks]) => {
+ ...recorded('changes'),
+ 'head': null,
+ for (final kind in [
+ 'members',
+ 'agents',
+ 'groups',
+ 'machines',
+ 'starts',
+ 'progress',
+ ])
+ kind: <dynamic>[],
+ 'derivedEtag': etag,
+ 'derivedUnchanged': true,
+ 'derived': {'readMarks': ?marks},
+ };
+
+ test(
+ 'summary etag: is held from a first load and sent back with the cursor',
+ () {
+ final parts = SuperTabParts.firstLoad(load());
+ expect(parts.derivedEtag, 'e1');
+ expect(Uri.splitQueryString(parts.changesQuery), {
+ 'cursor': parts.cursor,
+ 'derivedEtag': 'e1',
+ });
+ },
+ );
+
+ test('summary etag: an unchanged reply keeps the summary held and takes its marks', () {
+ final parts = SuperTabParts.firstLoad(load());
+ final held = jsonEncode(parts.derived);
+ final change = parts.apply(unchanged('e1', {'aeef': 9}));
+ expect(jsonEncode(parts.derived), held);
+ expect(parts.derived['agents'], isNotEmpty);
+ expect(parts.derivedEtag, 'e1');
+ expect(change.readMarksOnly, isTrue);
+ });
+
+ test('summary etag: a reply with the summary replaces it and its etag', () {
+ final parts = SuperTabParts.firstLoad(load());
+ parts.apply({
+ ...recorded('changes'),
+ 'derivedEtag': 'e2',
+ 'derived': {
+ 'agents': <String, dynamic>{},
+ 'machines': <String, dynamic>{},
+ 'mentionsWaiting': 4,
+ },
+ });
+ expect(parts.derivedEtag, 'e2');
+ expect(parts.derived['mentionsWaiting'], 4);
+ expect(parts.derived['agents'], isEmpty);
+ });
+
+ test('summary etag: "unchanged" for an etag it does not hold is refused, not taken as no summary', () {
+ final parts = SuperTabParts.firstLoad(load());
+ expect(() => parts.apply(unchanged('e9')), throwsFormatException);
+ final older = SuperTabParts.firstLoad(recorded('first_load'));
+ expect(() => older.apply(unchanged('e1')), throwsFormatException);
+ });
+
+ test(
+ 'summary etag: an older backend sends none, and none is sent back',
+ () {
+ final parts = SuperTabParts.firstLoad(recorded('first_load'));
+ expect(parts.derivedEtag, isNull);
+ expect(Uri.splitQueryString(parts.changesQuery), {
+ 'cursor': parts.cursor,
+ });
+ final rolledBack = SuperTabParts.firstLoad(load())
+ ..apply(recorded('changes'));
+ expect(rolledBack.derivedEtag, isNull);
+ },
+ );
+
+ test('summary etag: a saved store keeps its etag', () {
+ final parts = SuperTabParts.firstLoad(load());
+ final kept = SuperTabParts.fromJson(
+ jsonDecode(jsonEncode(parts.toJson())) as Map<String, dynamic>,
+ );
+ expect(kept.derivedEtag, 'e1');
+ expect(jsonEncode(kept.apply(unchanged('e1')).changed), 'false');
+ expect(jsonEncode(kept.derived), jsonEncode(parts.derived));
+ });
+ }
+
test('what it keeps reads back as the same store', () {
final parts = SuperTabParts.firstLoad(recorded('first_load'))
..apply(recorded('changes'))
diff --git a/desktop/test/support/super_tab_fixture.dart b/desktop/test/support/super_tab_fixture.dart
index ca548376d..228832152 100644
--- a/desktop/test/support/super_tab_fixture.dart
+++ b/desktop/test/support/super_tab_fixture.dart
@@ -58,6 +58,26 @@ class SwarmPartsBackend {
/// The visibility epoch: a test moves it to have the next change read answer a reload.
int epoch = 1;
+ /// A backend with the summary etag (backend read/derivedEtag.ts, #2906). Off, it is one from before it, or one rolled
+ /// back: no etag is sent, and a change read naming one is refused by its strict query.
+ bool stampsDerived = true;
+
+ /// Change reads answered without the summary, its etag still matching.
+ int unchangedSummaries = 0;
+
+ static const _markFields = {
+ 'readMarks',
+ 'unreadMarks',
+ 'threadMarks',
+ 'chatSettings',
+ };
+
+ /// The summary without the reader's marks, as one string: equal summaries, equal etag.
+ static String _etag(Map<String, dynamic> derived) => jsonEncode({
+ for (final entry in derived.entries)
+ if (!_markFields.contains(entry.key)) entry.key: entry.value,
+ }).hashCode.toRadixString(36);
+
/// The ticks a change read sends again, as the 30 s overlap does.
int overlap = 3;
@@ -273,6 +293,11 @@ class SwarmPartsBackend {
for (final kind in _kinds) kind: <dynamic>[],
'derived': _derived(state, reader),
};
+ if (stampsDerived) {
+ answer['derivedEtag'] = _etag(
+ answer['derived'] as Map<String, dynamic>,
+ );
+ }
for (final entry in _ordered()) {
final kind = entry.key.substring(0, entry.key.indexOf(':'));
if (kind == 'head' || entry.value.deleted) continue;
@@ -284,6 +309,14 @@ class SwarmPartsBackend {
}
if (segments.length == 4 && route == 'changes') {
reads.add(path);
+ final held = uri.queryParameters['derivedEtag'];
+ if (held != null && !stampsDerived) {
+ throw const SuperTabFailure(
+ 'VALIDATION_ERROR',
+ 'Unrecognized key(s) in object: \'derivedEtag\'',
+ statusCode: 400,
+ );
+ }
final cursor = RegExp(r'^c(\d+)e(\d+)$')
.firstMatch(uri.queryParameters['cursor'] ?? '');
if (cursor == null || int.parse(cursor.group(2)!) != epoch) {
@@ -303,6 +336,18 @@ class SwarmPartsBackend {
for (final kind in _kinds) kind: <dynamic>[],
'derived': _derived(state, reader),
};
+ if (stampsDerived) {
+ final derived = answer['derived'] as Map<String, dynamic>;
+ final etag = answer['derivedEtag'] = _etag(derived);
+ if (held == etag) {
+ unchangedSummaries++;
+ answer['derivedUnchanged'] = true;
+ answer['derived'] = {
+ for (final entry in derived.entries)
+ if (_markFields.contains(entry.key)) entry.key: entry.value,
+ };
+ }
+ }
for (final entry in _ordered()) {
if (entry.value.changedAt <= since) continue;
if (_hides(entry.key, entry.value, reader, state)) continue;
diff --git a/docs/design/2026-10-26-swarm-state-parts.md b/docs/design/2026-10-26-swarm-state-parts.md
index 9a3ccaacc..0ae6bbe3c 100644
--- a/docs/design/2026-10-26-swarm-state-parts.md
+++ b/docs/design/2026-10-26-swarm-state-parts.md
@@ -566,6 +566,11 @@ stands. Every contributed machine has an entry in `derived.machines`. These are
- **The client merges** by `(kind, key)`, keeping the higher `version`. A tombstone removes the part. A dormant stub
moves the group out of the active list. The client replaces `derived` and moves to the returned `cursor`.
Overlap duplicates have equal versions and are ignored.
+- **The summary only when it changed** (task #2906, `read/derivedEtag.ts`). Every first load and change read also
+ carries `derivedEtag`, a hash of `derived` without the reader's marks. A client that sends it back as
+ `&derivedEtag=` gets, while it still matches, `derivedUnchanged: true` and a `derived` holding only its moved marks,
+ and keeps the summary it holds. A client that sends none gets `derived` whole. A client refused for the parameter (a
+ backend from before it) reads in full, which holds no etag.
- **Messages are not in the feed.** A group's `lastMessageSequence` tells the client there is something to read,
and it pages messages per group as today (4.6).
diff --git a/docs/plans/2026-11-04-derived-etag-plan.md b/docs/plans/2026-11-04-derived-etag-plan.md
new file mode 100644
index 000000000..af301ffc2
--- /dev/null
+++ b/docs/plans/2026-11-04-derived-etag-plan.md
@@ -0,0 +1,134 @@
+# Polls send the Swarm summary only when it changed: implementation plan
+
+This plan follows the project's way of working (writing-plans header, then test-driven tasks, implemented by the
+Engineer in this session; one review step on the pull request).
+
+**Goal:** A Swarm change read leaves out the machine and agent summary (`derived`) when it is the same as the one the
+reader already holds, so an unchanged poll costs a few hundred bytes at any Swarm size instead of 12 KB (36 machines)
+to 92 KB (360).
+
+**Architecture:** The backend hashes each reader's `derived` (without the reader's marks, which already go out as
+changes only) into a short `derivedEtag`, sent on every first load and change read. A client that holds one sends it
+back as `derivedEtag=` on its next change read; when it still matches, the reply's `derived` holds only the marks and
+says `derivedUnchanged: true`, and the client keeps the summary it holds. A client that sends nothing (every released
+one) gets the whole summary as today. This is the host feed's `held` etag (read/hostEtag.ts) applied to the app
+windows' reads.
+
+**Tech Stack:** Backend TypeScript (Fastify, zod, Prisma/MongoDB), vitest; Desktop/web and phone Flutter (Dart).
+
+**Spec:** Task #2906 (Swarm conversation d49673c3ec733ee886f9bca8); measurement by Wren13 for #2637, PR #2909
+(`backend/integration/bench/machineCapCost.ts`); read API design docs/design/2026-10-26-swarm-state-parts.md, section 4.
+
+## Global Constraints
+
+- Released clients keep working: a change read without `derivedEtag` answers exactly as today, plus the new top-level
+ `derivedEtag` field (unknown top-level fields are ignored by released Desktop, phone and CLI).
+- Unchanged polls carry no summary: only the reader's moved marks (`readMarks`, `unreadMarks`, `threadMarks`,
+ `chatSettings`) stay in `derived`.
+- Red-first tests; bytes per poll before/after at 36 and 360 machines in the pull request; one review step.
+- Swarm code stays out of cli/src/core. Desktop and phone `parts.dart` stay identical copies (mobile
+ test/swarm/desktop_parity_test.dart).
+- Desktop tests use a memory store and a fake sign-in; nothing signs the Mac out of the real CLI.
+
+## Design
+
+- Growth: nothing is stored. The etag lives in each client's held parts (and Desktop's saved view) and is replaced by
+ every read.
+- Concurrency: none added. The etag is computed from the reply the read already builds, inside the same read.
+- Read cost: no new database read. One SHA-256 over the reply's own summary JSON per read (under 1 ms at 360
+ machines); an unchanged poll sends the marks only.
+- Scale: at 10x (360 machines) an unchanged poll stays at the size of the parts that changed. The first metric to warn
+ is the reply size of a poll after a summary change, which still carries the whole summary (bench row "every machine
+ reading").
+- Compatibility: old client and new backend: no etag sent, whole summary, as today. New client and old backend: no etag
+ received, none sent. New client after a backend rollback: its held etag is refused by the old backend's strict query
+ (VALIDATION_ERROR); the client then drops what it holds and reads in full once, getting no etag.
+- Migration and rollback: none stored, so nothing to migrate. Rollback costs each open window one first load.
+- Audience: the etag hashes the summary of the reader it was built for, after the server's visibility filtering, and
+ is only compared with that reader's own echo. It reveals nothing beyond the summary the reader already received.
+- Identity: the read's actor, as today; no machine owner's login.
+- Failure: an etag mismatch or a malformed etag sends the whole summary. A reply saying "unchanged" for an etag the
+ client does not hold is a format error, which the client answers with a first load.
+- Platforms: backend; Desktop (macOS, Windows, Linux) and web share `desktop/lib/super_tabs`; phone (`mobile/lib/swarm`).
+ The CLI keeps reading as a released client (whole summary).
+
+## Review Focus
+
+1. A summary change of any kind, from any agent's `downSince` to an avatar link, after an unchanged poll: the next
+ poll must send the whole summary. Integration test "a summary change after an unchanged poll sends it whole".
+2. Marks moved in an otherwise unchanged poll: they still arrive. Integration test "an unchanged poll still sends the
+ reader's moved marks".
+3. Two readers of one Swarm see different summaries: one reader's etag never matches for another. Integration test
+ "an etag is the reader's own".
+4. A backend rolled back under a client holding an etag: the client reads in full, without the etag. Desktop and phone
+ tests "a refused etag reads in full once, then polls without it".
+5. A saved view restored on start: its held etag comes back with its summary (Desktop toJson/fromJson). Dart test
+ "a saved store keeps its etag".
+
+---
+
+### Task 1: The backend stamps the summary and leaves it out when the reader holds it
+
+**Files:**
+- Create: `backend/src/superTabs/read/derivedEtag.ts`, `backend/src/superTabs/read/derivedEtag.test.ts`
+- Modify: `backend/src/superTabs/read/hostEtag.ts` (export its sorted JSON), `backend/src/superTabs/read/contract.ts`,
+ `backend/src/superTabs/read/service.ts` (`firstLoad`, `changes`), `backend/src/routes/superTabs.ts` (`changesQuery`)
+- Test: `backend/src/superTabs/read/derivedEtag.test.ts`, `backend/src/routes/superTabs.test.ts`,
+ `backend/integration/superTabRead.test.ts`
+
+**Interfaces:**
+- Produces: `derivedEtag(derived: Derived): string` (22 base64url characters; marks fields ignored);
+ `MARK_FIELDS = ['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'] as const`;
+ `SwarmReads.changes(user, tabId, cursor, asAgentId?, options: { lightMachines?: boolean; derivedEtag?: string })`;
+ contract: `FirstLoad.derivedEtag: string`, `ChangesDelta.derivedEtag: string`, `ChangesDelta.derivedUnchanged?: true`,
+ `ChangesDelta.derived: Derived | ReaderMarks` where `ReaderMarks = Pick<Derived, MarkField>`;
+ route query `derivedEtag` (`/^[A-Za-z0-9_-]{1,64}$/`).
+
+- [ ] **Step 1: Failing tests.** `derivedEtag.test.ts`: equal for the same summary built in another key order; equal
+ whatever the marks; different when any value of `agents`, `machines`, `avatars` or a digest differs.
+ Route test: `?cursor=abc&derivedEtag=e1` passes `{ derivedEtag: 'e1' }`; `?cursor=abc` still passes `{}`;
+ `derivedEtag=bad!` is refused with 400. Integration (`changes since a cursor`): a first load's `derivedEtag` equals the
+ next change read's; echoed, the reply has `derivedUnchanged: true` and `derived` without `agents`/`machines`; no echo
+ or another etag sends the whole summary; a summary change after an unchanged poll sends it whole with a new etag; an
+ unchanged poll still sends moved read marks; an etag is the reader's own.
+- [ ] **Step 2: Run them, watch them fail.** `npx vitest run src/superTabs/read/derivedEtag.test.ts src/routes/superTabs.test.ts`
+ and `npx vitest run --config vitest.super-tabs.config.ts integration/superTabRead.test.ts -t "etag"`.
+- [ ] **Step 3: Implement** the interfaces above. `changes` builds `derived` as today, computes the etag, and replaces
+ `derived` with its marks alone when `options.derivedEtag` equals it.
+- [ ] **Step 4: Run the same tests, watch them pass;** `npx tsc --noEmit`.
+- [ ] **Step 5: Commit** the tests first, then the code.
+
+### Task 2: Desktop/web and phone send the etag back and keep the summary they hold
+
+**Files:**
+- Modify: `desktop/lib/super_tabs/parts.dart` and its copy `mobile/lib/swarm/parts.dart`;
+ `desktop/lib/super_tabs/controller.dart` (`_readParts`); `mobile/lib/swarm/swarm_reader.dart` (`_readParts`);
+ `desktop/test/support/super_tab_fixture.dart` (the fake backend answers etags as the backend does)
+- Test: `desktop/test/super_tab_parts_reading_test.dart`, `mobile/test/swarm/swarm_reader_test.dart`
+
+**Interfaces:**
+- Consumes: Task 1's `derivedEtag`, `derivedUnchanged` and `derivedEtag=` query.
+- Produces: `SuperTabParts.derivedEtag` (`String?`), `SuperTabParts.changesQuery` (`String`: `cursor=…` plus
+ `&derivedEtag=…` when one is held), kept by `toJson`/`fromJson`.
+
+- [ ] **Step 1: Failing tests.** Desktop: "an unchanged poll sends the etag and keeps the summary" (second change read's
+ path has `derivedEtag`, the fake answers without `agents`, `snapshot` still has its agents and a moved mark arrives);
+ "a summary change replaces it"; "a refused etag reads in full once, then polls without it"; "a saved store keeps its
+ etag". Phone: the same first and third.
+- [ ] **Step 2: Run them, watch them fail.** `flutter test test/super_tab_parts_reading_test.dart` (desktop),
+ `flutter test test/swarm/swarm_reader_test.dart test/swarm/desktop_parity_test.dart` (mobile).
+- [ ] **Step 3: Implement.** `apply` keeps `_derived` when `derivedUnchanged` is true and the reply's etag is the one
+ held (else a FormatException: half merged, first load); the readers poll with `changesQuery` and treat a
+ VALIDATION_ERROR on a read that sent an etag as they treat FORBIDDEN (first load).
+- [ ] **Step 4: Run the same tests, watch them pass;** `flutter analyze` on the changed files.
+- [ ] **Step 5: Commit** the tests first, then the code.
+
+### Task 3: Bytes per poll at 36 and 360 machines, before and after
+
+**Files:**
+- Create: `backend/integration/bench/derivedEtagCost.ts` (Wren13's machineCapCost.ts harness, with an etag window
+ beside the old one)
+
+- [ ] **Step 1:** Run the bench at 36 and 360 machines on main's behaviour (old windows) and with the etag window; the
+ table goes in the pull request: reply KB per poll after nothing, after every machine's reading, after one post.
+- [ ] **Step 2: Commit.**
diff --git a/mobile/lib/swarm/parts.dart b/mobile/lib/swarm/parts.dart
index 06a186b3f..acf823696 100644
--- a/mobile/lib/swarm/parts.dart
+++ b/mobile/lib/swarm/parts.dart
@@ -55,6 +55,7 @@ class SuperTabParts {
required this.epoch,
required this.lastChangedAt,
required this.githubApp,
+ this.derivedEtag,
});
/// A first load (section 4.3). Throws a [FormatException] for a response of another format.
@@ -76,6 +77,7 @@ class SuperTabParts {
// Null for a Swarm with no part changed yet (contract.ts `ReadMeta`).
lastChangedAt: _int(answer['lastChangedAt'] ?? 0),
githubApp: answer['githubApp'] == true,
+ derivedEtag: _etag(answer),
);
parts._seen['head'] = _int(parts._head['version']);
for (final kind in _lists) {
@@ -131,6 +133,26 @@ class SuperTabParts {
/// Whether the backend has the GitHub app, as the first load says.
final bool githubApp;
+ /// The etag of the summary held (`derived` without the marks, backend read/derivedEtag.ts, #2906), as the last read
+ /// sent it: sent back with the cursor, so a change read leaves the summary out while it still matches. Null from a
+ /// backend without it, and then none is sent.
+ String? derivedEtag;
+
+ /// What a change read asks after `?`: the cursor, and the summary's etag when one is held.
+ String get changesQuery {
+ final etag = derivedEtag;
+ final query = 'cursor=${Uri.encodeQueryComponent(cursor)}';
+ return etag == null
+ ? query
+ : '$query&derivedEtag=${Uri.encodeQueryComponent(etag)}';
+ }
+
+ static String? _etag(Map<String, dynamic> answer) =>
+ switch (answer['derivedEtag']) {
+ final String etag when etag.isNotEmpty => etag,
+ _ => null,
+ };
+
Map<String, dynamic> _head;
Map<String, dynamic> _derived;
final _live = <String, Map<String, Map<String, dynamic>>>{};
@@ -269,6 +291,14 @@ class SuperTabParts {
/// Merges one change read (section 4.4): a higher version wins, a tombstone removes the part, a dormant stub moves
/// the group out of the active ones, `derived` is replaced, and the cursor moves. A reload answer is the caller's.
SuperTabPartsChange apply(Map<String, dynamic> changes) {
+ // Without the summary, a reply is only right for the one held: checked before anything is merged.
+ final unchanged = changes['derivedUnchanged'] == true;
+ final etag = _etag(changes);
+ if (unchanged && (etag == null || etag != derivedEtag)) {
+ throw const FormatException(
+ 'A Swarm read left out a summary this reader does not hold.',
+ );
+ }
var changed = false;
// Anything but a machine's own report moved.
var beyondMachines = false;
@@ -333,11 +363,14 @@ class SuperTabParts {
_takeThreadMarks(derived.remove('threadMarks')) |
_takeChatSettings(derived.remove('chatSettings')) |
readMarksMoved;
- if (!const DeepCollectionEquality().equals(derived, _derived)) {
+ // Unchanged: `derived` held only the marks, and the summary held stands.
+ if (!unchanged &&
+ !const DeepCollectionEquality().equals(derived, _derived)) {
if (!_sameBeyondMachines(derived, _derived)) beyondMachines = true;
_derived = derived;
changed = true;
}
+ derivedEtag = etag;
cursor = _cursor(changes);
epoch = _int(changes['epoch'] ?? epoch);
lastChangedAt = _int(changes['lastChangedAt'] ?? lastChangedAt);
@@ -494,6 +527,7 @@ class SuperTabParts {
for (final kind in _lists)
kind: (_live[kind] ?? const {}).values.toList(),
'derived': _derived,
+ if (derivedEtag != null) 'derivedEtag': derivedEtag,
},
'dormant': _dormant.values.toList(),
'seen': Map<String, int>.of(_seen),
diff --git a/mobile/lib/swarm/swarm_reader.dart b/mobile/lib/swarm/swarm_reader.dart
index febe3b2f4..8804911ef 100644
--- a/mobile/lib/swarm/swarm_reader.dart
+++ b/mobile/lib/swarm/swarm_reader.dart
@@ -267,9 +267,7 @@ class SwarmReader extends ChangeNotifier {
final held = _parts;
if (held != null) {
try {
- final answer = await _get(
- '$_path/changes?cursor=${Uri.encodeQueryComponent(held.cursor)}',
- );
+ final answer = await _get('$_path/changes?${held.changesQuery}');
if (answer['reload'] != true) {
final SuperTabPartsChange change;
try {
@@ -283,7 +281,14 @@ class SwarmReader extends ChangeNotifier {
return (changed: change.changed || stale, view: null);
}
} on SuperTabFailure catch (failure) {
- if (!const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) rethrow;
+ // A backend rolled back to one without the summary etag refuses it
+ // (#2906): read in full, which holds none.
+ final etagRefused =
+ failure.code == 'VALIDATION_ERROR' && held.derivedEtag != null;
+ if (!etagRefused &&
+ !const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) {
+ rethrow;
+ }
}
_parts = null;
}
diff --git a/mobile/test/swarm/swarm_reader_test.dart b/mobile/test/swarm/swarm_reader_test.dart
index 51a48812f..6fa27631d 100644
--- a/mobile/test/swarm/swarm_reader_test.dart
+++ b/mobile/test/swarm/swarm_reader_test.dart
@@ -67,6 +67,69 @@ void main() {
expect(swarm.paths.last, '/api/super-tabs/$tab/changes?cursor=c2');
});
+ // #2906: the machine and agent summary is sent again only when it changed.
+ test(
+ 'an unchanged poll echoes the summary etag and keeps the summary it holds',
+ () async {
+ final swarm = FakeSwarm();
+ swarm.routes['/api/super-tabs/$tab/parts'] = (_) => {
+ ...firstLoad(cursor: 'c1', derived: {'mentionsWaiting': 2}),
+ 'derivedEtag': 'e1',
+ };
+ swarm.routes['/api/super-tabs/$tab/changes'] = (_) => {
+ ...changes(cursor: 'c2'),
+ 'derivedEtag': 'e1',
+ 'derivedUnchanged': true,
+ };
+ final r = reader(swarm);
+ addTearDown(r.dispose);
+
+ await r.refresh();
+ await r.refresh();
+
+ expect(
+ swarm.paths.last,
+ '/api/super-tabs/$tab/changes?cursor=c1&derivedEtag=e1',
+ );
+ expect(r.snapshot!.mentionsWaiting, 2);
+ expect(r.error, isNull);
+ },
+ );
+
+ test('a refused etag (a backend rolled back) reads in full once, then polls without it', () async {
+ var loads = 0;
+ final swarm = FakeSwarm();
+ swarm.routes['/api/super-tabs/$tab/parts'] = (_) => {
+ ...firstLoad(cursor: 'c${++loads}'),
+ if (loads == 1) 'derivedEtag': 'e1',
+ };
+ swarm.routes['/api/super-tabs/$tab/changes'] = (_) {
+ if (swarm.paths.last.contains('derivedEtag')) {
+ throw SuperTabFailure(
+ 'VALIDATION_ERROR',
+ "Unrecognized key(s) in object: 'derivedEtag'",
+ statusCode: 400,
+ );
+ }
+ return changes(cursor: 'c9');
+ };
+ final r = reader(swarm);
+ addTearDown(r.dispose);
+
+ await r.refresh();
+ await r.refresh();
+ await r.refresh();
+
+ expect(swarm.paths, [
+ '/api/super-tabs/$tab/parts',
+ '/api/super-tabs/$tab/changes?cursor=c1&derivedEtag=e1',
+ '/api/super-tabs/$tab/parts',
+ '/api/super-tabs/$tab/changes?cursor=c2',
+ ]);
+ expect(r.error, isNull);
+ expect(r.accessRevoked, isFalse);
+ });
+
test('a reload answer starts again from a first load', () async {
var loads = 0;
final swarm = FakeSwarm();