The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm polls send the machine and agent summary only when it changed (#3011)
harness-for-autonomous[bot]2b164811e255
diff --git a/backend/integration/bench/derivedEtagCost.ts b/backend/integration/bench/derivedEtagCost.tsnew file mode 100644index 000000000..d921819be--- /dev/null+++ b/backend/integration/bench/derivedEtagCost.ts@@ -0,0 +1,126 @@+import { setTimeout as sleep } from 'node:timers/promises'+import type { AuthUser } from '../../src/lib/ssoAuth.js'+import type { Changes } from '../../src/superTabs/read/contract.js'+import { readDelta } from '../../src/superTabs/cachedReads.js'+import { listGroupIndexes } from '../../src/superTabs/parts/indexed.js'+import { parseArgs } from './config.js'+import { partsDriver } from './drivers/parts.js'+import type { Reader, Reply } from './driver.js'+import { machineId, seedAccounts } from './fixtures.js'+import { storeRoutes } from './legacyDriver.js'+import { localDatabase } from './mongo.js'+import { plan } from './plan.js'+import { seedSwarm } from './seed.js'+import { benchStamps } from './stamps.js'++/** Past the change reads' 30 s overlap, so the polls after it resend nothing they sent before. */+const QUIET_MS = 35_000+/** Windows of each kind, the same at every size, so a window's cost compares across sizes. */+const READERS = 10++/**+ * What an app window's change poll costs with and without the summary etag (task #2906), on Wren13's machine-count+ * harness (machineCapCost.ts, PR #2909): a Swarm of `--machines` machines, each its own person's, and READERS windows of+ * each kind polling changes. `old` sends no etag, as every released client; `etag` echoes the `derivedEtag` of its last+ * read, as this release's Desktop, web and phone do. Rounds as there: after nothing, after every machine's reading, after+ * one post. Per round, reply KB per poll, the `derived` KB in it, and how many replies left the summary out. From+ * `backend/`:+ *+ * NODE_ENV=test HARNESS_CREDENTIAL_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA= \+ * DATABASE_URL=mongodb://127.0.0.1:1/bench-never-used npx tsx integration/bench/derivedEtagCost.ts --machines 36 [--rounds 3]+ */+async function main() {+ const argv = process.argv.slice(2)+ const at = argv.indexOf('--rounds'), rounds = at < 0 ? 3 : Number(argv[at + 1])+ const { config } = parseArgs(argv.filter((arg, i) => arg !== '--rounds' && argv[i - 1] !== '--rounds'))+ config.desktopReaders = Math.min(config.desktopReaders, config.machines)+ config.hostReaders = Math.min(config.hostReaders, config.machines)+ const database = await localDatabase()+ await listGroupIndexes(database.db)+ const stamps = await benchStamps('memory', readDelta(database.db))+ let clock = Date.now()+ try {+ const people = await seedAccounts(database.db, config.machines)+ const driver = await partsDriver(database.db, () => people, { now: () => clock })+ const { call, close } = await storeRoutes(database.db, () => people)+ const { swarm } = await seedSwarm(driver, config, plan(config), people)+ const derivedBytes = new Map<Reader, number>(), unchanged = new Map<Reader, boolean>()+ const window = (user: AuthUser, echoes: boolean): Reader => {+ let cursor: string | null = null, etag: string | null = null+ const reader: Reader = { label: echoes ? 'etag' : 'old', async poll() {+ const reply = cursor === null+ ? await call<{ cursor: string; derivedEtag?: string; derived: unknown }>(user, 'GET', `/api/super-tabs/${swarm.tabId}/parts`)+ : await call<Changes>(user, 'GET', `/api/super-tabs/${swarm.tabId}/changes?cursor=${cursor}${echoes && etag ? `&derivedEtag=${etag}` : ''}`)+ const data = reply.data as { reload?: boolean; cursor: string; derivedEtag?: string; derived: unknown; derivedUnchanged?: true } | null+ if (data?.reload) throw new Error('A window was told to reload')+ if (data) {+ cursor = data.cursor+ etag = data.derivedEtag ?? null+ derivedBytes.set(reader, JSON.stringify(data.derived ?? null).length)+ unchanged.set(reader, data.derivedUnchanged === true)+ }+ return reply+ } }+ return reader+ }+ const kinds: Record<string, Reader[]> = {+ old: Array.from({ length: READERS }, (_, i) => window(people[i % people.length]!, false)),+ etag: Array.from({ length: READERS }, (_, i) => window(people[i % people.length]!, true)),+ }+ const all = Object.values(kinds).flat()+ let n = 0+ const reading = (machine: number): Promise<unknown> => driver.observeMachine(swarm.tabId, people[machine]!, machineId(machine), {+ generation: swarm.generations[machine]!, quotas: [], tools: {}, runningHarnessIds: machine === 0 ? ['bench-lead'] : [], stoppedHarnessIds: [],+ resources: { cpuPercent: 10 + (n++ % 50), memoryTotalBytes: 32e9, memoryUsedBytes: 4e9, diskFreeBytes: 120e9, diskTotalBytes: 500e9,+ projectDiskFreeBytes: 120e9, deviceType: 'Laptop', chip: 'Bench', os: 'macOS', gpuPercent: null } })+ const post = () => driver.execute(swarm.tabId, people[0]!, `etag-cost-${n++}`,+ { type: 'message.send', conversationId: swarm.workers[0]!.group, text: 'One post.', replyTo: null, recipientIds: [] })+ for (const r of all) await r.poll()+ // A first reading after the seed changes more than the reading: done before the quiet wait, as machineCapCost.ts does.+ await Promise.all(people.map((_, m) => reading(m)))+ await sleep(QUIET_MS)+ clock = Date.now()+ for (const r of all) await r.poll()++ process.stdout.write(`${config.machines} machines (${people.length} people), ${config.groups} groups; ${READERS} windows of each kind\n\n`)+ process.stdout.write('| Before each round | Window | Polls | Reply KB/poll | `derived` KB/poll | Summary left out |\n|---|---|---|---|---|---|\n')+ const phases: Array<[string, () => Promise<unknown>]> = [['nothing', async () => {}],+ ['every machine reading', () => Promise.all(people.map((_, m) => reading(m)))], ['one post', post]]+ for (const [label, before] of phases) {+ const sums = Object.fromEntries(Object.keys(kinds).map(k => [k, { polls: 0, reply: 0, derived: 0, unchanged: 0 }]))+ for (let r = 0; r < rounds; r++) {+ await before()+ await sleep(200)+ for (const [kind, readers] of Object.entries(kinds)) {+ const replies = await Promise.all(readers.map(p => p.poll()))+ check(kind, replies)+ const s = sums[kind]!+ s.polls += readers.length+ s.reply += replies.reduce((sum, reply) => sum + reply.bytes, 0)+ s.derived += readers.reduce((sum, p) => sum + (derivedBytes.get(p) ?? 0), 0)+ s.unchanged += readers.filter(p => unchanged.get(p)).length+ }+ await sleep(3_000)+ }+ for (const [kind, s] of Object.entries(sums)) {+ process.stdout.write(`| ${label} | ${kind} | ${s.polls} | ${(s.reply / s.polls / 1024).toFixed(2)} | ${(s.derived / s.polls / 1024).toFixed(2)} | ` ++ `${s.unchanged} of ${s.polls} |\n`)+ }+ }+ await close()+ await driver.close()+ } finally {+ await stamps.close()+ await database.stop()+ }+}++function check(kind: string, replies: Reply[]) {+ const failed = replies.find(reply => reply.code)+ if (failed) throw new Error(`${kind}: ${failed.code}`)+}++main().catch(error => {+ process.stderr.write(`${error instanceof Error ? error.stack : String(error)}\n`)+ process.exitCode = 1+})diff --git a/backend/integration/superTabRead.test.ts b/backend/integration/superTabRead.test.tsindex 7680589cc..7b2aa6bf2 100644--- a/backend/integration/superTabRead.test.ts+++ b/backend/integration/superTabRead.test.ts@@ -127,7 +127,7 @@ describe('first load (design 4.3)', () => {clock = NOWconst { status, body } = await get(`/api/super-tabs/${s.tabId}/parts`, peer)expect(status).toBe(200)- expect(Object.keys(body.data).sort()).toEqual(['agents', 'cursor', 'derived', 'epoch', 'format', 'groups', 'head', 'lastChangedAt',+ expect(Object.keys(body.data).sort()).toEqual(['agents', 'cursor', 'derived', 'derivedEtag', 'epoch', 'format', 'groups', 'head', 'lastChangedAt','machines', 'members', 'progress', 'serverTime', 'starts', 'steps', 'tabId'].sort())expect(body.data).toMatchObject({ format: 2, tabId: s.tabId, serverTime: NOW, epoch: 1, lastChangedAt: MIGRATED, steps: true,head: { key: 'head', version: R, changedAt: MIGRATED, data: { id: s.tabId, name: 'Pilot' } } })@@ -1240,6 +1240,74 @@ describe('decisions asked of one person (v2)', () => {})})+// Task #2906: an unchanged poll leaves out the machine and agent summary the reader already holds (12 KB a poll at 36+// machines, 92 KB at 360). A released client, which echoes no etag, still gets it whole.+describe('the summary only when it changed (derivedEtag)', () => {+ const read = async (url: string, as: AuthUser) => { clock = Date.now(); return (await get(url, as)).body.data }+ const poll = (tabId: string, cursor: string, as: AuthUser, etag?: string) =>+ read(`/api/super-tabs/${tabId}/changes?cursor=${cursor}${etag ? `&derivedEtag=${etag}` : ''}`, as)+ /** An open decision asked of peer, written as a row (as `askedBefore` does): a command would leave its notices and wakes+ * running into the next tests, whose database-byte counts (group 3b) then count them. */+ const ask = async (tabId: string, conversationId: string, lead: string) => {+ await db.superTabDecision.create({ data: { id: `etag-${randomUUID()}`, tabId, batchId: `${conversationId}-etag`, projectId: 'lead',+ conversationId, assignmentId: null, personId: peer.sub, askerAgentId: lead, question: 'Ship how?', options: ['Behind a flag', 'To everyone'],+ recommended: 0, unblocks: 'The release', createdAt: new Date() } })+ }++ it('stamps a first load and a change read alike, and an echoed etag gets no summary', async () => {+ const s = await scene()+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)+ expect(first.derivedEtag).toMatch(/^[A-Za-z0-9_-]{22}$/)+ const unchanged = await poll(s.tabId, first.cursor, peer, first.derivedEtag)+ expect(unchanged).toMatchObject({ reload: false, derivedEtag: first.derivedEtag, derivedUnchanged: true })+ expect(Object.keys(unchanged.derived).filter(k => !['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'].includes(k))).toEqual([])+ })++ it('sends the whole summary to a client that echoes none, or another etag', async () => {+ const s = await scene()+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)+ for (const etag of [undefined, 'not-the-etag']) {+ const reply = await poll(s.tabId, first.cursor, peer, etag)+ expect(reply.derivedUnchanged, String(etag)).toBeUndefined()+ expect(reply.derivedEtag, String(etag)).toBe(first.derivedEtag)+ expect(reply.derived, String(etag)).toEqual(first.derived)+ }+ })++ it('a summary change after an unchanged poll sends it whole, with a new etag', async () => {+ const s = await scene()+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)+ expect(first.derived.decisionsWaiting).toBe(0)+ const quiet = await poll(s.tabId, first.cursor, peer, first.derivedEtag)+ expect(quiet.derivedUnchanged).toBe(true)+ await ask(s.tabId, s.web, s.lead)+ const moved = await poll(s.tabId, quiet.cursor, peer, quiet.derivedEtag)+ expect(moved.derivedUnchanged).toBeUndefined()+ expect(moved.derivedEtag).not.toBe(first.derivedEtag)+ expect(moved.derived).toMatchObject({ decisionsWaiting: 1, agents: first.derived.agents })+ expect((await poll(s.tabId, moved.cursor, peer, moved.derivedEtag)).derivedUnchanged).toBe(true)+ })++ it('an unchanged poll still sends the reader\'s moved marks', async () => {+ const s = await scene()+ const first = await read(`/api/super-tabs/${s.tabId}/parts`, peer)+ await writeReadMark(db, { tabId: s.tabId, personId: peer.sub, conversationId: s.web, sequence: 3, now: Date.now() })+ const reply = await poll(s.tabId, first.cursor, peer, first.derivedEtag)+ expect(reply.derivedUnchanged).toBe(true)+ expect(reply.derived).toEqual({ readMarks: { [s.web]: 3 } })+ })++ it('an etag is the reader\'s own: another reader\'s summary never matches it', async () => {+ const s = await scene()+ await ask(s.tabId, s.web, s.lead)+ const [mine, theirs] = [await read(`/api/super-tabs/${s.tabId}/parts`, peer), await read(`/api/super-tabs/${s.tabId}/parts`, owner)]+ expect(mine.derivedEtag).not.toBe(theirs.derivedEtag)+ const reply = await poll(s.tabId, mine.cursor, peer, theirs.derivedEtag)+ expect(reply.derivedUnchanged).toBeUndefined()+ expect(reply.derived).toMatchObject({ decisionsWaiting: 1 })+ })+})+describe('routes given only a store', () => {it('read from that store\'s database, not the app-wide client', async () => {const s = await scene()diff --git a/backend/src/routes/superTabs.test.ts b/backend/src/routes/superTabs.test.tsindex 1fee25446..7256a9809 100644--- a/backend/src/routes/superTabs.test.ts+++ b/backend/src/routes/superTabs.test.ts@@ -334,6 +334,23 @@ describe('the machine lines of a change read', () => {expect(changes).not.toHaveBeenCalled()})+ // Task #2906: a client that holds the summary echoes its etag; a released client sends none and is asked nothing new.+ it('passes the summary etag a client echoes, with lines or without', async () => {+ expect((await get('/api/super-tabs/tab1/changes?cursor=abc&derivedEtag=e1_-Z')).statusCode).toBe(200)+ expect(changes.mock.calls[0]!.slice(2)).toEqual(['abc', undefined, { derivedEtag: 'e1_-Z' }])+ expect((await get('/api/super-tabs/tab1/changes?cursor=abc&machines=light&derivedEtag=e2')).statusCode).toBe(200)+ expect(changes.mock.calls[1]!.slice(2)).toEqual(['abc', undefined, { lightMachines: true, derivedEtag: 'e2' }])+ })++ it('refuses a malformed summary etag', async () => {+ for (const etag of ['bad!', '', 'x'.repeat(65)]) {+ const res = await get(`/api/super-tabs/tab1/changes?cursor=abc&derivedEtag=${etag}`)+ expect(res.statusCode, etag).toBe(400)+ expect(res.json().error.code).toBe('VALIDATION_ERROR')+ }+ expect(changes).not.toHaveBeenCalled()+ })+it('serves the whole readings on demand, for an agent too', async () => {expect((await get('/api/super-tabs/tab1/machine-readings?asAgentId=agent1')).statusCode).toBe(200)expect(machineReadings.mock.calls[0]!.slice(1)).toEqual(['tab1', 'agent1'])diff --git a/backend/src/routes/superTabs.ts b/backend/src/routes/superTabs.tsindex fd79a3c43..3592f2caf 100644--- a/backend/src/routes/superTabs.ts+++ b/backend/src/routes/superTabs.ts@@ -81,7 +81,9 @@ const workspaceJoin = z.object({ operationId: id }).strict()const personOnly = (user: AuthUser) => { if (isTeamSession(user)) fail(TEAM_SESSION_LIMITED, TEAM_SESSION_MESSAGE, 403) }const listQuery = z.object({ after: id.optional(), mentions: z.literal('1').optional() }).strict()const cursor = z.string().min(1).max(512).regex(/^[A-Za-z0-9_-]+$/)-const changesQuery = agentQuery.extend({ cursor, machines: z.literal('light').optional() })+// `derivedEtag`: the summary's etag the client holds (read/derivedEtag.ts, task #2906).+const changesQuery = agentQuery.extend({ cursor, machines: z.literal('light').optional(),+ derivedEtag: z.string().min(1).max(64).regex(/^[A-Za-z0-9_-]+$/).optional() })const costsQuery = z.object({ days: z.coerce.number().int().min(1).max(MAX_DAYS).default(DEFAULT_DAYS), asAgentId: id.optional() }).strict()const leaderboardQuery = z.object({ projectId: id, week: z.coerce.number().int().nonnegative().safe().optional(), period: z.enum(PERIODS).optional(), scope: z.literal('all').optional() }).strict().refine(query => query.week === undefined || query.period === undefined, 'Ask for a week or a period, not both.')@@ -521,7 +523,8 @@ export async function superTabRoutes(app: FastifyInstance, options: { store?: Suapp.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof changesQuery> }>('/api/super-tabs/:tabId/changes',{ preHandler: [validateParams(tabParams), validateQuery(changesQuery)] },(req, reply) => respond(reply, noting(req, () => reads.changes(req.user!, req.params.tabId, req.query.cursor, req.query.asAgentId,- req.query.machines === 'light' ? { lightMachines: true } : {}), nothingChanged)))+ { ...(req.query.machines === 'light' ? { lightMachines: true } : {}), ...(req.query.derivedEtag ? { derivedEtag: req.query.derivedEtag } : {}) }),+ nothingChanged)))// The whole readings on demand, for a client that takes lines in its changes (read/changes.ts).app.get<{ Params: z.infer<typeof tabParams>; Querystring: z.infer<typeof agentQuery> }>('/api/super-tabs/:tabId/machine-readings',{ preHandler: [validateParams(tabParams), validateQuery(agentQuery)] },diff --git a/backend/src/superTabs/read/contract.ts b/backend/src/superTabs/read/contract.tsindex 755fa1d29..226ca0ae8 100644--- a/backend/src/superTabs/read/contract.ts+++ b/backend/src/superTabs/read/contract.ts@@ -153,9 +153,15 @@ export interface FirstLoad extends ReadMeta {starts: LivePart<StartView>[]progress: LivePart<ProgressView>[]derived: Derived+ /** Opaque: the reader's summary (`derived` without its marks) as a hash (derivedEtag.ts). Send it back as `derivedEtag` on+ * the next change read. Absent from an older backend. */+ derivedEtag: string}-/** GET /api/super-tabs/:tabId/changes?cursor=[&asAgentId=] */+/** The reader's own marks of `derived`, which a change read sends only as they move. */+export type ReaderMarks = Pick<Derived, 'readMarks' | 'unreadMarks' | 'threadMarks' | 'chatSettings'>++/** GET /api/super-tabs/:tabId/changes?cursor=[&asAgentId=][&machines=light][&derivedEtag=] */export type Changes = ChangesReload | ChangesDelta/** Drop the store and do a first load: bad or old cursor, another epoch, or older than tombstones are kept. */export interface ChangesReload { reload: true }@@ -175,7 +181,13 @@ export interface ChangesDelta extends ReadMeta {machineReadings?: MachineReading[]starts: PartChange<StartView>[]progress: PartChange<ProgressView>[]- derived: Derived+ /** The whole summary, or with `derivedUnchanged` only the reader's moved marks: the client keeps the summary it holds. */+ derived: Derived | ReaderMarks+ /** As a first load's: the summary this reply's reader has now, sent whether or not `derived` carries it. */+ derivedEtag: string+ /** Only when the client sent the `derivedEtag` its summary still hashes to (task #2906): `derived` then holds only the+ * reader's moved marks. A client that sends no etag, as every released one, always gets the whole summary. */+ derivedUnchanged?: true}/** GET /api/super-tabs/:tabId/groups?dormant=1[&projectId=][&before=][&asAgentId=]: newest activity first, 50 a page. */diff --git a/backend/src/superTabs/read/derivedEtag.test.ts b/backend/src/superTabs/read/derivedEtag.test.tsnew file mode 100644index 000000000..27983e6cb--- /dev/null+++ b/backend/src/superTabs/read/derivedEtag.test.ts@@ -0,0 +1,48 @@+import { describe, expect, it } from 'vitest'+import type { Derived } from './contract.js'+import { derivedEtag, MARK_FIELDS } from './derivedEtag.js'++const summary = (): Derived => ({+ agents: { a1: { downSince: null, limitedUntil: null, workDecision: null, roleId: null, roleName: 'Engineer' } },+ machines: { m1: { availableSlots: 2, teamWork: { allowed: true } } },+ mentionsWaiting: 1, mentionsDigest: 'd1', avatars: { 'person:p1': 'https://example.test/a?sig=1' },+})++// Task #2906: the etag a client echoes so an unchanged poll leaves the summary out.+describe('derivedEtag', () => {+ it('is short and URL-safe', () => {+ expect(derivedEtag(summary())).toMatch(/^[A-Za-z0-9_-]{22}$/)+ })++ it('is the same for the same summary built in another key order', () => {+ const s = summary()+ const reordered: Derived = { avatars: s.avatars, mentionsDigest: s.mentionsDigest, mentionsWaiting: s.mentionsWaiting,+ machines: { m1: { teamWork: { allowed: true }, availableSlots: 2 } }, agents: s.agents }+ expect(derivedEtag(reordered)).toBe(derivedEtag(s))+ })++ it('leaves out the reader\'s marks, which a change read sends only as they move', () => {+ expect(MARK_FIELDS).toEqual(['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'])+ const marked: Derived = { ...summary(), readMarks: { g1: 4 }, unreadMarks: { g1: 3 }, threadMarks: { g1: { r1: 2 } }, chatSettings: { g1: 'all' } }+ expect(derivedEtag(marked)).toBe(derivedEtag(summary()))+ })++ it('moves with any other value', () => {+ const base = derivedEtag(summary())+ const changes: Array<(d: Derived) => void> = [+ d => { d.agents.a1!.downSince = '2026-11-04T00:00:00.000Z' },+ d => { d.agents.a1!.roleName = 'Reviewer' },+ d => { d.machines.m1!.availableSlots = 1 },+ d => { d.machines.m1!.teamWork = { allowed: false, reason: 'full' } },+ d => { d.avatars = { 'person:p1': 'https://example.test/a?sig=2' } },+ d => { d.mentionsWaiting = 2 },+ d => { d.decisionsDigest = 'x' },+ d => { delete d.avatars },+ ]+ for (const change of changes) {+ const d = summary()+ change(d)+ expect(derivedEtag(d), change.toString()).not.toBe(base)+ }+ })+})diff --git a/backend/src/superTabs/read/derivedEtag.ts b/backend/src/superTabs/read/derivedEtag.tsnew file mode 100644index 000000000..6a6f225b7--- /dev/null+++ b/backend/src/superTabs/read/derivedEtag.ts@@ -0,0 +1,23 @@+import { createHash } from 'node:crypto'+import type { Derived } from './contract.js'+import { sorted } from './hostEtag.js'++/** The reader's own marks: a change read sends only those moved since its cursor, so they are no part of the summary. */+export const MARK_FIELDS = ['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'] as const+export type MarkField = typeof MARK_FIELDS[number]++/**+ * A reader's summary (`derived` without its marks) as a short hash (task #2906): equal summaries, equal etag, whatever+ * order their records were built in. A client echoes it as `derivedEtag` on its next change read, and while it still+ * matches the reply leaves the summary out (read/service.ts changes), as a host's `held` etag does (hostEtag.ts).+ */+export function derivedEtag(derived: Derived): string {+ const summary: Record<string, unknown> = { ...derived }+ for (const field of MARK_FIELDS) delete summary[field]+ return createHash('sha256').update(sorted(summary)).digest('base64url').slice(0, 22)+}++/** Only the reader's marks of `derived`, the fields it has. */+export function marksOf(derived: Derived): Pick<Derived, MarkField> {+ return Object.fromEntries(MARK_FIELDS.filter(field => derived[field] !== undefined).map(field => [field, derived[field]]))+}diff --git a/backend/src/superTabs/read/hostEtag.ts b/backend/src/superTabs/read/hostEtag.tsindex c6fc32a2a..3869025f9 100644--- a/backend/src/superTabs/read/hostEtag.ts+++ b/backend/src/superTabs/read/hostEtag.ts@@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'import type { HostTabRows } from './contract.js'/** JSON with sorted keys, so equal rows hash equal whatever order their records were built in. */-const sorted = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === 'object' && !Array.isArray(item)+export const sorted = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === 'object' && !Array.isArray(item)? Object.fromEntries(Object.entries(item as Record<string, unknown>).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item)/**diff --git a/backend/src/superTabs/read/service.ts b/backend/src/superTabs/read/service.tsindex 21364a3ca..e9103e6cd 100644--- a/backend/src/superTabs/read/service.ts+++ b/backend/src/superTabs/read/service.ts@@ -33,7 +33,8 @@ import { LEDGER_WEEKS } from '../leaderboard/record.js'import type { PeriodName } from '../leaderboard/rolling.js'import { leaderboard } from '../leaderboard/stats.js'import { WEEK_MS } from '../leaderboard/week.js'-import { READ_FORMAT, SWARM_MIGRATING, type Changes, type Derived, type MachineReadings, type DormantGroups, type FirstLoad, type OneGroup, type SwarmList } from './contract.js'+import { derivedEtag, marksOf } from './derivedEtag.js'+import { READ_FORMAT, SWARM_MIGRATING, type Changes, type Derived, type ReaderMarks, type MachineReadings, type DormantGroups, type FirstLoad, type OneGroup, type SwarmList } from './contract.js'import { AvatarLinks } from '../avatarLinks.js'import { cursorFor, readCursor } from './cursor.js'import { changesFor } from './changes.js'@@ -73,6 +74,14 @@ function readableMarks(read: Read, marks: ReadMarkRow[], threads: Map<string, Ma...(Object.keys(threadMarks).length ? { threadMarks } : {}), ...(Object.keys(chatSettings).length ? { chatSettings } : {}) }}+/** A reply's `derived` with its etag, or only the reader's marks when `held` is still the etag of the summary (task #2906). */+function stamped(derived: Derived): { derived: Derived; derivedEtag: string }+function stamped(derived: Derived, held: string | undefined): { derived: Derived | ReaderMarks; derivedEtag: string; derivedUnchanged?: true }+function stamped(derived: Derived, held?: string) {+ const etag = derivedEtag(derived)+ return held === etag ? { derived: marksOf(derived), derivedEtag: etag, derivedUnchanged: true as const } : { derived, derivedEtag: etag }+}+/** A change poll's light base (parts/light.ts): the same for every reader of a Swarm, so one load per stamp serves them all. */const LIGHT = { groups: 'working' } as const@@ -114,8 +123,8 @@ export class SwarmReads {return { format: READ_FORMAT, tabId, cursor: cursorFor(t0, loaded.visibilityEpoch), serverTime: t0, lastChangedAt, epoch: loaded.visibilityEpoch,// This backend opens steps (task steps design): a newer CLI sends `step` only to a backend that says so....(this.options.githubApp ? { githubApp: true as const } : {}), steps: true as const, ...view.parts,- derived: { ...await this.withAvatars(view, t0), ...readableMarks(read, await this.marksOf(tx, tabId, read.actor),- await this.threadMarksOf(tx, tabId, read.actor), await this.settingsOf(tx, tabId, read.actor)) } }+ ...stamped({ ...await this.withAvatars(view, t0), ...readableMarks(read, await this.marksOf(tx, tabId, read.actor),+ await this.threadMarksOf(tx, tabId, read.actor), await this.settingsOf(tx, tabId, read.actor)) }) }})}@@ -163,7 +172,10 @@ export class SwarmReads {})}- async changes(user: AuthUser, tabId: string, cursor: string, asAgentId?: string, options: { lightMachines?: boolean } = {}): Promise<Changes> {+ /** `derivedEtag`: the etag of the summary the client holds (derivedEtag.ts); while it still matches, the reply leaves the+ * summary out and sends only the reader's moved marks. */+ async changes(user: AuthUser, tabId: string, cursor: string, asAgentId?: string,+ options: { lightMachines?: boolean; derivedEtag?: string } = {}): Promise<Changes> {const t0 = this.now()return this.read(user, tabId, async (tx, cached) => {// Light (parts/light.ts): changed parts come whole from the change feed; the rest is read only to check visibility and@@ -197,7 +209,7 @@ export class SwarmReads {if (!asAgentId) sent.groups = await this.servedTops.served(tabId, read.view.state, sent.groups)return { format: READ_FORMAT, reload: false, cursor: cursorFor(t0, read.parts.visibilityEpoch), serverTime: t0, lastChangedAt: read.lastChangedAt,epoch: read.parts.visibilityEpoch, ...sent,- derived: { ...await this.withAvatars(read.view, t0), ...readableMarks(read, marks, threadMarks, settings) } }+ ...stamped({ ...await this.withAvatars(read.view, t0), ...readableMarks(read, marks, threadMarks, settings) }, options.derivedEtag) }})}diff --git a/desktop/lib/super_tabs/controller.dart b/desktop/lib/super_tabs/controller.dartindex 4f9795826..830935435 100644--- a/desktop/lib/super_tabs/controller.dart+++ b/desktop/lib/super_tabs/controller.dart@@ -4038,7 +4038,7 @@ class SuperTabController extends ChangeNotifier {try {final answer = await request('GET',- '$_path/changes?cursor=${Uri.encodeQueryComponent(held.cursor)}',+ '$_path/changes?${held.changesQuery}',);if (answer['reload'] != true) {final SuperTabPartsChange change;@@ -4059,7 +4059,14 @@ class SuperTabController extends ChangeNotifier {);}} on SuperTabFailure catch (failure) {- if (!const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) rethrow;+ // A backend rolled back to one without the summary etag refuses it+ // (#2906): read in full, which holds none.+ final etagRefused =+ failure.code == 'VALIDATION_ERROR' && held.derivedEtag != null;+ if (!etagRefused &&+ !const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) {+ rethrow;+ }}_parts = null;}diff --git a/desktop/lib/super_tabs/parts.dart b/desktop/lib/super_tabs/parts.dartindex 06a186b3f..acf823696 100644--- a/desktop/lib/super_tabs/parts.dart+++ b/desktop/lib/super_tabs/parts.dart@@ -55,6 +55,7 @@ class SuperTabParts {required this.epoch,required this.lastChangedAt,required this.githubApp,+ this.derivedEtag,});/// A first load (section 4.3). Throws a [FormatException] for a response of another format.@@ -76,6 +77,7 @@ class SuperTabParts {// Null for a Swarm with no part changed yet (contract.ts `ReadMeta`).lastChangedAt: _int(answer['lastChangedAt'] ?? 0),githubApp: answer['githubApp'] == true,+ derivedEtag: _etag(answer),);parts._seen['head'] = _int(parts._head['version']);for (final kind in _lists) {@@ -131,6 +133,26 @@ class SuperTabParts {/// Whether the backend has the GitHub app, as the first load says.final bool githubApp;+ /// The etag of the summary held (`derived` without the marks, backend read/derivedEtag.ts, #2906), as the last read+ /// sent it: sent back with the cursor, so a change read leaves the summary out while it still matches. Null from a+ /// backend without it, and then none is sent.+ String? derivedEtag;++ /// What a change read asks after `?`: the cursor, and the summary's etag when one is held.+ String get changesQuery {+ final etag = derivedEtag;+ final query = 'cursor=${Uri.encodeQueryComponent(cursor)}';+ return etag == null+ ? query+ : '$query&derivedEtag=${Uri.encodeQueryComponent(etag)}';+ }++ static String? _etag(Map<String, dynamic> answer) =>+ switch (answer['derivedEtag']) {+ final String etag when etag.isNotEmpty => etag,+ _ => null,+ };+Map<String, dynamic> _head;Map<String, dynamic> _derived;final _live = <String, Map<String, Map<String, dynamic>>>{};@@ -269,6 +291,14 @@ class SuperTabParts {/// Merges one change read (section 4.4): a higher version wins, a tombstone removes the part, a dormant stub moves/// the group out of the active ones, `derived` is replaced, and the cursor moves. A reload answer is the caller's.SuperTabPartsChange apply(Map<String, dynamic> changes) {+ // Without the summary, a reply is only right for the one held: checked before anything is merged.+ final unchanged = changes['derivedUnchanged'] == true;+ final etag = _etag(changes);+ if (unchanged && (etag == null || etag != derivedEtag)) {+ throw const FormatException(+ 'A Swarm read left out a summary this reader does not hold.',+ );+ }var changed = false;// Anything but a machine's own report moved.var beyondMachines = false;@@ -333,11 +363,14 @@ class SuperTabParts {_takeThreadMarks(derived.remove('threadMarks')) |_takeChatSettings(derived.remove('chatSettings')) |readMarksMoved;- if (!const DeepCollectionEquality().equals(derived, _derived)) {+ // Unchanged: `derived` held only the marks, and the summary held stands.+ if (!unchanged &&+ !const DeepCollectionEquality().equals(derived, _derived)) {if (!_sameBeyondMachines(derived, _derived)) beyondMachines = true;_derived = derived;changed = true;}+ derivedEtag = etag;cursor = _cursor(changes);epoch = _int(changes['epoch'] ?? epoch);lastChangedAt = _int(changes['lastChangedAt'] ?? lastChangedAt);@@ -494,6 +527,7 @@ class SuperTabParts {for (final kind in _lists)kind: (_live[kind] ?? const {}).values.toList(),'derived': _derived,+ if (derivedEtag != null) 'derivedEtag': derivedEtag,},'dormant': _dormant.values.toList(),'seen': Map<String, int>.of(_seen),diff --git a/desktop/test/super_tab_parts_reading_test.dart b/desktop/test/super_tab_parts_reading_test.dartindex d3dfaa373..25f4cd8f3 100644--- a/desktop/test/super_tab_parts_reading_test.dart+++ b/desktop/test/super_tab_parts_reading_test.dart@@ -123,6 +123,65 @@ void main() {},);+ // #2906: the machine and agent summary is sent again only when it changed.+ test(+ 'an unchanged poll echoes the summary etag and keeps the summary it holds',+ () async {+ server.state['mentionsWaiting'] = 2;+ final c = open();+ await c.refresh();+ await c.refresh();+ final echoed = Uri.parse(reads.last);+ expect(echoed.path, '/api/super-tabs/tab/changes');+ expect(echoed.queryParameters['derivedEtag'], isNotEmpty);+ expect(server.parts.unchangedSummaries, 1);+ expect(c.snapshot!.mentionsWaiting, 2);++ // The summary moves: the next read sends it whole, and it replaces the one held.+ server.state['mentionsWaiting'] = 3;+ await c.refresh();+ expect(server.parts.unchangedSummaries, 1);+ expect(c.snapshot!.mentionsWaiting, 3);+ await c.refresh();+ expect(server.parts.unchangedSummaries, 2);+ expect(c.snapshot!.mentionsWaiting, 3);+ expect(c.error, isNull);+ },+ );++ test('a refused etag (a backend rolled back) reads in full once, then polls without it', () async {+ final c = open();+ await c.refresh();+ await c.refresh();+ expect(Uri.parse(reads.last).queryParameters['derivedEtag'], isNotNull);+ server.parts.stampsDerived = false;+ await c.refresh();+ await c.refresh();+ expect(reads.skip(2).map((p) => Uri.parse(p).path), [+ '/api/super-tabs/tab/changes',+ '/api/super-tabs/tab/parts',+ '/api/super-tabs/tab/changes',+ ]);+ expect(+ Uri.parse(reads.last).queryParameters,+ isNot(contains('derivedEtag')),+ );+ expect(c.error, isNull);+ expect(c.accessRevoked, isFalse);+ });++ test('a backend without the etag is polled as before', () async {+ server.parts.stampsDerived = false;+ final c = open();+ await c.refresh();+ await c.refresh();+ await c.refresh();+ for (final path in reads.skip(1)) {+ expect(Uri.parse(path).queryParameters.keys, ['cursor']);+ }+ expect(c.error, isNull);+ });+test('a removed group leaves the snapshot and its local state goes',() async {diff --git a/desktop/test/super_tab_parts_test.dart b/desktop/test/super_tab_parts_test.dartindex 34679f13a..48bde0831 100644--- a/desktop/test/super_tab_parts_test.dart+++ b/desktop/test/super_tab_parts_test.dart@@ -287,6 +287,100 @@ void main() {expect(snapshot.people.map((p) => p.id), ['6a86', '6a87']);});+ // #2906: a change read leaves out the summary the reader holds while its etag still matches.+ {+ Map<String, dynamic> load() => {+ ...recorded('first_load'),+ 'derivedEtag': 'e1',+ };+ // Nothing else moved either: no head, no parts.+ Map<String, dynamic> unchanged(String etag, [Map? marks]) => {+ ...recorded('changes'),+ 'head': null,+ for (final kind in [+ 'members',+ 'agents',+ 'groups',+ 'machines',+ 'starts',+ 'progress',+ ])+ kind: <dynamic>[],+ 'derivedEtag': etag,+ 'derivedUnchanged': true,+ 'derived': {'readMarks': ?marks},+ };++ test(+ 'summary etag: is held from a first load and sent back with the cursor',+ () {+ final parts = SuperTabParts.firstLoad(load());+ expect(parts.derivedEtag, 'e1');+ expect(Uri.splitQueryString(parts.changesQuery), {+ 'cursor': parts.cursor,+ 'derivedEtag': 'e1',+ });+ },+ );++ test('summary etag: an unchanged reply keeps the summary held and takes its marks', () {+ final parts = SuperTabParts.firstLoad(load());+ final held = jsonEncode(parts.derived);+ final change = parts.apply(unchanged('e1', {'aeef': 9}));+ expect(jsonEncode(parts.derived), held);+ expect(parts.derived['agents'], isNotEmpty);+ expect(parts.derivedEtag, 'e1');+ expect(change.readMarksOnly, isTrue);+ });++ test('summary etag: a reply with the summary replaces it and its etag', () {+ final parts = SuperTabParts.firstLoad(load());+ parts.apply({+ ...recorded('changes'),+ 'derivedEtag': 'e2',+ 'derived': {+ 'agents': <String, dynamic>{},+ 'machines': <String, dynamic>{},+ 'mentionsWaiting': 4,+ },+ });+ expect(parts.derivedEtag, 'e2');+ expect(parts.derived['mentionsWaiting'], 4);+ expect(parts.derived['agents'], isEmpty);+ });++ test('summary etag: "unchanged" for an etag it does not hold is refused, not taken as no summary', () {+ final parts = SuperTabParts.firstLoad(load());+ expect(() => parts.apply(unchanged('e9')), throwsFormatException);+ final older = SuperTabParts.firstLoad(recorded('first_load'));+ expect(() => older.apply(unchanged('e1')), throwsFormatException);+ });++ test(+ 'summary etag: an older backend sends none, and none is sent back',+ () {+ final parts = SuperTabParts.firstLoad(recorded('first_load'));+ expect(parts.derivedEtag, isNull);+ expect(Uri.splitQueryString(parts.changesQuery), {+ 'cursor': parts.cursor,+ });+ final rolledBack = SuperTabParts.firstLoad(load())+ ..apply(recorded('changes'));+ expect(rolledBack.derivedEtag, isNull);+ },+ );++ test('summary etag: a saved store keeps its etag', () {+ final parts = SuperTabParts.firstLoad(load());+ final kept = SuperTabParts.fromJson(+ jsonDecode(jsonEncode(parts.toJson())) as Map<String, dynamic>,+ );+ expect(kept.derivedEtag, 'e1');+ expect(jsonEncode(kept.apply(unchanged('e1')).changed), 'false');+ expect(jsonEncode(kept.derived), jsonEncode(parts.derived));+ });+ }+test('what it keeps reads back as the same store', () {final parts = SuperTabParts.firstLoad(recorded('first_load'))..apply(recorded('changes'))diff --git a/desktop/test/support/super_tab_fixture.dart b/desktop/test/support/super_tab_fixture.dartindex ca548376d..228832152 100644--- a/desktop/test/support/super_tab_fixture.dart+++ b/desktop/test/support/super_tab_fixture.dart@@ -58,6 +58,26 @@ class SwarmPartsBackend {/// The visibility epoch: a test moves it to have the next change read answer a reload.int epoch = 1;+ /// A backend with the summary etag (backend read/derivedEtag.ts, #2906). Off, it is one from before it, or one rolled+ /// back: no etag is sent, and a change read naming one is refused by its strict query.+ bool stampsDerived = true;++ /// Change reads answered without the summary, its etag still matching.+ int unchangedSummaries = 0;++ static const _markFields = {+ 'readMarks',+ 'unreadMarks',+ 'threadMarks',+ 'chatSettings',+ };++ /// The summary without the reader's marks, as one string: equal summaries, equal etag.+ static String _etag(Map<String, dynamic> derived) => jsonEncode({+ for (final entry in derived.entries)+ if (!_markFields.contains(entry.key)) entry.key: entry.value,+ }).hashCode.toRadixString(36);+/// The ticks a change read sends again, as the 30 s overlap does.int overlap = 3;@@ -273,6 +293,11 @@ class SwarmPartsBackend {for (final kind in _kinds) kind: <dynamic>[],'derived': _derived(state, reader),};+ if (stampsDerived) {+ answer['derivedEtag'] = _etag(+ answer['derived'] as Map<String, dynamic>,+ );+ }for (final entry in _ordered()) {final kind = entry.key.substring(0, entry.key.indexOf(':'));if (kind == 'head' || entry.value.deleted) continue;@@ -284,6 +309,14 @@ class SwarmPartsBackend {}if (segments.length == 4 && route == 'changes') {reads.add(path);+ final held = uri.queryParameters['derivedEtag'];+ if (held != null && !stampsDerived) {+ throw const SuperTabFailure(+ 'VALIDATION_ERROR',+ 'Unrecognized key(s) in object: \'derivedEtag\'',+ statusCode: 400,+ );+ }final cursor = RegExp(r'^c(\d+)e(\d+)$').firstMatch(uri.queryParameters['cursor'] ?? '');if (cursor == null || int.parse(cursor.group(2)!) != epoch) {@@ -303,6 +336,18 @@ class SwarmPartsBackend {for (final kind in _kinds) kind: <dynamic>[],'derived': _derived(state, reader),};+ if (stampsDerived) {+ final derived = answer['derived'] as Map<String, dynamic>;+ final etag = answer['derivedEtag'] = _etag(derived);+ if (held == etag) {+ unchangedSummaries++;+ answer['derivedUnchanged'] = true;+ answer['derived'] = {+ for (final entry in derived.entries)+ if (_markFields.contains(entry.key)) entry.key: entry.value,+ };+ }+ }for (final entry in _ordered()) {if (entry.value.changedAt <= since) continue;if (_hides(entry.key, entry.value, reader, state)) continue;diff --git a/docs/design/2026-10-26-swarm-state-parts.md b/docs/design/2026-10-26-swarm-state-parts.mdindex 9a3ccaacc..0ae6bbe3c 100644--- a/docs/design/2026-10-26-swarm-state-parts.md+++ b/docs/design/2026-10-26-swarm-state-parts.md@@ -566,6 +566,11 @@ stands. Every contributed machine has an entry in `derived.machines`. These are- **The client merges** by `(kind, key)`, keeping the higher `version`. A tombstone removes the part. A dormant stubmoves the group out of the active list. The client replaces `derived` and moves to the returned `cursor`.Overlap duplicates have equal versions and are ignored.+- **The summary only when it changed** (task #2906, `read/derivedEtag.ts`). Every first load and change read also+ carries `derivedEtag`, a hash of `derived` without the reader's marks. A client that sends it back as+ `&derivedEtag=` gets, while it still matches, `derivedUnchanged: true` and a `derived` holding only its moved marks,+ and keeps the summary it holds. A client that sends none gets `derived` whole. A client refused for the parameter (a+ backend from before it) reads in full, which holds no etag.- **Messages are not in the feed.** A group's `lastMessageSequence` tells the client there is something to read,and it pages messages per group as today (4.6).diff --git a/docs/plans/2026-11-04-derived-etag-plan.md b/docs/plans/2026-11-04-derived-etag-plan.mdnew file mode 100644index 000000000..af301ffc2--- /dev/null+++ b/docs/plans/2026-11-04-derived-etag-plan.md@@ -0,0 +1,134 @@+# Polls send the Swarm summary only when it changed: implementation plan++This plan follows the project's way of working (writing-plans header, then test-driven tasks, implemented by the+Engineer in this session; one review step on the pull request).++**Goal:** A Swarm change read leaves out the machine and agent summary (`derived`) when it is the same as the one the+reader already holds, so an unchanged poll costs a few hundred bytes at any Swarm size instead of 12 KB (36 machines)+to 92 KB (360).++**Architecture:** The backend hashes each reader's `derived` (without the reader's marks, which already go out as+changes only) into a short `derivedEtag`, sent on every first load and change read. A client that holds one sends it+back as `derivedEtag=` on its next change read; when it still matches, the reply's `derived` holds only the marks and+says `derivedUnchanged: true`, and the client keeps the summary it holds. A client that sends nothing (every released+one) gets the whole summary as today. This is the host feed's `held` etag (read/hostEtag.ts) applied to the app+windows' reads.++**Tech Stack:** Backend TypeScript (Fastify, zod, Prisma/MongoDB), vitest; Desktop/web and phone Flutter (Dart).++**Spec:** Task #2906 (Swarm conversation d49673c3ec733ee886f9bca8); measurement by Wren13 for #2637, PR #2909+(`backend/integration/bench/machineCapCost.ts`); read API design docs/design/2026-10-26-swarm-state-parts.md, section 4.++## Global Constraints++- Released clients keep working: a change read without `derivedEtag` answers exactly as today, plus the new top-level+ `derivedEtag` field (unknown top-level fields are ignored by released Desktop, phone and CLI).+- Unchanged polls carry no summary: only the reader's moved marks (`readMarks`, `unreadMarks`, `threadMarks`,+ `chatSettings`) stay in `derived`.+- Red-first tests; bytes per poll before/after at 36 and 360 machines in the pull request; one review step.+- Swarm code stays out of cli/src/core. Desktop and phone `parts.dart` stay identical copies (mobile+ test/swarm/desktop_parity_test.dart).+- Desktop tests use a memory store and a fake sign-in; nothing signs the Mac out of the real CLI.++## Design++- Growth: nothing is stored. The etag lives in each client's held parts (and Desktop's saved view) and is replaced by+ every read.+- Concurrency: none added. The etag is computed from the reply the read already builds, inside the same read.+- Read cost: no new database read. One SHA-256 over the reply's own summary JSON per read (under 1 ms at 360+ machines); an unchanged poll sends the marks only.+- Scale: at 10x (360 machines) an unchanged poll stays at the size of the parts that changed. The first metric to warn+ is the reply size of a poll after a summary change, which still carries the whole summary (bench row "every machine+ reading").+- Compatibility: old client and new backend: no etag sent, whole summary, as today. New client and old backend: no etag+ received, none sent. New client after a backend rollback: its held etag is refused by the old backend's strict query+ (VALIDATION_ERROR); the client then drops what it holds and reads in full once, getting no etag.+- Migration and rollback: none stored, so nothing to migrate. Rollback costs each open window one first load.+- Audience: the etag hashes the summary of the reader it was built for, after the server's visibility filtering, and+ is only compared with that reader's own echo. It reveals nothing beyond the summary the reader already received.+- Identity: the read's actor, as today; no machine owner's login.+- Failure: an etag mismatch or a malformed etag sends the whole summary. A reply saying "unchanged" for an etag the+ client does not hold is a format error, which the client answers with a first load.+- Platforms: backend; Desktop (macOS, Windows, Linux) and web share `desktop/lib/super_tabs`; phone (`mobile/lib/swarm`).+ The CLI keeps reading as a released client (whole summary).++## Review Focus++1. A summary change of any kind, from any agent's `downSince` to an avatar link, after an unchanged poll: the next+ poll must send the whole summary. Integration test "a summary change after an unchanged poll sends it whole".+2. Marks moved in an otherwise unchanged poll: they still arrive. Integration test "an unchanged poll still sends the+ reader's moved marks".+3. Two readers of one Swarm see different summaries: one reader's etag never matches for another. Integration test+ "an etag is the reader's own".+4. A backend rolled back under a client holding an etag: the client reads in full, without the etag. Desktop and phone+ tests "a refused etag reads in full once, then polls without it".+5. A saved view restored on start: its held etag comes back with its summary (Desktop toJson/fromJson). Dart test+ "a saved store keeps its etag".++---++### Task 1: The backend stamps the summary and leaves it out when the reader holds it++**Files:**+- Create: `backend/src/superTabs/read/derivedEtag.ts`, `backend/src/superTabs/read/derivedEtag.test.ts`+- Modify: `backend/src/superTabs/read/hostEtag.ts` (export its sorted JSON), `backend/src/superTabs/read/contract.ts`,+ `backend/src/superTabs/read/service.ts` (`firstLoad`, `changes`), `backend/src/routes/superTabs.ts` (`changesQuery`)+- Test: `backend/src/superTabs/read/derivedEtag.test.ts`, `backend/src/routes/superTabs.test.ts`,+ `backend/integration/superTabRead.test.ts`++**Interfaces:**+- Produces: `derivedEtag(derived: Derived): string` (22 base64url characters; marks fields ignored);+ `MARK_FIELDS = ['readMarks', 'unreadMarks', 'threadMarks', 'chatSettings'] as const`;+ `SwarmReads.changes(user, tabId, cursor, asAgentId?, options: { lightMachines?: boolean; derivedEtag?: string })`;+ contract: `FirstLoad.derivedEtag: string`, `ChangesDelta.derivedEtag: string`, `ChangesDelta.derivedUnchanged?: true`,+ `ChangesDelta.derived: Derived | ReaderMarks` where `ReaderMarks = Pick<Derived, MarkField>`;+ route query `derivedEtag` (`/^[A-Za-z0-9_-]{1,64}$/`).++- [ ] **Step 1: Failing tests.** `derivedEtag.test.ts`: equal for the same summary built in another key order; equal+ whatever the marks; different when any value of `agents`, `machines`, `avatars` or a digest differs.+ Route test: `?cursor=abc&derivedEtag=e1` passes `{ derivedEtag: 'e1' }`; `?cursor=abc` still passes `{}`;+ `derivedEtag=bad!` is refused with 400. Integration (`changes since a cursor`): a first load's `derivedEtag` equals the+ next change read's; echoed, the reply has `derivedUnchanged: true` and `derived` without `agents`/`machines`; no echo+ or another etag sends the whole summary; a summary change after an unchanged poll sends it whole with a new etag; an+ unchanged poll still sends moved read marks; an etag is the reader's own.+- [ ] **Step 2: Run them, watch them fail.** `npx vitest run src/superTabs/read/derivedEtag.test.ts src/routes/superTabs.test.ts`+ and `npx vitest run --config vitest.super-tabs.config.ts integration/superTabRead.test.ts -t "etag"`.+- [ ] **Step 3: Implement** the interfaces above. `changes` builds `derived` as today, computes the etag, and replaces+ `derived` with its marks alone when `options.derivedEtag` equals it.+- [ ] **Step 4: Run the same tests, watch them pass;** `npx tsc --noEmit`.+- [ ] **Step 5: Commit** the tests first, then the code.++### Task 2: Desktop/web and phone send the etag back and keep the summary they hold++**Files:**+- Modify: `desktop/lib/super_tabs/parts.dart` and its copy `mobile/lib/swarm/parts.dart`;+ `desktop/lib/super_tabs/controller.dart` (`_readParts`); `mobile/lib/swarm/swarm_reader.dart` (`_readParts`);+ `desktop/test/support/super_tab_fixture.dart` (the fake backend answers etags as the backend does)+- Test: `desktop/test/super_tab_parts_reading_test.dart`, `mobile/test/swarm/swarm_reader_test.dart`++**Interfaces:**+- Consumes: Task 1's `derivedEtag`, `derivedUnchanged` and `derivedEtag=` query.+- Produces: `SuperTabParts.derivedEtag` (`String?`), `SuperTabParts.changesQuery` (`String`: `cursor=…` plus+ `&derivedEtag=…` when one is held), kept by `toJson`/`fromJson`.++- [ ] **Step 1: Failing tests.** Desktop: "an unchanged poll sends the etag and keeps the summary" (second change read's+ path has `derivedEtag`, the fake answers without `agents`, `snapshot` still has its agents and a moved mark arrives);+ "a summary change replaces it"; "a refused etag reads in full once, then polls without it"; "a saved store keeps its+ etag". Phone: the same first and third.+- [ ] **Step 2: Run them, watch them fail.** `flutter test test/super_tab_parts_reading_test.dart` (desktop),+ `flutter test test/swarm/swarm_reader_test.dart test/swarm/desktop_parity_test.dart` (mobile).+- [ ] **Step 3: Implement.** `apply` keeps `_derived` when `derivedUnchanged` is true and the reply's etag is the one+ held (else a FormatException: half merged, first load); the readers poll with `changesQuery` and treat a+ VALIDATION_ERROR on a read that sent an etag as they treat FORBIDDEN (first load).+- [ ] **Step 4: Run the same tests, watch them pass;** `flutter analyze` on the changed files.+- [ ] **Step 5: Commit** the tests first, then the code.++### Task 3: Bytes per poll at 36 and 360 machines, before and after++**Files:**+- Create: `backend/integration/bench/derivedEtagCost.ts` (Wren13's machineCapCost.ts harness, with an etag window+ beside the old one)++- [ ] **Step 1:** Run the bench at 36 and 360 machines on main's behaviour (old windows) and with the etag window; the+ table goes in the pull request: reply KB per poll after nothing, after every machine's reading, after one post.+- [ ] **Step 2: Commit.**diff --git a/mobile/lib/swarm/parts.dart b/mobile/lib/swarm/parts.dartindex 06a186b3f..acf823696 100644--- a/mobile/lib/swarm/parts.dart+++ b/mobile/lib/swarm/parts.dart@@ -55,6 +55,7 @@ class SuperTabParts {required this.epoch,required this.lastChangedAt,required this.githubApp,+ this.derivedEtag,});/// A first load (section 4.3). Throws a [FormatException] for a response of another format.@@ -76,6 +77,7 @@ class SuperTabParts {// Null for a Swarm with no part changed yet (contract.ts `ReadMeta`).lastChangedAt: _int(answer['lastChangedAt'] ?? 0),githubApp: answer['githubApp'] == true,+ derivedEtag: _etag(answer),);parts._seen['head'] = _int(parts._head['version']);for (final kind in _lists) {@@ -131,6 +133,26 @@ class SuperTabParts {/// Whether the backend has the GitHub app, as the first load says.final bool githubApp;+ /// The etag of the summary held (`derived` without the marks, backend read/derivedEtag.ts, #2906), as the last read+ /// sent it: sent back with the cursor, so a change read leaves the summary out while it still matches. Null from a+ /// backend without it, and then none is sent.+ String? derivedEtag;++ /// What a change read asks after `?`: the cursor, and the summary's etag when one is held.+ String get changesQuery {+ final etag = derivedEtag;+ final query = 'cursor=${Uri.encodeQueryComponent(cursor)}';+ return etag == null+ ? query+ : '$query&derivedEtag=${Uri.encodeQueryComponent(etag)}';+ }++ static String? _etag(Map<String, dynamic> answer) =>+ switch (answer['derivedEtag']) {+ final String etag when etag.isNotEmpty => etag,+ _ => null,+ };+Map<String, dynamic> _head;Map<String, dynamic> _derived;final _live = <String, Map<String, Map<String, dynamic>>>{};@@ -269,6 +291,14 @@ class SuperTabParts {/// Merges one change read (section 4.4): a higher version wins, a tombstone removes the part, a dormant stub moves/// the group out of the active ones, `derived` is replaced, and the cursor moves. A reload answer is the caller's.SuperTabPartsChange apply(Map<String, dynamic> changes) {+ // Without the summary, a reply is only right for the one held: checked before anything is merged.+ final unchanged = changes['derivedUnchanged'] == true;+ final etag = _etag(changes);+ if (unchanged && (etag == null || etag != derivedEtag)) {+ throw const FormatException(+ 'A Swarm read left out a summary this reader does not hold.',+ );+ }var changed = false;// Anything but a machine's own report moved.var beyondMachines = false;@@ -333,11 +363,14 @@ class SuperTabParts {_takeThreadMarks(derived.remove('threadMarks')) |_takeChatSettings(derived.remove('chatSettings')) |readMarksMoved;- if (!const DeepCollectionEquality().equals(derived, _derived)) {+ // Unchanged: `derived` held only the marks, and the summary held stands.+ if (!unchanged &&+ !const DeepCollectionEquality().equals(derived, _derived)) {if (!_sameBeyondMachines(derived, _derived)) beyondMachines = true;_derived = derived;changed = true;}+ derivedEtag = etag;cursor = _cursor(changes);epoch = _int(changes['epoch'] ?? epoch);lastChangedAt = _int(changes['lastChangedAt'] ?? lastChangedAt);@@ -494,6 +527,7 @@ class SuperTabParts {for (final kind in _lists)kind: (_live[kind] ?? const {}).values.toList(),'derived': _derived,+ if (derivedEtag != null) 'derivedEtag': derivedEtag,},'dormant': _dormant.values.toList(),'seen': Map<String, int>.of(_seen),diff --git a/mobile/lib/swarm/swarm_reader.dart b/mobile/lib/swarm/swarm_reader.dartindex febe3b2f4..8804911ef 100644--- a/mobile/lib/swarm/swarm_reader.dart+++ b/mobile/lib/swarm/swarm_reader.dart@@ -267,9 +267,7 @@ class SwarmReader extends ChangeNotifier {final held = _parts;if (held != null) {try {- final answer = await _get(- '$_path/changes?cursor=${Uri.encodeQueryComponent(held.cursor)}',- );+ final answer = await _get('$_path/changes?${held.changesQuery}');if (answer['reload'] != true) {final SuperTabPartsChange change;try {@@ -283,7 +281,14 @@ class SwarmReader extends ChangeNotifier {return (changed: change.changed || stale, view: null);}} on SuperTabFailure catch (failure) {- if (!const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) rethrow;+ // A backend rolled back to one without the summary etag refuses it+ // (#2906): read in full, which holds none.+ final etagRefused =+ failure.code == 'VALIDATION_ERROR' && held.derivedEtag != null;+ if (!etagRefused &&+ !const {'FORBIDDEN', 'NOT_FOUND'}.contains(failure.code)) {+ rethrow;+ }}_parts = null;}diff --git a/mobile/test/swarm/swarm_reader_test.dart b/mobile/test/swarm/swarm_reader_test.dartindex 51a48812f..6fa27631d 100644--- a/mobile/test/swarm/swarm_reader_test.dart+++ b/mobile/test/swarm/swarm_reader_test.dart@@ -67,6 +67,69 @@ void main() {expect(swarm.paths.last, '/api/super-tabs/$tab/changes?cursor=c2');});+ // #2906: the machine and agent summary is sent again only when it changed.+ test(+ 'an unchanged poll echoes the summary etag and keeps the summary it holds',+ () async {+ final swarm = FakeSwarm();+ swarm.routes['/api/super-tabs/$tab/parts'] = (_) => {+ ...firstLoad(cursor: 'c1', derived: {'mentionsWaiting': 2}),+ 'derivedEtag': 'e1',+ };+ swarm.routes['/api/super-tabs/$tab/changes'] = (_) => {+ ...changes(cursor: 'c2'),+ 'derivedEtag': 'e1',+ 'derivedUnchanged': true,+ };+ final r = reader(swarm);+ addTearDown(r.dispose);++ await r.refresh();+ await r.refresh();++ expect(+ swarm.paths.last,+ '/api/super-tabs/$tab/changes?cursor=c1&derivedEtag=e1',+ );+ expect(r.snapshot!.mentionsWaiting, 2);+ expect(r.error, isNull);+ },+ );++ test('a refused etag (a backend rolled back) reads in full once, then polls without it', () async {+ var loads = 0;+ final swarm = FakeSwarm();+ swarm.routes['/api/super-tabs/$tab/parts'] = (_) => {+ ...firstLoad(cursor: 'c${++loads}'),+ if (loads == 1) 'derivedEtag': 'e1',+ };+ swarm.routes['/api/super-tabs/$tab/changes'] = (_) {+ if (swarm.paths.last.contains('derivedEtag')) {+ throw SuperTabFailure(+ 'VALIDATION_ERROR',+ "Unrecognized key(s) in object: 'derivedEtag'",+ statusCode: 400,+ );+ }+ return changes(cursor: 'c9');+ };+ final r = reader(swarm);+ addTearDown(r.dispose);++ await r.refresh();+ await r.refresh();+ await r.refresh();++ expect(swarm.paths, [+ '/api/super-tabs/$tab/parts',+ '/api/super-tabs/$tab/changes?cursor=c1&derivedEtag=e1',+ '/api/super-tabs/$tab/parts',+ '/api/super-tabs/$tab/changes?cursor=c2',+ ]);+ expect(r.error, isNull);+ expect(r.accessRevoked, isFalse);+ });+test('a reload answer starts again from a first load', () async {var loads = 0;final swarm = FakeSwarm();