The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Swarm cache stamps never repeat: Redis's clock in microseconds; CI runs the Redis-backed tests (Dee) (#2458)
0xken1537b62749b8f7
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 574a2bf3c..b55d2856e 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -370,6 +370,16 @@ jobs:if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.backend == 'true'runs-on: ubuntu-latesttimeout-minutes: 10+ # A real Redis for the Redis-backed unit tests, which skip without one: a cache stamp that could repeat passed CI unseen+ # for that reason (review of #2456).+ services:+ redis:+ image: redis:7.2.5+ ports: ['6379:6379']+ options: >-+ --health-cmd "redis-cli ping" --health-interval 2s --health-timeout 2s --health-retries 15+ env:+ REDIS_TEST_URL: redis://127.0.0.1:6379/15defaults:run:working-directory: backend@@ -386,7 +396,9 @@ jobs:# The whole backend unit suite (src/**/*.test.ts, including the shared desk layout test): it once ran only the desk# test, and main broke 48 team-session tests unseen (#1926).- name: Backend unit suite- run: npm test+ run: npm test -- --reporter=default --reporter=json --outputFile.json=vitest-results.json+ - name: The Redis-backed tests ran+ run: node scripts/redis-tests-ran.mjs vitest-results.jsonwebsite-checks:concurrency:diff --git a/backend/scripts/redis-tests-ran.mjs b/backend/scripts/redis-tests-ran.mjsnew file mode 100644index 000000000..e1ebd8cad--- /dev/null+++ b/backend/scripts/redis-tests-ran.mjs@@ -0,0 +1,27 @@+// CI's check that the Redis-backed unit tests ran against its Redis service instead of skipping: they skip without+// REDIS_TEST_URL, and a cache stamp that could repeat passed CI unseen for that reason (review of #2456).+// Usage: node scripts/redis-tests-ran.mjs <vitest JSON report>, from backend/. Each test file that reads REDIS_TEST_URL+// must have run every test, and passed at least one.+import { readFileSync, readdirSync } from 'node:fs'+import { join, relative } from 'node:path'++const report = JSON.parse(readFileSync(process.argv[2], 'utf8'))+const files = (dir) => readdirSync(dir, { withFileTypes: true })+ .flatMap(entry => entry.isDirectory() ? files(join(dir, entry.name)) : [join(dir, entry.name)])+const redisBacked = files('src').filter(file => file.endsWith('.test.ts') && readFileSync(file, 'utf8').includes('REDIS_TEST_URL'))++const problems = []+if (!redisBacked.length) problems.push('no test file reads REDIS_TEST_URL')+for (const file of redisBacked) {+ const result = report.testResults.find(r => relative(process.cwd(), r.name) === file)+ const statuses = (result?.assertionResults ?? []).map(a => a.status)+ const passed = statuses.filter(s => s === 'passed').length+ const notRun = statuses.filter(s => s !== 'passed' && s !== 'failed').length+ if (!result) problems.push(`${file}: not in the report`)+ else if (!passed || notRun) problems.push(`${file}: ${passed} passed, ${notRun} skipped`)+ else console.log(`${file}: ${passed} passed against Redis`)+}+if (problems.length) {+ console.error(`The Redis-backed tests did not all run:\n${problems.join('\n')}`)+ process.exit(1)+}diff --git a/backend/src/lib/redisRateCounter.test.ts b/backend/src/lib/redisRateCounter.test.tsindex cae574791..582d725bd 100644--- a/backend/src/lib/redisRateCounter.test.ts+++ b/backend/src/lib/redisRateCounter.test.ts@@ -1,4 +1,5 @@import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'+import { randomUUID } from 'node:crypto'import { execFileSync, spawn, type ChildProcess } from 'node:child_process'import { createServer, type AddressInfo } from 'node:net'import { Redis } from 'ioredis'@@ -7,14 +8,23 @@ import { redisWindowCounter } from './redisRateCounter.js'// A real client: the shared setup's never connects (noRedis.test-support.ts).vi.unmock('ioredis')-// Against a real redis-server, so the script is what is tested. Skipped where none is installed (CI's Linux runners).-const hasRedis = (() => { try { execFileSync('redis-server', ['--version'], { stdio: 'pipe' }); return true } catch { return false } })()+// Against a real Redis, so the script is what is tested: REDIS_TEST_URL when set (CI's Redis service), with keys of this run+// alone, removed afterwards; otherwise a redis-server of its own where one is installed. Skipped with neither.+const url = process.env.REDIS_TEST_URL+const hasRedis = !!url || (() => { try { execFileSync('redis-server', ['--version'], { stdio: 'pipe' }); return true } catch { return false } })()+const run = randomUUID()+const k = (name: string) => `rl:t:${run}:${name}`describe.skipIf(!hasRedis)('Redis window counter', () => {- let server: ChildProcess+ let server: ChildProcess | undefinedlet redis: RedisbeforeAll(async () => {+ if (url) {+ redis = new Redis(url, { lazyConnect: true })+ await redis.connect()+ return+ }const port = await new Promise<number>((resolve) => {const probe = createServer().listen(0, '127.0.0.1', () => { const { port } = probe.address() as AddressInfo; probe.close(() => resolve(port)) })})@@ -22,22 +32,27 @@ describe.skipIf(!hasRedis)('Redis window counter', () => {redis = new Redis({ port, host: '127.0.0.1', lazyConnect: true, retryStrategy: () => 50 })for (let i = 0; i < 100; i++) { try { await redis.connect(); break } catch { await new Promise((r) => setTimeout(r, 50)) } }})- afterAll(() => { redis?.disconnect(); server?.kill() })+ afterAll(async () => {+ const used = await redis?.keys(k('*')) ?? []+ if (url && used.length) await redis.del(...used)+ redis?.disconnect()+ server?.kill()+ })it('counts each key in one round trip and returns the counts after this hit', async () => {const count = redisWindowCounter(redis)- const both = [{ key: 'rl:t:a:1', windowMs: 60_000 }, { key: 'rl:t:b:1', windowMs: 1_000 }]+ const both = [{ key: k('a:1'), windowMs: 60_000 }, { key: k('b:1'), windowMs: 1_000 }]expect(await count(both)).toEqual([1, 1])expect(await count(both)).toEqual([2, 2])- expect(await count([{ key: 'rl:t:a:1', windowMs: 60_000 }])).toEqual([3])+ expect(await count([{ key: k('a:1'), windowMs: 60_000 }])).toEqual([3])})it('lets each key expire with its own window, so nothing is left behind', async () => {const count = redisWindowCounter(redis)- await count([{ key: 'rl:t:minute:1', windowMs: 60_000 }, { key: 'rl:t:second:1', windowMs: 1_000 }])+ await count([{ key: k('minute:1'), windowMs: 60_000 }, { key: k('second:1'), windowMs: 1_000 }])- const [minute, second] = [await redis.pttl('rl:t:minute:1'), await redis.pttl('rl:t:second:1')]+ const [minute, second] = [await redis.pttl(k('minute:1')), await redis.pttl(k('second:1'))]expect(minute).toBeGreaterThan(59_000)expect(minute).toBeLessThanOrEqual(60_000)diff --git a/backend/src/lib/terminalRouteRedis.test.ts b/backend/src/lib/terminalRouteRedis.test.tsindex 03865e2b8..5d481c62b 100644--- a/backend/src/lib/terminalRouteRedis.test.ts+++ b/backend/src/lib/terminalRouteRedis.test.ts@@ -1,7 +1,8 @@import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'import type { UpBusMsg } from './tunnel.js'-// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI has none, so this file is skipped there). The bus+// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI's Redis service; skipped without one, which CI's+// redis-tests-ran step refuses). The bus// connects to it as REDIS_URL. One process stands for several: a process's channel is just `termpod:`/`uppod:` + its id,// so subscribing to another id here is what that other process would do.const url = process.env.REDIS_TEST_URLdiff --git a/backend/src/superTabs/cache.ts b/backend/src/superTabs/cache.tsindex 95ac9d7b3..590d0134d 100644--- a/backend/src/superTabs/cache.ts+++ b/backend/src/superTabs/cache.ts@@ -179,16 +179,25 @@ export class CacheView {export const STAMP_TTL_MS = 30 * 86_400_000const stampKey = (tabId: string) => `swarm:stamp:${tabId}`/**- * A Swarm's first bump starts both stamps from Redis's clock in ms, then moves the named ones by one. Redis losing the key- * (a restart, the expiry) therefore never hands out a stamp a process may still hold: it starts again above it, as long as- * a Swarm has fewer commits than milliseconds between its first and its loss.+ * Moves each named stamp to Redis's clock in microseconds, or one past its value if that is higher, so a stamp only grows+ * and no bump hands out a stamp already handed out: a process serves what it holds at an unchanged stamp (SwarmCache), and a+ * delivery poll skips its revocation check while the main stamp holds (store.ts deliveries). No two bumps run in one+ * microsecond (Redis runs one script at a time, each longer than that), so every stamp is its bump's time and none runs+ * ahead of the clock. When Redis loses the key (a restart, the expiry), the next bump starts above every stamp handed out.+ * When it comes back with an older value (a snapshot, a lagging replica), reads see that older stamp again until the next+ * bump of each scope moves it above them all; held entries live at most CACHE_MAX_AGE_MS meanwhile. In milliseconds, a+ * burst of bumps ran ahead of the clock and a loss within that millisecond handed a stamp out again (review of #2456). A first bump starts the stamps it does not name one+ * below the clock. Stamps are compared only for equality: the jump from older millisecond stamps is one more change.*/-const BUMP = `if redis.call('exists', KEYS[1]) == 0 then- local t = redis.call('time')- local base = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)- redis.call('hset', KEYS[1], 'main', base - 1, 'machine', base - 1)+const BUMP = `local t = redis.call('time')+local now = tonumber(t[1]) * 1000000 + tonumber(t[2])+if redis.call('exists', KEYS[1]) == 0 then+ redis.call('hset', KEYS[1], 'main', string.format('%.0f', now - 1), 'machine', string.format('%.0f', now - 1))+end+for i = 2, #ARGV do+ local next = tonumber(redis.call('hget', KEYS[1], ARGV[i]) or 0) + 1+ redis.call('hset', KEYS[1], ARGV[i], string.format('%.0f', math.max(next, now)))end-for i = 2, #ARGV do redis.call('hincrby', KEYS[1], ARGV[i], 1) endredis.call('pexpire', KEYS[1], ARGV[1])return 1`diff --git a/backend/src/superTabs/cacheRedis.test.ts b/backend/src/superTabs/cacheRedis.test.tsindex 2c210caa5..6be650fec 100644--- a/backend/src/superTabs/cacheRedis.test.ts+++ b/backend/src/superTabs/cacheRedis.test.ts@@ -3,8 +3,8 @@ import { Redis } from 'ioredis'import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'import { redisStamps, STAMP_TTL_MS, type StampScope } from './cache.js'-// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI has none, so this file is skipped there). Only keys of-// this run are touched, and they are removed afterwards.+// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI's Redis service; skipped without one, which CI's+// redis-tests-ran step refuses). Only keys of this run are touched, and they are removed afterwards.const url = process.env.REDIS_TEST_URL// A real client: the shared setup's never connects (noRedis.test-support.ts).vi.unmock('ioredis')@@ -36,7 +36,9 @@ describe.skipIf(!url)('redisStamps', () => {const before = Date.now()await stamps.bump(id, ['main'])const first = (await stamps.read([id])).get(id)!- expect(first.main).toBeGreaterThan(before - 60_000)+ // Redis's clock in microseconds.+ expect(first.main).toBeGreaterThan((before - 60_000) * 1000)+ expect(first.main).toBeLessThan((Date.now() + 60_000) * 1000)expect(first.machine).toBe(first.main - 1)await stamps.bump(id, ['main']); await stamps.bump(id, ['main'])const old = (await stamps.read([id])).get(id)!.main@@ -45,15 +47,57 @@ describe.skipIf(!url)('redisStamps', () => {expect((await stamps.read([id])).get(id)!.main).toBeGreaterThan(old)})- it('moves only the scopes named, by one each', async () => {+ // Review of #2456: a stamp the old Redis handed out must never come back for other data, or a process serves what it holds+ // (cache.ts) and a delivery poll skips its revocation check (store.ts deliveries). Each bump is at least Redis's clock.+ it('moves forward even after Redis comes back with an older value, as from a snapshot or a lagging replica', async () => {+ const stamps = redisStamps(redis, bus), id = tab()+ used.push(id)+ await stamps.bump(id, ['main', 'machine'])+ const older = (await stamps.read([id])).get(id)!+ for (let i = 0; i < 5; i++) await stamps.bump(id, ['main', 'machine'])+ const newest = (await stamps.read([id])).get(id)!+ await redis.hset(`swarm:stamp:${id}`, 'main', older.main, 'machine', older.machine)+ await stamps.bump(id, ['main', 'machine'])+ const after = (await stamps.read([id])).get(id)!+ expect(after.main).toBeGreaterThan(newest.main)+ expect(after.machine).toBeGreaterThan(newest.machine)+ })++ it('moves each stamp strictly forward on every bump, many in one millisecond', async () => {+ const stamps = redisStamps(redis, bus), id = tab()+ used.push(id)+ const seen: number[] = []+ for (let i = 0; i < 50; i++) {+ await stamps.bump(id, ['main'])+ seen.push((await stamps.read([id])).get(id)!.main)+ }+ for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeGreaterThan(seen[i - 1]!)+ })++ it('moves a stamp ahead of the clock by exactly one, never back to the clock', async () => {+ const stamps = redisStamps(redis, bus), id = tab()+ used.push(id)+ await stamps.bump(id, ['main', 'machine'])+ // As a burst or a faster clock could leave it: 10 s above Redis's clock, in microseconds.+ const ahead = (Date.now() + 10_000) * 1000+ await redis.hset(`swarm:stamp:${id}`, 'main', ahead, 'machine', ahead)+ await stamps.bump(id, ['main'])+ expect((await stamps.read([id])).get(id)).toEqual({ main: ahead + 1, machine: ahead })+ })++ it('moves only the scopes named', async () => {const stamps = redisStamps(redis, bus), id = tab()used.push(id)await stamps.bump(id, ['main'])const start = (await stamps.read([id])).get(id)!await stamps.bump(id, ['machine'])- expect((await stamps.read([id])).get(id)).toEqual({ main: start.main, machine: start.machine + 1 })+ const machine = (await stamps.read([id])).get(id)!+ expect(machine.main).toBe(start.main)+ expect(machine.machine).toBeGreaterThan(start.machine)await stamps.bump(id, ['main', 'machine'])- expect((await stamps.read([id])).get(id)).toEqual({ main: start.main + 1, machine: start.machine + 2 })+ const both = (await stamps.read([id])).get(id)!+ expect(both.main).toBeGreaterThan(machine.main)+ expect(both.machine).toBeGreaterThan(machine.machine)})it('reads several Swarms in one call, and lets a quiet Swarm\'s stamp expire', async () => {diff --git a/docs/design/2026-10-28-swarm-read-cache.md b/docs/design/2026-10-28-swarm-read-cache.mdindex 3f5cada71..6fb4a27e0 100644--- a/docs/design/2026-10-28-swarm-read-cache.md+++ b/docs/design/2026-10-28-swarm-read-cache.md@@ -14,6 +14,11 @@ bumps), `backend/src/superTabs/cachedReads.ts` (reads built from what the cache`changedAt`) therefore no longer moves every 15 s for every machine. Clients keep the part with the higherversion and never read `changedAt`.- Touches change no data and move nothing.+ - Each move sets the stamp to Redis's clock in microseconds, or one past its value if that is higher+ (`cache.ts BUMP`). A stamp is therefore its bump's time and never runs ahead of the clock, so no bump hands out+ a stamp already handed out, even after Redis lost the key or came back with an older value (a restart, a+ snapshot, a lagging replica). After an older value, reads see it again until the next bump of each scope; held+ entries live at most 60 s meanwhile. Millisecond stamps could repeat on a bump (review of #2456).- **Bump after commit.** Every part write records its Swarm in the transaction (`parts/write.ts guardedWrite`,`parts/load.ts insertParts`). `swarmTransaction` bumps those Swarms after the commit, before the caller answers. Anyread after the answer, on any process, therefore sees the change. This covers the store's commands, sweeps,