Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Swarm cache stamps never repeat: Redis's clock in microseconds; CI runs the Redis-backed tests (Dee) (#2458)

0xken1537b62749b8f7
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 574a2bf3c..b55d2856e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -370,6 +370,16 @@ jobs:
if: needs.plan.outputs.ready == 'true' && needs.plan.outputs.backend == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
+ # A real Redis for the Redis-backed unit tests, which skip without one: a cache stamp that could repeat passed CI unseen
+ # for that reason (review of #2456).
+ services:
+ redis:
+ image: redis:7.2.5
+ ports: ['6379:6379']
+ options: >-
+ --health-cmd "redis-cli ping" --health-interval 2s --health-timeout 2s --health-retries 15
+ env:
+ REDIS_TEST_URL: redis://127.0.0.1:6379/15
defaults:
run:
working-directory: backend
@@ -386,7 +396,9 @@ jobs:
# The whole backend unit suite (src/**/*.test.ts, including the shared desk layout test): it once ran only the desk
# test, and main broke 48 team-session tests unseen (#1926).
- name: Backend unit suite
- run: npm test
+ run: npm test -- --reporter=default --reporter=json --outputFile.json=vitest-results.json
+ - name: The Redis-backed tests ran
+ run: node scripts/redis-tests-ran.mjs vitest-results.json
website-checks:
concurrency:
diff --git a/backend/scripts/redis-tests-ran.mjs b/backend/scripts/redis-tests-ran.mjs
new file mode 100644
index 000000000..e1ebd8cad
--- /dev/null
+++ b/backend/scripts/redis-tests-ran.mjs
@@ -0,0 +1,27 @@
+// CI's check that the Redis-backed unit tests ran against its Redis service instead of skipping: they skip without
+// REDIS_TEST_URL, and a cache stamp that could repeat passed CI unseen for that reason (review of #2456).
+// Usage: node scripts/redis-tests-ran.mjs <vitest JSON report>, from backend/. Each test file that reads REDIS_TEST_URL
+// must have run every test, and passed at least one.
+import { readFileSync, readdirSync } from 'node:fs'
+import { join, relative } from 'node:path'
+
+const report = JSON.parse(readFileSync(process.argv[2], 'utf8'))
+const files = (dir) => readdirSync(dir, { withFileTypes: true })
+ .flatMap(entry => entry.isDirectory() ? files(join(dir, entry.name)) : [join(dir, entry.name)])
+const redisBacked = files('src').filter(file => file.endsWith('.test.ts') && readFileSync(file, 'utf8').includes('REDIS_TEST_URL'))
+
+const problems = []
+if (!redisBacked.length) problems.push('no test file reads REDIS_TEST_URL')
+for (const file of redisBacked) {
+ const result = report.testResults.find(r => relative(process.cwd(), r.name) === file)
+ const statuses = (result?.assertionResults ?? []).map(a => a.status)
+ const passed = statuses.filter(s => s === 'passed').length
+ const notRun = statuses.filter(s => s !== 'passed' && s !== 'failed').length
+ if (!result) problems.push(`${file}: not in the report`)
+ else if (!passed || notRun) problems.push(`${file}: ${passed} passed, ${notRun} skipped`)
+ else console.log(`${file}: ${passed} passed against Redis`)
+}
+if (problems.length) {
+ console.error(`The Redis-backed tests did not all run:\n${problems.join('\n')}`)
+ process.exit(1)
+}
diff --git a/backend/src/lib/redisRateCounter.test.ts b/backend/src/lib/redisRateCounter.test.ts
index cae574791..582d725bd 100644
--- a/backend/src/lib/redisRateCounter.test.ts
+++ b/backend/src/lib/redisRateCounter.test.ts
@@ -1,4 +1,5 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
+import { randomUUID } from 'node:crypto'
import { execFileSync, spawn, type ChildProcess } from 'node:child_process'
import { createServer, type AddressInfo } from 'node:net'
import { Redis } from 'ioredis'
@@ -7,14 +8,23 @@ import { redisWindowCounter } from './redisRateCounter.js'
// A real client: the shared setup's never connects (noRedis.test-support.ts).
vi.unmock('ioredis')
-// Against a real redis-server, so the script is what is tested. Skipped where none is installed (CI's Linux runners).
-const hasRedis = (() => { try { execFileSync('redis-server', ['--version'], { stdio: 'pipe' }); return true } catch { return false } })()
+// Against a real Redis, so the script is what is tested: REDIS_TEST_URL when set (CI's Redis service), with keys of this run
+// alone, removed afterwards; otherwise a redis-server of its own where one is installed. Skipped with neither.
+const url = process.env.REDIS_TEST_URL
+const hasRedis = !!url || (() => { try { execFileSync('redis-server', ['--version'], { stdio: 'pipe' }); return true } catch { return false } })()
+const run = randomUUID()
+const k = (name: string) => `rl:t:${run}:${name}`
describe.skipIf(!hasRedis)('Redis window counter', () => {
- let server: ChildProcess
+ let server: ChildProcess | undefined
let redis: Redis
beforeAll(async () => {
+ if (url) {
+ redis = new Redis(url, { lazyConnect: true })
+ await redis.connect()
+ return
+ }
const port = await new Promise<number>((resolve) => {
const probe = createServer().listen(0, '127.0.0.1', () => { const { port } = probe.address() as AddressInfo; probe.close(() => resolve(port)) })
})
@@ -22,22 +32,27 @@ describe.skipIf(!hasRedis)('Redis window counter', () => {
redis = new Redis({ port, host: '127.0.0.1', lazyConnect: true, retryStrategy: () => 50 })
for (let i = 0; i < 100; i++) { try { await redis.connect(); break } catch { await new Promise((r) => setTimeout(r, 50)) } }
})
- afterAll(() => { redis?.disconnect(); server?.kill() })
+ afterAll(async () => {
+ const used = await redis?.keys(k('*')) ?? []
+ if (url && used.length) await redis.del(...used)
+ redis?.disconnect()
+ server?.kill()
+ })
it('counts each key in one round trip and returns the counts after this hit', async () => {
const count = redisWindowCounter(redis)
- const both = [{ key: 'rl:t:a:1', windowMs: 60_000 }, { key: 'rl:t:b:1', windowMs: 1_000 }]
+ const both = [{ key: k('a:1'), windowMs: 60_000 }, { key: k('b:1'), windowMs: 1_000 }]
expect(await count(both)).toEqual([1, 1])
expect(await count(both)).toEqual([2, 2])
- expect(await count([{ key: 'rl:t:a:1', windowMs: 60_000 }])).toEqual([3])
+ expect(await count([{ key: k('a:1'), windowMs: 60_000 }])).toEqual([3])
})
it('lets each key expire with its own window, so nothing is left behind', async () => {
const count = redisWindowCounter(redis)
- await count([{ key: 'rl:t:minute:1', windowMs: 60_000 }, { key: 'rl:t:second:1', windowMs: 1_000 }])
+ await count([{ key: k('minute:1'), windowMs: 60_000 }, { key: k('second:1'), windowMs: 1_000 }])
- const [minute, second] = [await redis.pttl('rl:t:minute:1'), await redis.pttl('rl:t:second:1')]
+ const [minute, second] = [await redis.pttl(k('minute:1')), await redis.pttl(k('second:1'))]
expect(minute).toBeGreaterThan(59_000)
expect(minute).toBeLessThanOrEqual(60_000)
diff --git a/backend/src/lib/terminalRouteRedis.test.ts b/backend/src/lib/terminalRouteRedis.test.ts
index 03865e2b8..5d481c62b 100644
--- a/backend/src/lib/terminalRouteRedis.test.ts
+++ b/backend/src/lib/terminalRouteRedis.test.ts
@@ -1,7 +1,8 @@
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import type { UpBusMsg } from './tunnel.js'
-// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI has none, so this file is skipped there). The bus
+// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI's Redis service; skipped without one, which CI's
+// redis-tests-ran step refuses). The bus
// connects to it as REDIS_URL. One process stands for several: a process's channel is just `termpod:`/`uppod:` + its id,
// so subscribing to another id here is what that other process would do.
const url = process.env.REDIS_TEST_URL
diff --git a/backend/src/superTabs/cache.ts b/backend/src/superTabs/cache.ts
index 95ac9d7b3..590d0134d 100644
--- a/backend/src/superTabs/cache.ts
+++ b/backend/src/superTabs/cache.ts
@@ -179,16 +179,25 @@ export class CacheView {
export const STAMP_TTL_MS = 30 * 86_400_000
const stampKey = (tabId: string) => `swarm:stamp:${tabId}`
/**
- * A Swarm's first bump starts both stamps from Redis's clock in ms, then moves the named ones by one. Redis losing the key
- * (a restart, the expiry) therefore never hands out a stamp a process may still hold: it starts again above it, as long as
- * a Swarm has fewer commits than milliseconds between its first and its loss.
+ * Moves each named stamp to Redis's clock in microseconds, or one past its value if that is higher, so a stamp only grows
+ * and no bump hands out a stamp already handed out: a process serves what it holds at an unchanged stamp (SwarmCache), and a
+ * delivery poll skips its revocation check while the main stamp holds (store.ts deliveries). No two bumps run in one
+ * microsecond (Redis runs one script at a time, each longer than that), so every stamp is its bump's time and none runs
+ * ahead of the clock. When Redis loses the key (a restart, the expiry), the next bump starts above every stamp handed out.
+ * When it comes back with an older value (a snapshot, a lagging replica), reads see that older stamp again until the next
+ * bump of each scope moves it above them all; held entries live at most CACHE_MAX_AGE_MS meanwhile. In milliseconds, a
+ * burst of bumps ran ahead of the clock and a loss within that millisecond handed a stamp out again (review of #2456). A first bump starts the stamps it does not name one
+ * below the clock. Stamps are compared only for equality: the jump from older millisecond stamps is one more change.
*/
-const BUMP = `if redis.call('exists', KEYS[1]) == 0 then
- local t = redis.call('time')
- local base = tonumber(t[1]) * 1000 + math.floor(tonumber(t[2]) / 1000)
- redis.call('hset', KEYS[1], 'main', base - 1, 'machine', base - 1)
+const BUMP = `local t = redis.call('time')
+local now = tonumber(t[1]) * 1000000 + tonumber(t[2])
+if redis.call('exists', KEYS[1]) == 0 then
+ redis.call('hset', KEYS[1], 'main', string.format('%.0f', now - 1), 'machine', string.format('%.0f', now - 1))
+end
+for i = 2, #ARGV do
+ local next = tonumber(redis.call('hget', KEYS[1], ARGV[i]) or 0) + 1
+ redis.call('hset', KEYS[1], ARGV[i], string.format('%.0f', math.max(next, now)))
end
-for i = 2, #ARGV do redis.call('hincrby', KEYS[1], ARGV[i], 1) end
redis.call('pexpire', KEYS[1], ARGV[1])
return 1`
diff --git a/backend/src/superTabs/cacheRedis.test.ts b/backend/src/superTabs/cacheRedis.test.ts
index 2c210caa5..6be650fec 100644
--- a/backend/src/superTabs/cacheRedis.test.ts
+++ b/backend/src/superTabs/cacheRedis.test.ts
@@ -3,8 +3,8 @@ import { Redis } from 'ioredis'
import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'
import { redisStamps, STAMP_TTL_MS, type StampScope } from './cache.js'
-// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI has none, so this file is skipped there). Only keys of
-// this run are touched, and they are removed afterwards.
+// A real Redis: REDIS_TEST_URL, e.g. redis://127.0.0.1:6379/15 (CI's Redis service; skipped without one, which CI's
+// redis-tests-ran step refuses). Only keys of this run are touched, and they are removed afterwards.
const url = process.env.REDIS_TEST_URL
// A real client: the shared setup's never connects (noRedis.test-support.ts).
vi.unmock('ioredis')
@@ -36,7 +36,9 @@ describe.skipIf(!url)('redisStamps', () => {
const before = Date.now()
await stamps.bump(id, ['main'])
const first = (await stamps.read([id])).get(id)!
- expect(first.main).toBeGreaterThan(before - 60_000)
+ // Redis's clock in microseconds.
+ expect(first.main).toBeGreaterThan((before - 60_000) * 1000)
+ expect(first.main).toBeLessThan((Date.now() + 60_000) * 1000)
expect(first.machine).toBe(first.main - 1)
await stamps.bump(id, ['main']); await stamps.bump(id, ['main'])
const old = (await stamps.read([id])).get(id)!.main
@@ -45,15 +47,57 @@ describe.skipIf(!url)('redisStamps', () => {
expect((await stamps.read([id])).get(id)!.main).toBeGreaterThan(old)
})
- it('moves only the scopes named, by one each', async () => {
+ // Review of #2456: a stamp the old Redis handed out must never come back for other data, or a process serves what it holds
+ // (cache.ts) and a delivery poll skips its revocation check (store.ts deliveries). Each bump is at least Redis's clock.
+ it('moves forward even after Redis comes back with an older value, as from a snapshot or a lagging replica', async () => {
+ const stamps = redisStamps(redis, bus), id = tab()
+ used.push(id)
+ await stamps.bump(id, ['main', 'machine'])
+ const older = (await stamps.read([id])).get(id)!
+ for (let i = 0; i < 5; i++) await stamps.bump(id, ['main', 'machine'])
+ const newest = (await stamps.read([id])).get(id)!
+ await redis.hset(`swarm:stamp:${id}`, 'main', older.main, 'machine', older.machine)
+ await stamps.bump(id, ['main', 'machine'])
+ const after = (await stamps.read([id])).get(id)!
+ expect(after.main).toBeGreaterThan(newest.main)
+ expect(after.machine).toBeGreaterThan(newest.machine)
+ })
+
+ it('moves each stamp strictly forward on every bump, many in one millisecond', async () => {
+ const stamps = redisStamps(redis, bus), id = tab()
+ used.push(id)
+ const seen: number[] = []
+ for (let i = 0; i < 50; i++) {
+ await stamps.bump(id, ['main'])
+ seen.push((await stamps.read([id])).get(id)!.main)
+ }
+ for (let i = 1; i < seen.length; i++) expect(seen[i]).toBeGreaterThan(seen[i - 1]!)
+ })
+
+ it('moves a stamp ahead of the clock by exactly one, never back to the clock', async () => {
+ const stamps = redisStamps(redis, bus), id = tab()
+ used.push(id)
+ await stamps.bump(id, ['main', 'machine'])
+ // As a burst or a faster clock could leave it: 10 s above Redis's clock, in microseconds.
+ const ahead = (Date.now() + 10_000) * 1000
+ await redis.hset(`swarm:stamp:${id}`, 'main', ahead, 'machine', ahead)
+ await stamps.bump(id, ['main'])
+ expect((await stamps.read([id])).get(id)).toEqual({ main: ahead + 1, machine: ahead })
+ })
+
+ it('moves only the scopes named', async () => {
const stamps = redisStamps(redis, bus), id = tab()
used.push(id)
await stamps.bump(id, ['main'])
const start = (await stamps.read([id])).get(id)!
await stamps.bump(id, ['machine'])
- expect((await stamps.read([id])).get(id)).toEqual({ main: start.main, machine: start.machine + 1 })
+ const machine = (await stamps.read([id])).get(id)!
+ expect(machine.main).toBe(start.main)
+ expect(machine.machine).toBeGreaterThan(start.machine)
await stamps.bump(id, ['main', 'machine'])
- expect((await stamps.read([id])).get(id)).toEqual({ main: start.main + 1, machine: start.machine + 2 })
+ const both = (await stamps.read([id])).get(id)!
+ expect(both.main).toBeGreaterThan(machine.main)
+ expect(both.machine).toBeGreaterThan(machine.machine)
})
it('reads several Swarms in one call, and lets a quiet Swarm\'s stamp expire', async () => {
diff --git a/docs/design/2026-10-28-swarm-read-cache.md b/docs/design/2026-10-28-swarm-read-cache.md
index 3f5cada71..6fb4a27e0 100644
--- a/docs/design/2026-10-28-swarm-read-cache.md
+++ b/docs/design/2026-10-28-swarm-read-cache.md
@@ -14,6 +14,11 @@ bumps), `backend/src/superTabs/cachedReads.ts` (reads built from what the cache
`changedAt`) therefore no longer moves every 15 s for every machine. Clients keep the part with the higher
version and never read `changedAt`.
- Touches change no data and move nothing.
+ - Each move sets the stamp to Redis's clock in microseconds, or one past its value if that is higher
+ (`cache.ts BUMP`). A stamp is therefore its bump's time and never runs ahead of the clock, so no bump hands out
+ a stamp already handed out, even after Redis lost the key or came back with an older value (a restart, a
+ snapshot, a lagging replica). After an older value, reads see it again until the next bump of each scope; held
+ entries live at most 60 s meanwhile. Millisecond stamps could repeat on a bump (review of #2456).
- **Bump after commit.** Every part write records its Swarm in the transaction (`parts/write.ts guardedWrite`,
`parts/load.ts insertParts`). `swarmTransaction` bumps those Swarms after the commit, before the caller answers. Any
read after the answer, on any process, therefore sees the change. This covers the store's commands, sweeps,