The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(devlog): a Redis that fails or comes back empty no longer reads as "no app used in months" (#990)
0xken154bb53122fa08
diff --git a/backend/src/lib/deviceKeyLog.test.ts b/backend/src/lib/deviceKeyLog.test.tsindex c6170184c..dadee3782 100644--- a/backend/src/lib/deviceKeyLog.test.ts+++ b/backend/src/lib/deviceKeyLog.test.ts@@ -52,6 +52,7 @@ vi.mock('./bus.js', () => ({const ops: Array<() => void> = []const chain = {hset: (key: string, field: string, value: string) => { ops.push(() => { m.hsetCalls++; (m.seen[key] ??= {})[field] = value }); return chain },+ hsetnx: (key: string, field: string, value: string) => { ops.push(() => { (m.seen[key] ??= {})[field] ??= value }); return chain },expire: () => chain,exec: async () => { for (const op of ops) op(); return [] },}@@ -290,14 +291,23 @@ describe('when a key was last seen', () => {touchDeviceKey('user-seen', pubKey, 1_000_000)touchDeviceKey('user-seen', pubKey, 1_000_000 + 60_000)await vi.waitFor(() => expect(m.hsetCalls).toBe(1))- expect(await deviceKeysSeen('user-seen')).toEqual({ [pubKey]: 1_000_000 })+ expect(await deviceKeysSeen('user-seen')).toEqual({ seen: { [pubKey]: 1_000_000 }, since: 1_000_000 })touchDeviceKey('user-seen', pubKey, 1_000_000 + 2 * 60 * 60 * 1000)await vi.waitFor(() => expect(m.hsetCalls).toBe(2))})- it('ignores anything that is not a key', async () => {+ it('keeps when the record began, which a later key does not move', async () => {+ touchDeviceKey('user-since', key(7).pub, 5_000)+ touchDeviceKey('user-since', key(8).pub, 9_000)+ await vi.waitFor(() => expect(m.hsetCalls).toBe(2))+ expect(await deviceKeysSeen('user-since')).toEqual({ seen: { [key(7).pub]: 5_000, [key(8).pub]: 9_000 }, since: 5_000 })+ })++ it('ignores anything that is not a key, and has no beginning when nothing was recorded', async () => {touchDeviceKey('user-seen-2', 'not a key')touchDeviceKey('user-seen-2', undefined)- expect(await deviceKeysSeen('user-seen-2')).toEqual({})+ expect(await deviceKeysSeen('user-seen-2')).toEqual({ seen: {}, since: null })+ m.seen['devkeys:seen:user-seen-3'] = { _since: 'x', junk: '5' }+ expect(await deviceKeysSeen('user-seen-3')).toEqual({ seen: {}, since: null })})})diff --git a/backend/src/lib/deviceKeyLog.ts b/backend/src/lib/deviceKeyLog.tsindex 295780996..d39ab910e 100644--- a/backend/src/lib/deviceKeyLog.ts+++ b/backend/src/lib/deviceKeyLog.ts@@ -203,6 +203,11 @@ async function signOut(userId: string, kind: string, machineId: string, pubKey:// this is what "seen" means: the key opened an E2EE session (`e2e_hello` carries it in the clear, on// its way to a machine). Only a hint for that list — nothing trusts or distrusts a key by it — so a// client claiming someone else's key here can at most keep that one off the suggestion.+//+// `_since` (in the same hash, so it goes when the hash goes — a TTL, an eviction, a Redis that comes+// back empty) is when this record began: a key with no time here is unused only if the record is+// older than the question. Without it, a Redis that lost its data read as "no app used in months",+// and a machine's sweep (cli deviceLogSyncer `sweepStale`) would have removed every old app key.const SEEN_TTL_SEC = 400 * 24 * 60 * 60/** One write per key per this long, per process: a phone opens a session per machine it shows. */@@ -211,6 +216,7 @@ const SEEN_MEMORY_MAX = 10_000const seenWritten = new Map<string, number>()const seenKey = (userId: string): string => `devkeys:seen:${userId}`const PUB_RE = /^[A-Za-z0-9+/]{43}=$/+const SINCE = '_since'export function touchDeviceKey(userId: string, pubKey: unknown, now = Date.now()): void {if (typeof pubKey !== 'string' || !PUB_RE.test(pubKey)) return@@ -219,17 +225,24 @@ export function touchDeviceKey(userId: string, pubKey: unknown, now = Date.now()if (last !== undefined && now - last < SEEN_WRITE_EVERY_MS) returnif (seenWritten.size >= SEEN_MEMORY_MAX) seenWritten.clear()seenWritten.set(memo, now)- void pub.multi().hset(seenKey(userId), pubKey, String(now)).expire(seenKey(userId), SEEN_TTL_SEC).exec()+ void pub.multi().hsetnx(seenKey(userId), SINCE, String(now)).hset(seenKey(userId), pubKey, String(now))+ .expire(seenKey(userId), SEEN_TTL_SEC).exec().catch(() => { seenWritten.delete(memo) })}-/** When each of the account's keys was last seen, by pub (ms). Empty when nothing was recorded. */-export async function deviceKeysSeen(userId: string): Promise<Record<string, number>> {+/**+ * When each of the account's keys was last seen, by pub (ms), and since when that record runs (null when+ * there is none). Throws when Redis cannot be read: "could not ask" must never read as "nothing seen".+ */+export async function deviceKeysSeen(userId: string): Promise<{ seen: Record<string, number>; since: number | null }> {const raw = await pub.hgetall(seenKey(userId))- const out: Record<string, number> = {}+ const seen: Record<string, number> = {}+ let since: number | null = nullfor (const [k, v] of Object.entries(raw ?? {})) {const n = Number(v)- if (PUB_RE.test(k) && Number.isSafeInteger(n) && n > 0) out[k] = n+ if (!Number.isSafeInteger(n) || n <= 0) continue+ if (k === SINCE) since = n+ else if (PUB_RE.test(k)) seen[k] = n}- return out+ return { seen, since }}diff --git a/backend/src/routes/deviceKeys.test.ts b/backend/src/routes/deviceKeys.test.tsindex 541eb3b4d..e56a7e5fd 100644--- a/backend/src/routes/deviceKeys.test.ts+++ b/backend/src/routes/deviceKeys.test.ts@@ -77,13 +77,15 @@ describe('device key routes', () => {expect(r.json().error.code).toBe('WRONG_CHANNEL')})- it('says when each key was last seen, and nothing when that cannot be read', async () => {- m.seen.mockResolvedValue({ k: 5 })+ it('says when each key was last seen, and fails — never "nothing seen" — when that cannot be read', async () => {+ m.seen.mockResolvedValue({ seen: { k: 5 }, since: 3 })const r = await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })- expect(r.json().data).toEqual({ seen: { k: 5 } })+ expect(r.json().data).toEqual({ seen: { k: 5 }, since: 3 })expect(m.seen).toHaveBeenCalledWith('user-1')m.seen.mockRejectedValue(new Error('redis down'))- expect((await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })).json().data).toEqual({ seen: {} })+ const down = await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })+ expect(down.statusCode).toBe(503)+ expect(down.json().error.code).toBe('SEEN_UNAVAILABLE')})it('limits how often one account appends', async () => {diff --git a/backend/src/routes/deviceKeys.ts b/backend/src/routes/deviceKeys.tsindex 2c7c08d2f..d1487ee3d 100644--- a/backend/src/routes/deviceKeys.ts+++ b/backend/src/routes/deviceKeys.ts@@ -42,13 +42,15 @@ export async function deviceKeyRoutes(app: FastifyInstance): Promise<void> {})// When each key last opened a session — a hint for the Devices list, which offers to remove apps not- // seen in a long while (a browser whose data was cleared never signs its own removal).+ // seen in a long while (a browser whose data was cleared never signs its own removal) — and `since`,+ // when that record began. A Redis that cannot be read is an error, never `{seen: {}}`: a machine's+ // sweep read that empty answer as "no app used in months" and would remove every old app key.app.get('/api/device-keys/seen', async (req, reply) => {reply.header('Cache-Control', 'no-store')try {- sendSuccess(reply, { seen: await deviceKeysSeen(req.user!.sub) })+ sendSuccess(reply, await deviceKeysSeen(req.user!.sub))} catch {- sendSuccess(reply, { seen: {} })+ sendError(reply, 'When your devices were last used cannot be read right now.', 'SEEN_UNAVAILABLE', 503)}})diff --git a/cli/src/gateway/start.ts b/cli/src/gateway/start.tsindex abcc5a07d..2d9f8e2d8 100644--- a/cli/src/gateway/start.ts+++ b/cli/src/gateway/start.ts@@ -26,7 +26,7 @@ import { thisDeviceLabel } from '../lib/daemonState.js'import { switchAccountTrust } from '../lib/e2ee/accountTrust.js'import { b64d, b64e, fingerprint } from '../lib/e2ee/core.js'import { DeviceLogStore } from '../lib/e2ee/deviceLogStore.js'-import { DeviceLogSyncer, type DeviceLogFetched, type DeviceLogTrustOutcome } from '../lib/e2ee/deviceLogSyncer.js'+import { DeviceLogSyncer, type DeviceKeysSeen, type DeviceLogFetched, type DeviceLogTrustOutcome } from '../lib/e2ee/deviceLogSyncer.js'import { GroupSyncer, relayRequester, SELF_STAMP } from '../lib/e2ee/groupSyncer.js'import { MachinePeerStore } from '../lib/e2ee/machinePeers.js'import { E2eeStore } from '../lib/e2ee/store.js'@@ -206,12 +206,14 @@ export function startGateway(host: GatewayHost): StartedGateway {syncer.start()}if (host.signedIn) { groupStarted = true; syncer.start() }- // When each key last opened a session or read the log, from the backend: the Devices list's "last- // active", and what `sweepStale` judges an unused app key by. null when it could not be asked.- const deviceKeysSeen = async (): Promise<Record<string, number> | null> => {+ // When each key last opened a session or read the log, from the backend, and since when that record+ // runs: the Devices list's "last active", and what `sweepStale` judges an unused app key by. null when+ // it could not be asked; `since` null from a backend that does not say.+ const deviceKeysSeen = async (): Promise<DeviceKeysSeen | null> => {const r = await host.backend('GET', '/api/device-keys/seen').catch(() => null)- const seen = r?.status === 200 ? (r.body.data as { seen?: unknown } | undefined)?.seen : undefined- return seen && typeof seen === 'object' ? seen as Record<string, number> : null+ const data = r?.status === 200 ? r.body.data as { seen?: unknown; since?: unknown } | undefined : undefined+ if (!data?.seen || typeof data.seen !== 'object') return null+ return { seen: data.seen as Record<string, number>, since: Number.isSafeInteger(data.since) ? data.since as number : null }}const devlog = new DeviceLogSyncer({store: new DeviceLogStore(),@@ -397,7 +399,7 @@ export function startGateway(host: GatewayHost): StartedGateway {devicesList: async () => {// When each key last opened a session, from the backend — a hint for removing apps not used in a// long while. Without it the list is still the list.- return { status: 200, body: { ...devlog.list(), lastSeen: await deviceKeysSeen() ?? {} } }+ return { status: 200, body: { ...devlog.list(), lastSeen: (await deviceKeysSeen())?.seen ?? {} } }},devicesRemove: async (pub) => {syncer.remove(pub)diff --git a/cli/src/lib/e2ee/deviceLogSyncer.spec.ts b/cli/src/lib/e2ee/deviceLogSyncer.spec.tsindex 7b1be564a..c959b736f 100644--- a/cli/src/lib/e2ee/deviceLogSyncer.spec.ts+++ b/cli/src/lib/e2ee/deviceLogSyncer.spec.ts@@ -8,7 +8,7 @@ import {type DevLogEntry, type DevLogState,} from './deviceLog.js'import { DeviceLogStore } from './deviceLogStore.js'-import { DeviceLogSyncer, devLogDivergence, type DeviceLogAppendAnswer, type DeviceLogFetched, type DeviceLogRebaseline } from './deviceLogSyncer.js'+import { DeviceLogSyncer, devLogDivergence, type DeviceKeysSeen, type DeviceLogAppendAnswer, type DeviceLogFetched, type DeviceLogRebaseline } from './deviceLogSyncer.js'import { ADOPTED_SIGN_IN } from '../authSession.js'import { b64d, fingerprint } from './core.js'@@ -73,7 +73,7 @@ class FakeBackend {function setup(opts: {known?: string[]; tombstoned?: string[]; blocked?: string[]; store?: DeviceLogStore; backend?: FakeBackend; signIn?: () => string | null; signInAt?: number | nullisTrusted?: (pub: string) => boolean; signInAcct?: () => string | null; switchAccount?: (from: string, to: string) => void- now?: () => number; seen?: () => Promise<Record<string, number> | null>; log?: (line: string) => void+ now?: () => number; seen?: () => Promise<DeviceKeysSeen | null>; log?: (line: string) => void} = {}) {const backend = opts.backend ?? new FakeBackend()const store = opts.store ?? new DeviceLogStore(join(mkdtempSync(join(tmpdir(), 'devlog-')), 'devlog.json'))@@ -2123,13 +2123,16 @@ describe('DeviceLogSyncer.sweepStale', () => {const larger = pool.find((k) => k.pub > me.pub)!let clock: numberlet seen: Record<string, number> | null+ /** Since when the backend's record of last uses runs (null: it does not say). */+ let since: number | nulllet lines: string[]let t: ReturnType<typeof setup>beforeEach(() => {clock = NOWseen = {}+ since = 1_000lines = []- t = setup({ now: () => clock, seen: async () => seen, log: (l) => lines.push(l) })+ t = setup({ now: () => clock, seen: async () => seen && { seen, since }, log: (l) => lines.push(l) })})const active = (pub: string): boolean => !!t.backend.state.active[pub]@@ -2147,20 +2150,42 @@ describe('DeviceLogSyncer.sweepStale', () => {expect(active(stale.pub)).toBe(false)expect([usedLately, young, suspended, larger, me].every((k) => active(k.pub))).toBe(true)expect(t.backend.entries.at(-1)).toMatchObject({ op: 'remove', pub: stale.pub, signer: me.pub })- expect(lines).toContain(`[devlog] removed unused device old-browser (${fingerprint(b64d(stale.pub))}), last used 1970-01-01`)+ expect(lines).toContain(`[devlog] removed unused device old-browser (${fingerprint(b64d(stale.pub))}), not used since 1970-01-01`)})- it('a key the backend has no last use for counts as used when it was added', async () => {- const recent = key(50)- t.backend.add(recent, 'viewer', '', 'app', NOW - 200 * DAY)- seen = { [recent.pub]: NOW - 100 * DAY }+ it('a key last used over 180 days ago goes whatever the record says of the others', async () => {+ const old = key(50)+ t.backend.add(old, 'viewer', '', 'app', 1_000)+ seen = { [old.pub]: 2_000 }+ since = nullawait t.syncer.register()await t.syncer.sweepStale()- expect(active(recent.pub)).toBe(true)+ expect(active(old.pub)).toBe(false)+ expect(lines).toContain(`[devlog] removed unused device app (${fingerprint(b64d(old.pub))}), last used 1970-01-01`)+ })++ it('a key with no last use is unused only when the record has run since before the cutoff', async () => {+ // A Redis that came back empty (restart, eviction) answers no last use for anyone: that is "not+ // known", not "not used" — an app opened once a month must not be signed out by it.+ const app = key(50)+ t.backend.add(app, 'viewer', '', 'app', NOW - 200 * DAY)+ seen = { [app.pub]: NOW - 100 * DAY }+ await t.syncer.register()+ await t.syncer.sweepStale()+ expect(active(app.pub)).toBe(true)seen = {}+ since = null // the record is gone, or a backend that does not say+ clock += 7 * 60 * 60_000+ await t.syncer.sweepStale()+ expect(active(app.pub)).toBe(true)+ since = clock - 10 * DAY // the record started again 10 days ago+ clock += 7 * 60 * 60_000+ await t.syncer.sweepStale()+ expect(active(app.pub)).toBe(true)+ since = NOW - 190 * DAY // it has run for longer than the cutoff, and never saw this keyclock += 7 * 60 * 60_000await t.syncer.sweepStale()- expect(active(recent.pub)).toBe(false)+ expect(active(app.pub)).toBe(false)})it('the machine with the smallest key goes first; the next takes over a day later', async () => {diff --git a/cli/src/lib/e2ee/deviceLogSyncer.ts b/cli/src/lib/e2ee/deviceLogSyncer.tsindex 7a3756fc0..f9c64fb0e 100644--- a/cli/src/lib/e2ee/deviceLogSyncer.ts+++ b/cli/src/lib/e2ee/deviceLogSyncer.ts@@ -96,9 +96,9 @@ export interface DeviceLogSyncerDeps {resume?: () => void/** This machine's own key was removed from the log: it is signed out. */signedOut: () => void- /** When each key last opened a session or read the log (ms), from the backend; null when it could not+ /** When each key last opened a session or read the log, from the backend; null when it could not* be asked. Only `sweepStale` reads it. */- seen?: () => Promise<Record<string, number> | null>+ seen?: () => Promise<DeviceKeysSeen | null>/** Something a window shows changed (the list, the frozen line). */changed?: () => voidlog?: (line: string) => void@@ -181,6 +181,10 @@ const JOIN_PAGES = 200* a browser whose storage was cleared, an app reinstalled, never comes back for its key, and at 256* active keys (DEVLOG_MAX_ACTIVE) no new device can join the account. */export const STALE_AFTER_MS = 180 * 24 * 60 * 60_000+/** When each key was last used (ms, by pub), and since when that record runs (null: unknown). A key with+ * no time is unused only if the record is older than the question: a Redis that came back empty once+ * read as "no app used in months", and the sweep would have removed every old app key. */+export interface DeviceKeysSeen { seen: Record<string, number>; since: number | null }/** At most one sweep per this long on a machine; the timer that offers it runs every 10 minutes. */const SWEEP_EVERY_MS = 6 * 60 * 60_000/** At most this many removals per sweep (the next one goes on), each SWEEP_GAP_MS apart: the backend@@ -195,6 +199,7 @@ const GOSSIP_HASHES = 64interface Known { label: string; kind: DevLogMember['kind'] }const fp = (pub: string): string => fingerprint(b64d(pub))+const day = (ms: number): string => new Date(ms).toISOString().slice(0, 10)const uniq = (xs: readonly string[]): string[] => [...new Set(xs)]/**@@ -828,14 +833,22 @@ export class DeviceLogSyncer {const turn = machines.indexOf(selfPub)if (turn < 0) returnthis.lastSweep = now- const lastUsed = await seen()+ const got = await seen()// Not asked: try again on the next offer rather than read every key as never used.- if (!lastUsed) { this.lastSweep = null; return }+ if (!got) { this.lastSweep = null; return }const cutoff = now - STALE_AFTER_MS - turn * SWEEP_TURN_MSconst suspended = this.suspendedKeys()- const usedAt = (m: DevLogMember): number => { const t = lastUsed[m.pub]; return typeof t === 'number' && Number.isFinite(t) ? t : m.addedAt }+ // A key with no time is unused only when the record has run since before the cutoff.+ const usedAt = (m: DevLogMember): number | null => {+ const t = got.seen[m.pub]+ return typeof t === 'number' && Number.isFinite(t) ? t : null+ }+ const unusedSinceCutoff = (m: DevLogMember): boolean => {+ const t = usedAt(m)+ return t !== null ? t < cutoff : got.since !== null && got.since < cutoff+ }const stale = Object.values(state.active)- .filter((m) => m.kind === 'viewer' && m.pub !== selfPub && !suspended.includes(m.pub) && m.addedAt < cutoff && usedAt(m) < cutoff)+ .filter((m) => m.kind === 'viewer' && m.pub !== selfPub && !suspended.includes(m.pub) && m.addedAt < cutoff && unusedSinceCutoff(m)).sort((a, b) => a.seq - b.seq).slice(0, SWEEP_MAX)for (const [i, m] of stale.entries()) {@@ -844,7 +857,9 @@ export class DeviceLogSyncer {// Already out of the log (another machine's sweep, a person's own removal): nothing to do.if (!r.ok && r.error === 'NOT_IN_LOG') continueif (!r.ok) { this.deps.log?.(`[devlog] could not remove unused device ${m.label || '(no name)'} (${fp(m.pub)}): ${r.detail ?? r.error}`); return }- this.deps.log?.(`[devlog] removed unused device ${m.label || '(no name)'} (${fp(m.pub)}), last used ${new Date(usedAt(m)).toISOString().slice(0, 10)}`)+ const t = usedAt(m)+ const when = t !== null ? `last used ${day(t)}` : `not used since ${day(got.since!)}`+ this.deps.log?.(`[devlog] removed unused device ${m.label || '(no name)'} (${fp(m.pub)}), ${when}`)}} catch (err) {this.deps.log?.(`[devlog] sweep failed: ${err instanceof Error ? err.message : String(err)}`)