Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(devlog): a Redis that fails or comes back empty no longer reads as "no app used in months" (#990)

0xken154bb53122fa08
diff --git a/backend/src/lib/deviceKeyLog.test.ts b/backend/src/lib/deviceKeyLog.test.ts
index c6170184c..dadee3782 100644
--- a/backend/src/lib/deviceKeyLog.test.ts
+++ b/backend/src/lib/deviceKeyLog.test.ts
@@ -52,6 +52,7 @@ vi.mock('./bus.js', () => ({
const ops: Array<() => void> = []
const chain = {
hset: (key: string, field: string, value: string) => { ops.push(() => { m.hsetCalls++; (m.seen[key] ??= {})[field] = value }); return chain },
+ hsetnx: (key: string, field: string, value: string) => { ops.push(() => { (m.seen[key] ??= {})[field] ??= value }); return chain },
expire: () => chain,
exec: async () => { for (const op of ops) op(); return [] },
}
@@ -290,14 +291,23 @@ describe('when a key was last seen', () => {
touchDeviceKey('user-seen', pubKey, 1_000_000)
touchDeviceKey('user-seen', pubKey, 1_000_000 + 60_000)
await vi.waitFor(() => expect(m.hsetCalls).toBe(1))
- expect(await deviceKeysSeen('user-seen')).toEqual({ [pubKey]: 1_000_000 })
+ expect(await deviceKeysSeen('user-seen')).toEqual({ seen: { [pubKey]: 1_000_000 }, since: 1_000_000 })
touchDeviceKey('user-seen', pubKey, 1_000_000 + 2 * 60 * 60 * 1000)
await vi.waitFor(() => expect(m.hsetCalls).toBe(2))
})
- it('ignores anything that is not a key', async () => {
+ it('keeps when the record began, which a later key does not move', async () => {
+ touchDeviceKey('user-since', key(7).pub, 5_000)
+ touchDeviceKey('user-since', key(8).pub, 9_000)
+ await vi.waitFor(() => expect(m.hsetCalls).toBe(2))
+ expect(await deviceKeysSeen('user-since')).toEqual({ seen: { [key(7).pub]: 5_000, [key(8).pub]: 9_000 }, since: 5_000 })
+ })
+
+ it('ignores anything that is not a key, and has no beginning when nothing was recorded', async () => {
touchDeviceKey('user-seen-2', 'not a key')
touchDeviceKey('user-seen-2', undefined)
- expect(await deviceKeysSeen('user-seen-2')).toEqual({})
+ expect(await deviceKeysSeen('user-seen-2')).toEqual({ seen: {}, since: null })
+ m.seen['devkeys:seen:user-seen-3'] = { _since: 'x', junk: '5' }
+ expect(await deviceKeysSeen('user-seen-3')).toEqual({ seen: {}, since: null })
})
})
diff --git a/backend/src/lib/deviceKeyLog.ts b/backend/src/lib/deviceKeyLog.ts
index 295780996..d39ab910e 100644
--- a/backend/src/lib/deviceKeyLog.ts
+++ b/backend/src/lib/deviceKeyLog.ts
@@ -203,6 +203,11 @@ async function signOut(userId: string, kind: string, machineId: string, pubKey:
// this is what "seen" means: the key opened an E2EE session (`e2e_hello` carries it in the clear, on
// its way to a machine). Only a hint for that list — nothing trusts or distrusts a key by it — so a
// client claiming someone else's key here can at most keep that one off the suggestion.
+//
+// `_since` (in the same hash, so it goes when the hash goes — a TTL, an eviction, a Redis that comes
+// back empty) is when this record began: a key with no time here is unused only if the record is
+// older than the question. Without it, a Redis that lost its data read as "no app used in months",
+// and a machine's sweep (cli deviceLogSyncer `sweepStale`) would have removed every old app key.
const SEEN_TTL_SEC = 400 * 24 * 60 * 60
/** One write per key per this long, per process: a phone opens a session per machine it shows. */
@@ -211,6 +216,7 @@ const SEEN_MEMORY_MAX = 10_000
const seenWritten = new Map<string, number>()
const seenKey = (userId: string): string => `devkeys:seen:${userId}`
const PUB_RE = /^[A-Za-z0-9+/]{43}=$/
+const SINCE = '_since'
export function touchDeviceKey(userId: string, pubKey: unknown, now = Date.now()): void {
if (typeof pubKey !== 'string' || !PUB_RE.test(pubKey)) return
@@ -219,17 +225,24 @@ export function touchDeviceKey(userId: string, pubKey: unknown, now = Date.now()
if (last !== undefined && now - last < SEEN_WRITE_EVERY_MS) return
if (seenWritten.size >= SEEN_MEMORY_MAX) seenWritten.clear()
seenWritten.set(memo, now)
- void pub.multi().hset(seenKey(userId), pubKey, String(now)).expire(seenKey(userId), SEEN_TTL_SEC).exec()
+ void pub.multi().hsetnx(seenKey(userId), SINCE, String(now)).hset(seenKey(userId), pubKey, String(now))
+ .expire(seenKey(userId), SEEN_TTL_SEC).exec()
.catch(() => { seenWritten.delete(memo) })
}
-/** When each of the account's keys was last seen, by pub (ms). Empty when nothing was recorded. */
-export async function deviceKeysSeen(userId: string): Promise<Record<string, number>> {
+/**
+ * When each of the account's keys was last seen, by pub (ms), and since when that record runs (null when
+ * there is none). Throws when Redis cannot be read: "could not ask" must never read as "nothing seen".
+ */
+export async function deviceKeysSeen(userId: string): Promise<{ seen: Record<string, number>; since: number | null }> {
const raw = await pub.hgetall(seenKey(userId))
- const out: Record<string, number> = {}
+ const seen: Record<string, number> = {}
+ let since: number | null = null
for (const [k, v] of Object.entries(raw ?? {})) {
const n = Number(v)
- if (PUB_RE.test(k) && Number.isSafeInteger(n) && n > 0) out[k] = n
+ if (!Number.isSafeInteger(n) || n <= 0) continue
+ if (k === SINCE) since = n
+ else if (PUB_RE.test(k)) seen[k] = n
}
- return out
+ return { seen, since }
}
diff --git a/backend/src/routes/deviceKeys.test.ts b/backend/src/routes/deviceKeys.test.ts
index 541eb3b4d..e56a7e5fd 100644
--- a/backend/src/routes/deviceKeys.test.ts
+++ b/backend/src/routes/deviceKeys.test.ts
@@ -77,13 +77,15 @@ describe('device key routes', () => {
expect(r.json().error.code).toBe('WRONG_CHANNEL')
})
- it('says when each key was last seen, and nothing when that cannot be read', async () => {
- m.seen.mockResolvedValue({ k: 5 })
+ it('says when each key was last seen, and fails — never "nothing seen" — when that cannot be read', async () => {
+ m.seen.mockResolvedValue({ seen: { k: 5 }, since: 3 })
const r = await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })
- expect(r.json().data).toEqual({ seen: { k: 5 } })
+ expect(r.json().data).toEqual({ seen: { k: 5 }, since: 3 })
expect(m.seen).toHaveBeenCalledWith('user-1')
m.seen.mockRejectedValue(new Error('redis down'))
- expect((await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })).json().data).toEqual({ seen: {} })
+ const down = await app.inject({ method: 'GET', url: '/api/device-keys/seen', headers: auth })
+ expect(down.statusCode).toBe(503)
+ expect(down.json().error.code).toBe('SEEN_UNAVAILABLE')
})
it('limits how often one account appends', async () => {
diff --git a/backend/src/routes/deviceKeys.ts b/backend/src/routes/deviceKeys.ts
index 2c7c08d2f..d1487ee3d 100644
--- a/backend/src/routes/deviceKeys.ts
+++ b/backend/src/routes/deviceKeys.ts
@@ -42,13 +42,15 @@ export async function deviceKeyRoutes(app: FastifyInstance): Promise<void> {
})
// When each key last opened a session — a hint for the Devices list, which offers to remove apps not
- // seen in a long while (a browser whose data was cleared never signs its own removal).
+ // seen in a long while (a browser whose data was cleared never signs its own removal) — and `since`,
+ // when that record began. A Redis that cannot be read is an error, never `{seen: {}}`: a machine's
+ // sweep read that empty answer as "no app used in months" and would remove every old app key.
app.get('/api/device-keys/seen', async (req, reply) => {
reply.header('Cache-Control', 'no-store')
try {
- sendSuccess(reply, { seen: await deviceKeysSeen(req.user!.sub) })
+ sendSuccess(reply, await deviceKeysSeen(req.user!.sub))
} catch {
- sendSuccess(reply, { seen: {} })
+ sendError(reply, 'When your devices were last used cannot be read right now.', 'SEEN_UNAVAILABLE', 503)
}
})
diff --git a/cli/src/gateway/start.ts b/cli/src/gateway/start.ts
index abcc5a07d..2d9f8e2d8 100644
--- a/cli/src/gateway/start.ts
+++ b/cli/src/gateway/start.ts
@@ -26,7 +26,7 @@ import { thisDeviceLabel } from '../lib/daemonState.js'
import { switchAccountTrust } from '../lib/e2ee/accountTrust.js'
import { b64d, b64e, fingerprint } from '../lib/e2ee/core.js'
import { DeviceLogStore } from '../lib/e2ee/deviceLogStore.js'
-import { DeviceLogSyncer, type DeviceLogFetched, type DeviceLogTrustOutcome } from '../lib/e2ee/deviceLogSyncer.js'
+import { DeviceLogSyncer, type DeviceKeysSeen, type DeviceLogFetched, type DeviceLogTrustOutcome } from '../lib/e2ee/deviceLogSyncer.js'
import { GroupSyncer, relayRequester, SELF_STAMP } from '../lib/e2ee/groupSyncer.js'
import { MachinePeerStore } from '../lib/e2ee/machinePeers.js'
import { E2eeStore } from '../lib/e2ee/store.js'
@@ -206,12 +206,14 @@ export function startGateway(host: GatewayHost): StartedGateway {
syncer.start()
}
if (host.signedIn) { groupStarted = true; syncer.start() }
- // When each key last opened a session or read the log, from the backend: the Devices list's "last
- // active", and what `sweepStale` judges an unused app key by. null when it could not be asked.
- const deviceKeysSeen = async (): Promise<Record<string, number> | null> => {
+ // When each key last opened a session or read the log, from the backend, and since when that record
+ // runs: the Devices list's "last active", and what `sweepStale` judges an unused app key by. null when
+ // it could not be asked; `since` null from a backend that does not say.
+ const deviceKeysSeen = async (): Promise<DeviceKeysSeen | null> => {
const r = await host.backend('GET', '/api/device-keys/seen').catch(() => null)
- const seen = r?.status === 200 ? (r.body.data as { seen?: unknown } | undefined)?.seen : undefined
- return seen && typeof seen === 'object' ? seen as Record<string, number> : null
+ const data = r?.status === 200 ? r.body.data as { seen?: unknown; since?: unknown } | undefined : undefined
+ if (!data?.seen || typeof data.seen !== 'object') return null
+ return { seen: data.seen as Record<string, number>, since: Number.isSafeInteger(data.since) ? data.since as number : null }
}
const devlog = new DeviceLogSyncer({
store: new DeviceLogStore(),
@@ -397,7 +399,7 @@ export function startGateway(host: GatewayHost): StartedGateway {
devicesList: async () => {
// When each key last opened a session, from the backend — a hint for removing apps not used in a
// long while. Without it the list is still the list.
- return { status: 200, body: { ...devlog.list(), lastSeen: await deviceKeysSeen() ?? {} } }
+ return { status: 200, body: { ...devlog.list(), lastSeen: (await deviceKeysSeen())?.seen ?? {} } }
},
devicesRemove: async (pub) => {
syncer.remove(pub)
diff --git a/cli/src/lib/e2ee/deviceLogSyncer.spec.ts b/cli/src/lib/e2ee/deviceLogSyncer.spec.ts
index 7b1be564a..c959b736f 100644
--- a/cli/src/lib/e2ee/deviceLogSyncer.spec.ts
+++ b/cli/src/lib/e2ee/deviceLogSyncer.spec.ts
@@ -8,7 +8,7 @@ import {
type DevLogEntry, type DevLogState,
} from './deviceLog.js'
import { DeviceLogStore } from './deviceLogStore.js'
-import { DeviceLogSyncer, devLogDivergence, type DeviceLogAppendAnswer, type DeviceLogFetched, type DeviceLogRebaseline } from './deviceLogSyncer.js'
+import { DeviceLogSyncer, devLogDivergence, type DeviceKeysSeen, type DeviceLogAppendAnswer, type DeviceLogFetched, type DeviceLogRebaseline } from './deviceLogSyncer.js'
import { ADOPTED_SIGN_IN } from '../authSession.js'
import { b64d, fingerprint } from './core.js'
@@ -73,7 +73,7 @@ class FakeBackend {
function setup(opts: {
known?: string[]; tombstoned?: string[]; blocked?: string[]; store?: DeviceLogStore; backend?: FakeBackend; signIn?: () => string | null; signInAt?: number | null
isTrusted?: (pub: string) => boolean; signInAcct?: () => string | null; switchAccount?: (from: string, to: string) => void
- now?: () => number; seen?: () => Promise<Record<string, number> | null>; log?: (line: string) => void
+ now?: () => number; seen?: () => Promise<DeviceKeysSeen | null>; log?: (line: string) => void
} = {}) {
const backend = opts.backend ?? new FakeBackend()
const store = opts.store ?? new DeviceLogStore(join(mkdtempSync(join(tmpdir(), 'devlog-')), 'devlog.json'))
@@ -2123,13 +2123,16 @@ describe('DeviceLogSyncer.sweepStale', () => {
const larger = pool.find((k) => k.pub > me.pub)!
let clock: number
let seen: Record<string, number> | null
+ /** Since when the backend's record of last uses runs (null: it does not say). */
+ let since: number | null
let lines: string[]
let t: ReturnType<typeof setup>
beforeEach(() => {
clock = NOW
seen = {}
+ since = 1_000
lines = []
- t = setup({ now: () => clock, seen: async () => seen, log: (l) => lines.push(l) })
+ t = setup({ now: () => clock, seen: async () => seen && { seen, since }, log: (l) => lines.push(l) })
})
const active = (pub: string): boolean => !!t.backend.state.active[pub]
@@ -2147,20 +2150,42 @@ describe('DeviceLogSyncer.sweepStale', () => {
expect(active(stale.pub)).toBe(false)
expect([usedLately, young, suspended, larger, me].every((k) => active(k.pub))).toBe(true)
expect(t.backend.entries.at(-1)).toMatchObject({ op: 'remove', pub: stale.pub, signer: me.pub })
- expect(lines).toContain(`[devlog] removed unused device old-browser (${fingerprint(b64d(stale.pub))}), last used 1970-01-01`)
+ expect(lines).toContain(`[devlog] removed unused device old-browser (${fingerprint(b64d(stale.pub))}), not used since 1970-01-01`)
})
- it('a key the backend has no last use for counts as used when it was added', async () => {
- const recent = key(50)
- t.backend.add(recent, 'viewer', '', 'app', NOW - 200 * DAY)
- seen = { [recent.pub]: NOW - 100 * DAY }
+ it('a key last used over 180 days ago goes whatever the record says of the others', async () => {
+ const old = key(50)
+ t.backend.add(old, 'viewer', '', 'app', 1_000)
+ seen = { [old.pub]: 2_000 }
+ since = null
await t.syncer.register()
await t.syncer.sweepStale()
- expect(active(recent.pub)).toBe(true)
+ expect(active(old.pub)).toBe(false)
+ expect(lines).toContain(`[devlog] removed unused device app (${fingerprint(b64d(old.pub))}), last used 1970-01-01`)
+ })
+
+ it('a key with no last use is unused only when the record has run since before the cutoff', async () => {
+ // A Redis that came back empty (restart, eviction) answers no last use for anyone: that is "not
+ // known", not "not used" — an app opened once a month must not be signed out by it.
+ const app = key(50)
+ t.backend.add(app, 'viewer', '', 'app', NOW - 200 * DAY)
+ seen = { [app.pub]: NOW - 100 * DAY }
+ await t.syncer.register()
+ await t.syncer.sweepStale()
+ expect(active(app.pub)).toBe(true)
seen = {}
+ since = null // the record is gone, or a backend that does not say
+ clock += 7 * 60 * 60_000
+ await t.syncer.sweepStale()
+ expect(active(app.pub)).toBe(true)
+ since = clock - 10 * DAY // the record started again 10 days ago
+ clock += 7 * 60 * 60_000
+ await t.syncer.sweepStale()
+ expect(active(app.pub)).toBe(true)
+ since = NOW - 190 * DAY // it has run for longer than the cutoff, and never saw this key
clock += 7 * 60 * 60_000
await t.syncer.sweepStale()
- expect(active(recent.pub)).toBe(false)
+ expect(active(app.pub)).toBe(false)
})
it('the machine with the smallest key goes first; the next takes over a day later', async () => {
diff --git a/cli/src/lib/e2ee/deviceLogSyncer.ts b/cli/src/lib/e2ee/deviceLogSyncer.ts
index 7a3756fc0..f9c64fb0e 100644
--- a/cli/src/lib/e2ee/deviceLogSyncer.ts
+++ b/cli/src/lib/e2ee/deviceLogSyncer.ts
@@ -96,9 +96,9 @@ export interface DeviceLogSyncerDeps {
resume?: () => void
/** This machine's own key was removed from the log: it is signed out. */
signedOut: () => void
- /** When each key last opened a session or read the log (ms), from the backend; null when it could not
+ /** When each key last opened a session or read the log, from the backend; null when it could not
* be asked. Only `sweepStale` reads it. */
- seen?: () => Promise<Record<string, number> | null>
+ seen?: () => Promise<DeviceKeysSeen | null>
/** Something a window shows changed (the list, the frozen line). */
changed?: () => void
log?: (line: string) => void
@@ -181,6 +181,10 @@ const JOIN_PAGES = 200
* a browser whose storage was cleared, an app reinstalled, never comes back for its key, and at 256
* active keys (DEVLOG_MAX_ACTIVE) no new device can join the account. */
export const STALE_AFTER_MS = 180 * 24 * 60 * 60_000
+/** When each key was last used (ms, by pub), and since when that record runs (null: unknown). A key with
+ * no time is unused only if the record is older than the question: a Redis that came back empty once
+ * read as "no app used in months", and the sweep would have removed every old app key. */
+export interface DeviceKeysSeen { seen: Record<string, number>; since: number | null }
/** At most one sweep per this long on a machine; the timer that offers it runs every 10 minutes. */
const SWEEP_EVERY_MS = 6 * 60 * 60_000
/** At most this many removals per sweep (the next one goes on), each SWEEP_GAP_MS apart: the backend
@@ -195,6 +199,7 @@ const GOSSIP_HASHES = 64
interface Known { label: string; kind: DevLogMember['kind'] }
const fp = (pub: string): string => fingerprint(b64d(pub))
+const day = (ms: number): string => new Date(ms).toISOString().slice(0, 10)
const uniq = (xs: readonly string[]): string[] => [...new Set(xs)]
/**
@@ -828,14 +833,22 @@ export class DeviceLogSyncer {
const turn = machines.indexOf(selfPub)
if (turn < 0) return
this.lastSweep = now
- const lastUsed = await seen()
+ const got = await seen()
// Not asked: try again on the next offer rather than read every key as never used.
- if (!lastUsed) { this.lastSweep = null; return }
+ if (!got) { this.lastSweep = null; return }
const cutoff = now - STALE_AFTER_MS - turn * SWEEP_TURN_MS
const suspended = this.suspendedKeys()
- const usedAt = (m: DevLogMember): number => { const t = lastUsed[m.pub]; return typeof t === 'number' && Number.isFinite(t) ? t : m.addedAt }
+ // A key with no time is unused only when the record has run since before the cutoff.
+ const usedAt = (m: DevLogMember): number | null => {
+ const t = got.seen[m.pub]
+ return typeof t === 'number' && Number.isFinite(t) ? t : null
+ }
+ const unusedSinceCutoff = (m: DevLogMember): boolean => {
+ const t = usedAt(m)
+ return t !== null ? t < cutoff : got.since !== null && got.since < cutoff
+ }
const stale = Object.values(state.active)
- .filter((m) => m.kind === 'viewer' && m.pub !== selfPub && !suspended.includes(m.pub) && m.addedAt < cutoff && usedAt(m) < cutoff)
+ .filter((m) => m.kind === 'viewer' && m.pub !== selfPub && !suspended.includes(m.pub) && m.addedAt < cutoff && unusedSinceCutoff(m))
.sort((a, b) => a.seq - b.seq)
.slice(0, SWEEP_MAX)
for (const [i, m] of stale.entries()) {
@@ -844,7 +857,9 @@ export class DeviceLogSyncer {
// Already out of the log (another machine's sweep, a person's own removal): nothing to do.
if (!r.ok && r.error === 'NOT_IN_LOG') continue
if (!r.ok) { this.deps.log?.(`[devlog] could not remove unused device ${m.label || '(no name)'} (${fp(m.pub)}): ${r.detail ?? r.error}`); return }
- this.deps.log?.(`[devlog] removed unused device ${m.label || '(no name)'} (${fp(m.pub)}), last used ${new Date(usedAt(m)).toISOString().slice(0, 10)}`)
+ const t = usedAt(m)
+ const when = t !== null ? `last used ${day(t)}` : `not used since ${day(got.since!)}`
+ this.deps.log?.(`[devlog] removed unused device ${m.label || '(no name)'} (${fp(m.pub)}), ${when}`)
}
} catch (err) {
this.deps.log?.(`[devlog] sweep failed: ${err instanceof Error ? err.message : String(err)}`)