Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: bound the usage reconciler's claim ledger instead of growing it forever (#12605)
[._.]/ Adam Eivy4dd8b83611dc
diff --git a/server/services/usageBackfillWorker.test.js b/server/services/usageBackfillWorker.test.jsindex 2c172b0097..172967f34a 100644--- a/server/services/usageBackfillWorker.test.js+++ b/server/services/usageBackfillWorker.test.js@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';import { join } from 'path';import { tmpdir } from 'os';-import { __resetUsageClaims } from './usageReconciler.js';+import { __resetUsageClaims, pruneUsageClaims, readMeasuredUsage, snapshotUsageClaims } from './usageReconciler.js';import { scanHistoricalUsage } from './usageBackfillWorker.js';let root;@@ -96,6 +96,27 @@ describe('historical usage worker', () => {expect(result.total).toBe(0);expect(result.corrections).toEqual([]);});+ it('does not re-bill a message the live path claimed, even after a prune', async () => {+ const window = { startTime: '2026-07-01T10:00:00.000Z', endTime: '2026-07-01T10:10:00.000Z' };+ const live = await readMeasuredUsage({ workspacePath: workspace, ...window, family: 'claude', home });+ expect(live?.tokensOut).toBe(200);+ // A prune inside the retention bound keeps the claim for the hand-off.+ pruneUsageClaims(Date.parse(window.endTime) + 60 * 60 * 1000);+ const claimsSeed = snapshotUsageClaims();+ // The worker thread starts from its own empty copy of the ledger.+ __resetUsageClaims();+ await writeRun('run-overlapping-history', {+ providerId: 'claude-code-tui',+ model: 'claude-opus-5',+ workspacePath: workspace,+ startTime: '2026-07-01T10:02:00.000Z',+ endTime: '2026-07-01T10:12:00.000Z'+ });++ const result = await scanHistoricalUsage({ runsDir: root, home, claimsSeed });+ expect(result.corrections).toEqual([]);+ expect(result.claimsSnapshot.claimedMessages).toEqual(claimsSeed.claimedMessages);+ });});// A nested `--review-with grok` pass leaves a grok session but never a PortOSdiff --git a/server/services/usageReconciler.js b/server/services/usageReconciler.jsindex 1f275872e1..f033e3505e 100644--- a/server/services/usageReconciler.js+++ b/server/services/usageReconciler.js@@ -75,17 +75,48 @@ import { markUsageRunReconciled, recordRunUsage } from './usage.js';// per-message claim below, not by the window.export const WINDOW_SLACK_MS = 60_000;-// Messages already billed to a run, keyed `<transcript-key>:<message-key>`.+// How long a claim outlives the window that made it. A later run can only+// re-count a message whose timestamp falls inside ITS window, so a claim is+// needed only while some run still to be reconciled can start before the+// message. Every run path is bounded far below this (the CoS max-runtime+// ceiling, the toolkit's API run cap), and the ledger never sees in-flight+// runs, so a fixed bound stands in for "oldest pending run start". A run that+// outlives it can re-bill a pruned neighbour's overlap — the same exposure a+// server restart, which empties the ledger, already accepts.+export const CLAIM_RETENTION_MS = 48 * 60 * 60 * 1000;++// Messages already billed to a run: transcript key -> message key -> stamp.// A transcript message must be counted exactly ONCE across every run that can// see it: without this, two concurrent runs in the same cwd (or one run whose// window overlaps a neighbour's through WINDOW_SLACK_MS) each fold the same-// tokens and the cost report doubles. Module-instance-local by design — the-// live completion path (this module, in the main thread) and the historical+// tokens and the cost report doubles. The stamp is an UPPER bound on the+// message's own timestamp (the claiming run's window end), which is what makes+// `pruneUsageClaims` safe: a message older than every possible future window+// can never be counted again. Module-instance-local by design — the live+// completion path (this module, in the main thread) and the historical// backfill worker (a SEPARATE module instance in a worker thread) each get-// their own copy of this Set, so `snapshotUsageClaims`/`mergeUsageClaims`-// hand claims across that boundary; see `usageBackfill.js`. A restart loses-// the ledger, but a run that already completed is never reconciled again.-const claimedMessages = new Set();+// their own copy, so `snapshotUsageClaims`/`mergeUsageClaims` hand claims+// across that boundary; see `usageBackfill.js`. A restart loses the ledger,+// but a run that already completed is never reconciled again.+const claimedMessages = new Map();++const isClaimed = (fileKey, messageKey) => claimedMessages.get(fileKey)?.has(messageKey) === true;++const addClaim = (fileKey, messageKey, stampMs) => {+ let messages = claimedMessages.get(fileKey);+ if (!messages) {+ messages = new Map();+ claimedMessages.set(fileKey, messages);+ }+ messages.set(messageKey, Math.max(messages.get(messageKey) ?? stampMs, stampMs));+};++const removeClaim = (fileKey, messageKey) => {+ const messages = claimedMessages.get(fileKey);+ if (!messages) return;+ messages.delete(messageKey);+ if (messages.size === 0) claimedMessages.delete(fileKey);+};// The ABSOLUTE cumulative position of each Codex rollout that has already been// billed, keyed by file path. Codex reports cumulative totals, so a rollout that@@ -112,15 +143,19 @@ export function __resetUsageClaims() {* billed by both.*/export function snapshotUsageClaims() {+ const claimed = [];+ for (const [fileKey, messages] of claimedMessages) {+ for (const [messageKey, stampMs] of messages) claimed.push([fileKey, messageKey, stampMs]);+ }return {- claimedMessages: [...claimedMessages],+ claimedMessages: claimed,codexHighWater: [...codexHighWater.entries()]};}/** Merge claims made elsewhere (a worker thread's ledger) into this one. */export function mergeUsageClaims({ claimedMessages: claimed = [], codexHighWater: highWater = [] } = {}) {- for (const key of claimed) claimedMessages.add(key);+ for (const [fileKey, messageKey, stampMs] of claimed) addClaim(fileKey, messageKey, stampMs);for (const [path, value] of highWater) {const existing = codexHighWater.get(path);codexHighWater.set(path, existing ? {@@ -133,6 +168,25 @@ export function mergeUsageClaims({ claimedMessages: claimed = [], codexHighWater}}+/**+ * Drop every claim no run still to be reconciled can re-count (see+ * CLAIM_RETENTION_MS), so the ledger is bounded by recent activity rather than+ * by the life of the process. The Codex high-water marks are NOT pruned: they+ * are one entry per rollout file, not per message, and a rollout that resumes+ * after its mark was dropped would re-bill its whole cumulative total.+ *+ * @param {number} [nowMs]+ */+export function pruneUsageClaims(nowMs = Date.now()) {+ const cutoff = nowMs - CLAIM_RETENTION_MS - WINDOW_SLACK_MS;+ for (const [fileKey, messages] of claimedMessages) {+ for (const [messageKey, stampMs] of messages) {+ if (stampMs < cutoff) messages.delete(messageKey);+ }+ if (messages.size === 0) claimedMessages.delete(fileKey);+ }+}+// A run is attributed only to transcripts whose cwd matches. A CoS agent works// in a git worktree under the install's data dir, so the worktree path — not the// install root — is what the CLI records; matching is therefore exact on the@@ -404,8 +458,7 @@ async function readGrokSessions({ home, workspacePath, from, to, fold, reserveFrconst chatPath = join(sessionDir, 'chat_history.jsonl');const chatText = await tryReadFile(chatPath);if (!chatText) continue;- const claimKey = `${chatPath}:session`;- if (claimedMessages.has(claimKey)) continue;+ if (isClaimed(chatPath, 'session')) continue;const chat = parseGrokChatHistory(chatText);const estimated = {messages: chat.messages,@@ -415,7 +468,8 @@ async function readGrokSessions({ home, workspacePath, from, to, fold, reserveFrcacheWriteTokens: 0};if (totalTranscriptTokens(estimated) === 0) continue;- claim(claimKey);+ // The session can run past this window, so stamp it by its own end too.+ claim(chatPath, 'session', Number.isFinite(endedMs) ? endedMs : null);const named = chat.model || summary.current_model_id || null;const modelKey = named ?? UNKNOWN_MODEL;fold({ ...estimated, models: named ? [named] : [], byModel: { [modelKey]: { ...estimated } } }, 'estimate');@@ -614,11 +668,13 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam}};- // Keys reserved by THIS call, so a failed/empty read can release them.+ // [fileKey, messageKey] pairs reserved by THIS call, so a failed/empty read+ // can release them. Every message this call counts lies inside its window,+ // so the window end (or now, for an unbounded read) bounds its timestamp.const reserved = [];- const claim = (claimKey) => {- claimedMessages.add(claimKey);- reserved.push(claimKey);+ const claim = (fileKey, messageKey, laterStampMs = null) => {+ addClaim(fileKey, messageKey, Math.max(to ?? Date.now(), laterStampMs ?? -Infinity));+ reserved.push([fileKey, messageKey]);};// Codex high-water marks advanced by this call, as [path, previousValue], so// an empty read restores the prior boundary instead of stranding it.@@ -626,12 +682,9 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam// Per-file view of the global ledger: a message key is only meaningful within// its own transcript, so scope the claim by file path to avoid a same-id// collision across two different sessions.- const excludeFor = (fileKey) => {- const prefix = `${fileKey}:`;- return {- has: (messageKey) => claimedMessages.has(prefix + messageKey)- };- };+ const excludeFor = (fileKey) => ({+ has: (messageKey) => isClaimed(fileKey, messageKey)+ });// Reserve IMMEDIATELY after each file is parsed, before the next `await`.// Deferring every claim to the end of the read would reopen the race the// ledger exists to close: this function awaits once per file, so two@@ -639,14 +692,11 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam// bill it. Reserving synchronously per file means the second run's read of// file A already sees the first run's claim.const reserveFrom = (fileKey, parsed) => {- for (const key of parsed.countedKeys || []) {- const claimKey = `${fileKey}:${key}`;- claim(claimKey);- }+ for (const key of parsed.countedKeys || []) claim(fileKey, key);};// Nothing was attributable after all — release so a later run can claim it.const releaseReserved = () => {- for (const key of reserved) claimedMessages.delete(key);+ for (const [fileKey, messageKey] of reserved) removeClaim(fileKey, messageKey);for (const [path, previous] of codexReserved) {if (previous == null) codexHighWater.delete(path);else codexHighWater.set(path, previous);@@ -916,4 +966,5 @@ export async function recordCompletedRunUsage(metadata, output, { home = homedirmaintenance.markCurrentUnsettled();console.error(`❌ Failed to record usage: ${err.message}`);});+ pruneUsageClaims();}diff --git a/server/services/usageReconciler.test.js b/server/services/usageReconciler.test.jsindex 3f1fcbc15c..48deb7c6c1 100644--- a/server/services/usageReconciler.test.js+++ b/server/services/usageReconciler.test.js@@ -16,6 +16,9 @@ const {reconcileRunUsage,recordCompletedRunUsage,resolveFamilyProvider,+ pruneUsageClaims,+ snapshotUsageClaims,+ CLAIM_RETENTION_MS,__resetUsageClaims} = await import('./usageReconciler.js');@@ -762,6 +765,113 @@ describe('claim ledger — concurrency and cumulative rollouts', () => {});});+// The ledger used to grow by one key per billed message for the life of the+// process (#12605). A claim only matters while a run still to be reconciled can+// see its message, so claims past the retention bound are dropped.+describe('claim ledger retention', () => {+ const HOUR = 60 * 60 * 1000;+ const run = {+ providerId: 'claude-code-tui',+ model: 'claude-opus-5',+ workspacePath: WORKSPACE,+ startTime: '2026-07-01T10:00:00.000Z',+ endTime: '2026-07-01T10:10:00.000Z'+ };+ const overlapping = { ...run, startTime: '2026-07-01T10:02:00.000Z', endTime: '2026-07-01T10:12:00.000Z' };++ afterEach(() => {+ vi.useRealTimers();+ });++ it('stays bounded across many sequential completed runs', async () => {+ vi.useFakeTimers({ toFake: ['Date'] });+ const days = 12;+ const sizes = [];+ for (let day = 0; day < days; day += 1) {+ const start = Date.parse('2026-07-01T10:00:00.000Z') + day * 24 * HOUR;+ await writeClaudeSession(`day-${day}.jsonl`, [+ claudeAssistant({ id: `m-${day}`, timestamp: new Date(start + 5 * 60_000).toISOString() })+ ]);+ vi.setSystemTime(start + 11 * 60_000);+ await recordCompletedRunUsage({+ providerId: 'claude-code-tui',+ model: 'claude-opus-5',+ workspacePath: WORKSPACE,+ promptLength: 10,+ startTime: new Date(start).toISOString(),+ endTime: new Date(start + 10 * 60_000).toISOString()+ }, 'out', { home, providers: [] });+ sizes.push(snapshotUsageClaims().claimedMessages.length);+ }++ // Every run billed its own message…+ expect(recordRunUsage).toHaveBeenCalledTimes(days);+ for (const [records] of recordRunUsage.mock.calls) {+ expect(records).toEqual([expect.objectContaining({ source: 'measured', tokensOut: 50 })]);+ }+ // …but the ledger holds only the runs inside the retention bound.+ const retainedRuns = Math.ceil(CLAIM_RETENTION_MS / (24 * HOUR)) + 1;+ expect(Math.max(...sizes)).toBeLessThanOrEqual(retainedRuns);+ expect(sizes.at(-1)).toBeLessThanOrEqual(retainedRuns);+ });++ it('drops a claim older than the retention bound after a completed run', async () => {+ await writeClaudeSession('a.jsonl', [+ claudeAssistant({ id: 'm1', timestamp: '2026-07-01T10:05:00.000Z' })+ ]);+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(Date.parse(run.endTime) + CLAIM_RETENTION_MS + 2 * HOUR);++ await recordCompletedRunUsage({ ...run, promptLength: 10 }, 'out', { home, providers: [] });++ expect(recordRunUsage.mock.calls[0][0]).toEqual([expect.objectContaining({ tokensOut: 50 })]);+ expect(snapshotUsageClaims().claimedMessages).toEqual([]);+ });++ it('keeps a claim inside the retention bound, so an overlapping run still bills once', async () => {+ await writeClaudeSession('a.jsonl', [+ claudeAssistant({ id: 'm1', timestamp: '2026-07-01T10:05:00.000Z', output: 50 }),+ claudeAssistant({ id: 'm2', timestamp: '2026-07-01T10:06:00.000Z', output: 25 })+ ]);++ const first = await readMeasuredUsage({ ...run, family: 'claude', home });+ // A prune between the two reads — the second run is still pending.+ pruneUsageClaims(Date.parse(overlapping.endTime) + CLAIM_RETENTION_MS - HOUR);+ const second = await readMeasuredUsage({ ...overlapping, family: 'claude', home });++ expect(first.tokensOut).toBe(75);+ expect(second).toBeNull();+ });++ it('keeps a killed grok session claimed until the session itself ages out', async () => {+ const sessionDir = join(home, '.grok', 'sessions', encodeURIComponent(WORKSPACE), 'session-aaaa');+ await mkdir(sessionDir, { recursive: true });+ const lastActive = Date.parse('2026-07-03T10:00:00.000Z');+ await writeFile(join(sessionDir, 'summary.json'), JSON.stringify({+ created_at: '2026-07-01T10:05:00.000Z',+ last_active_at: new Date(lastActive).toISOString(),+ current_model_id: 'example-grok-model'+ }));+ await writeFile(join(sessionDir, 'chat_history.jsonl'), [+ JSON.stringify({ type: 'user', content: [{ type: 'text', text: 'a'.repeat(400) }] }),+ JSON.stringify({ type: 'assistant', content: 'b'.repeat(200), model_id: 'example-grok-model' })+ ].join('\n'));++ const first = await readMeasuredUsage({ ...run, family: 'grok', home });+ expect(first?.tokensOut).toBeGreaterThan(0);++ // Long past the claiming run's window, but the session stayed active+ // later, so a run overlapping its tail must still find it claimed.+ pruneUsageClaims(lastActive + HOUR);+ expect(await readMeasuredUsage({+ ...run, startTime: '2026-07-03T09:00:00.000Z', endTime: '2026-07-03T09:30:00.000Z', family: 'grok', home+ })).toBeNull();++ pruneUsageClaims(lastActive + CLAIM_RETENTION_MS + 2 * HOUR);+ expect(snapshotUsageClaims().claimedMessages).toEqual([]);+ });+});+describe('model attribution when the transcript disagrees', () => {const run = {providerId: 'claude-code-tui-bedrock',