Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: bound the usage reconciler's claim ledger instead of growing it forever (#12605)

[._.]/ Adam Eivy4dd8b83611dc
diff --git a/server/services/usageBackfillWorker.test.js b/server/services/usageBackfillWorker.test.js
index 2c172b0097..172967f34a 100644
--- a/server/services/usageBackfillWorker.test.js
+++ b/server/services/usageBackfillWorker.test.js
@@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { mkdir, mkdtemp, rm, writeFile } from 'fs/promises';
import { join } from 'path';
import { tmpdir } from 'os';
-import { __resetUsageClaims } from './usageReconciler.js';
+import { __resetUsageClaims, pruneUsageClaims, readMeasuredUsage, snapshotUsageClaims } from './usageReconciler.js';
import { scanHistoricalUsage } from './usageBackfillWorker.js';
let root;
@@ -96,6 +96,27 @@ describe('historical usage worker', () => {
expect(result.total).toBe(0);
expect(result.corrections).toEqual([]);
});
+ it('does not re-bill a message the live path claimed, even after a prune', async () => {
+ const window = { startTime: '2026-07-01T10:00:00.000Z', endTime: '2026-07-01T10:10:00.000Z' };
+ const live = await readMeasuredUsage({ workspacePath: workspace, ...window, family: 'claude', home });
+ expect(live?.tokensOut).toBe(200);
+ // A prune inside the retention bound keeps the claim for the hand-off.
+ pruneUsageClaims(Date.parse(window.endTime) + 60 * 60 * 1000);
+ const claimsSeed = snapshotUsageClaims();
+ // The worker thread starts from its own empty copy of the ledger.
+ __resetUsageClaims();
+ await writeRun('run-overlapping-history', {
+ providerId: 'claude-code-tui',
+ model: 'claude-opus-5',
+ workspacePath: workspace,
+ startTime: '2026-07-01T10:02:00.000Z',
+ endTime: '2026-07-01T10:12:00.000Z'
+ });
+
+ const result = await scanHistoricalUsage({ runsDir: root, home, claimsSeed });
+ expect(result.corrections).toEqual([]);
+ expect(result.claimsSnapshot.claimedMessages).toEqual(claimsSeed.claimedMessages);
+ });
});
// A nested `--review-with grok` pass leaves a grok session but never a PortOS
diff --git a/server/services/usageReconciler.js b/server/services/usageReconciler.js
index 1f275872e1..f033e3505e 100644
--- a/server/services/usageReconciler.js
+++ b/server/services/usageReconciler.js
@@ -75,17 +75,48 @@ import { markUsageRunReconciled, recordRunUsage } from './usage.js';
// per-message claim below, not by the window.
export const WINDOW_SLACK_MS = 60_000;
-// Messages already billed to a run, keyed `<transcript-key>:<message-key>`.
+// How long a claim outlives the window that made it. A later run can only
+// re-count a message whose timestamp falls inside ITS window, so a claim is
+// needed only while some run still to be reconciled can start before the
+// message. Every run path is bounded far below this (the CoS max-runtime
+// ceiling, the toolkit's API run cap), and the ledger never sees in-flight
+// runs, so a fixed bound stands in for "oldest pending run start". A run that
+// outlives it can re-bill a pruned neighbour's overlap — the same exposure a
+// server restart, which empties the ledger, already accepts.
+export const CLAIM_RETENTION_MS = 48 * 60 * 60 * 1000;
+
+// Messages already billed to a run: transcript key -> message key -> stamp.
// A transcript message must be counted exactly ONCE across every run that can
// see it: without this, two concurrent runs in the same cwd (or one run whose
// window overlaps a neighbour's through WINDOW_SLACK_MS) each fold the same
-// tokens and the cost report doubles. Module-instance-local by design — the
-// live completion path (this module, in the main thread) and the historical
+// tokens and the cost report doubles. The stamp is an UPPER bound on the
+// message's own timestamp (the claiming run's window end), which is what makes
+// `pruneUsageClaims` safe: a message older than every possible future window
+// can never be counted again. Module-instance-local by design — the live
+// completion path (this module, in the main thread) and the historical
// backfill worker (a SEPARATE module instance in a worker thread) each get
-// their own copy of this Set, so `snapshotUsageClaims`/`mergeUsageClaims`
-// hand claims across that boundary; see `usageBackfill.js`. A restart loses
-// the ledger, but a run that already completed is never reconciled again.
-const claimedMessages = new Set();
+// their own copy, so `snapshotUsageClaims`/`mergeUsageClaims` hand claims
+// across that boundary; see `usageBackfill.js`. A restart loses the ledger,
+// but a run that already completed is never reconciled again.
+const claimedMessages = new Map();
+
+const isClaimed = (fileKey, messageKey) => claimedMessages.get(fileKey)?.has(messageKey) === true;
+
+const addClaim = (fileKey, messageKey, stampMs) => {
+ let messages = claimedMessages.get(fileKey);
+ if (!messages) {
+ messages = new Map();
+ claimedMessages.set(fileKey, messages);
+ }
+ messages.set(messageKey, Math.max(messages.get(messageKey) ?? stampMs, stampMs));
+};
+
+const removeClaim = (fileKey, messageKey) => {
+ const messages = claimedMessages.get(fileKey);
+ if (!messages) return;
+ messages.delete(messageKey);
+ if (messages.size === 0) claimedMessages.delete(fileKey);
+};
// The ABSOLUTE cumulative position of each Codex rollout that has already been
// billed, keyed by file path. Codex reports cumulative totals, so a rollout that
@@ -112,15 +143,19 @@ export function __resetUsageClaims() {
* billed by both.
*/
export function snapshotUsageClaims() {
+ const claimed = [];
+ for (const [fileKey, messages] of claimedMessages) {
+ for (const [messageKey, stampMs] of messages) claimed.push([fileKey, messageKey, stampMs]);
+ }
return {
- claimedMessages: [...claimedMessages],
+ claimedMessages: claimed,
codexHighWater: [...codexHighWater.entries()]
};
}
/** Merge claims made elsewhere (a worker thread's ledger) into this one. */
export function mergeUsageClaims({ claimedMessages: claimed = [], codexHighWater: highWater = [] } = {}) {
- for (const key of claimed) claimedMessages.add(key);
+ for (const [fileKey, messageKey, stampMs] of claimed) addClaim(fileKey, messageKey, stampMs);
for (const [path, value] of highWater) {
const existing = codexHighWater.get(path);
codexHighWater.set(path, existing ? {
@@ -133,6 +168,25 @@ export function mergeUsageClaims({ claimedMessages: claimed = [], codexHighWater
}
}
+/**
+ * Drop every claim no run still to be reconciled can re-count (see
+ * CLAIM_RETENTION_MS), so the ledger is bounded by recent activity rather than
+ * by the life of the process. The Codex high-water marks are NOT pruned: they
+ * are one entry per rollout file, not per message, and a rollout that resumes
+ * after its mark was dropped would re-bill its whole cumulative total.
+ *
+ * @param {number} [nowMs]
+ */
+export function pruneUsageClaims(nowMs = Date.now()) {
+ const cutoff = nowMs - CLAIM_RETENTION_MS - WINDOW_SLACK_MS;
+ for (const [fileKey, messages] of claimedMessages) {
+ for (const [messageKey, stampMs] of messages) {
+ if (stampMs < cutoff) messages.delete(messageKey);
+ }
+ if (messages.size === 0) claimedMessages.delete(fileKey);
+ }
+}
+
// A run is attributed only to transcripts whose cwd matches. A CoS agent works
// in a git worktree under the install's data dir, so the worktree path — not the
// install root — is what the CLI records; matching is therefore exact on the
@@ -404,8 +458,7 @@ async function readGrokSessions({ home, workspacePath, from, to, fold, reserveFr
const chatPath = join(sessionDir, 'chat_history.jsonl');
const chatText = await tryReadFile(chatPath);
if (!chatText) continue;
- const claimKey = `${chatPath}:session`;
- if (claimedMessages.has(claimKey)) continue;
+ if (isClaimed(chatPath, 'session')) continue;
const chat = parseGrokChatHistory(chatText);
const estimated = {
messages: chat.messages,
@@ -415,7 +468,8 @@ async function readGrokSessions({ home, workspacePath, from, to, fold, reserveFr
cacheWriteTokens: 0
};
if (totalTranscriptTokens(estimated) === 0) continue;
- claim(claimKey);
+ // The session can run past this window, so stamp it by its own end too.
+ claim(chatPath, 'session', Number.isFinite(endedMs) ? endedMs : null);
const named = chat.model || summary.current_model_id || null;
const modelKey = named ?? UNKNOWN_MODEL;
fold({ ...estimated, models: named ? [named] : [], byModel: { [modelKey]: { ...estimated } } }, 'estimate');
@@ -614,11 +668,13 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam
}
};
- // Keys reserved by THIS call, so a failed/empty read can release them.
+ // [fileKey, messageKey] pairs reserved by THIS call, so a failed/empty read
+ // can release them. Every message this call counts lies inside its window,
+ // so the window end (or now, for an unbounded read) bounds its timestamp.
const reserved = [];
- const claim = (claimKey) => {
- claimedMessages.add(claimKey);
- reserved.push(claimKey);
+ const claim = (fileKey, messageKey, laterStampMs = null) => {
+ addClaim(fileKey, messageKey, Math.max(to ?? Date.now(), laterStampMs ?? -Infinity));
+ reserved.push([fileKey, messageKey]);
};
// Codex high-water marks advanced by this call, as [path, previousValue], so
// an empty read restores the prior boundary instead of stranding it.
@@ -626,12 +682,9 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam
// Per-file view of the global ledger: a message key is only meaningful within
// its own transcript, so scope the claim by file path to avoid a same-id
// collision across two different sessions.
- const excludeFor = (fileKey) => {
- const prefix = `${fileKey}:`;
- return {
- has: (messageKey) => claimedMessages.has(prefix + messageKey)
- };
- };
+ const excludeFor = (fileKey) => ({
+ has: (messageKey) => isClaimed(fileKey, messageKey)
+ });
// Reserve IMMEDIATELY after each file is parsed, before the next `await`.
// Deferring every claim to the end of the read would reopen the race the
// ledger exists to close: this function awaits once per file, so two
@@ -639,14 +692,11 @@ export async function readMeasuredUsage({ workspacePath, startTime, endTime, fam
// bill it. Reserving synchronously per file means the second run's read of
// file A already sees the first run's claim.
const reserveFrom = (fileKey, parsed) => {
- for (const key of parsed.countedKeys || []) {
- const claimKey = `${fileKey}:${key}`;
- claim(claimKey);
- }
+ for (const key of parsed.countedKeys || []) claim(fileKey, key);
};
// Nothing was attributable after all — release so a later run can claim it.
const releaseReserved = () => {
- for (const key of reserved) claimedMessages.delete(key);
+ for (const [fileKey, messageKey] of reserved) removeClaim(fileKey, messageKey);
for (const [path, previous] of codexReserved) {
if (previous == null) codexHighWater.delete(path);
else codexHighWater.set(path, previous);
@@ -916,4 +966,5 @@ export async function recordCompletedRunUsage(metadata, output, { home = homedir
maintenance.markCurrentUnsettled();
console.error(`❌ Failed to record usage: ${err.message}`);
});
+ pruneUsageClaims();
}
diff --git a/server/services/usageReconciler.test.js b/server/services/usageReconciler.test.js
index 3f1fcbc15c..48deb7c6c1 100644
--- a/server/services/usageReconciler.test.js
+++ b/server/services/usageReconciler.test.js
@@ -16,6 +16,9 @@ const {
reconcileRunUsage,
recordCompletedRunUsage,
resolveFamilyProvider,
+ pruneUsageClaims,
+ snapshotUsageClaims,
+ CLAIM_RETENTION_MS,
__resetUsageClaims
} = await import('./usageReconciler.js');
@@ -762,6 +765,113 @@ describe('claim ledger — concurrency and cumulative rollouts', () => {
});
});
+// The ledger used to grow by one key per billed message for the life of the
+// process (#12605). A claim only matters while a run still to be reconciled can
+// see its message, so claims past the retention bound are dropped.
+describe('claim ledger retention', () => {
+ const HOUR = 60 * 60 * 1000;
+ const run = {
+ providerId: 'claude-code-tui',
+ model: 'claude-opus-5',
+ workspacePath: WORKSPACE,
+ startTime: '2026-07-01T10:00:00.000Z',
+ endTime: '2026-07-01T10:10:00.000Z'
+ };
+ const overlapping = { ...run, startTime: '2026-07-01T10:02:00.000Z', endTime: '2026-07-01T10:12:00.000Z' };
+
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('stays bounded across many sequential completed runs', async () => {
+ vi.useFakeTimers({ toFake: ['Date'] });
+ const days = 12;
+ const sizes = [];
+ for (let day = 0; day < days; day += 1) {
+ const start = Date.parse('2026-07-01T10:00:00.000Z') + day * 24 * HOUR;
+ await writeClaudeSession(`day-${day}.jsonl`, [
+ claudeAssistant({ id: `m-${day}`, timestamp: new Date(start + 5 * 60_000).toISOString() })
+ ]);
+ vi.setSystemTime(start + 11 * 60_000);
+ await recordCompletedRunUsage({
+ providerId: 'claude-code-tui',
+ model: 'claude-opus-5',
+ workspacePath: WORKSPACE,
+ promptLength: 10,
+ startTime: new Date(start).toISOString(),
+ endTime: new Date(start + 10 * 60_000).toISOString()
+ }, 'out', { home, providers: [] });
+ sizes.push(snapshotUsageClaims().claimedMessages.length);
+ }
+
+ // Every run billed its own message…
+ expect(recordRunUsage).toHaveBeenCalledTimes(days);
+ for (const [records] of recordRunUsage.mock.calls) {
+ expect(records).toEqual([expect.objectContaining({ source: 'measured', tokensOut: 50 })]);
+ }
+ // …but the ledger holds only the runs inside the retention bound.
+ const retainedRuns = Math.ceil(CLAIM_RETENTION_MS / (24 * HOUR)) + 1;
+ expect(Math.max(...sizes)).toBeLessThanOrEqual(retainedRuns);
+ expect(sizes.at(-1)).toBeLessThanOrEqual(retainedRuns);
+ });
+
+ it('drops a claim older than the retention bound after a completed run', async () => {
+ await writeClaudeSession('a.jsonl', [
+ claudeAssistant({ id: 'm1', timestamp: '2026-07-01T10:05:00.000Z' })
+ ]);
+ vi.useFakeTimers({ toFake: ['Date'] });
+ vi.setSystemTime(Date.parse(run.endTime) + CLAIM_RETENTION_MS + 2 * HOUR);
+
+ await recordCompletedRunUsage({ ...run, promptLength: 10 }, 'out', { home, providers: [] });
+
+ expect(recordRunUsage.mock.calls[0][0]).toEqual([expect.objectContaining({ tokensOut: 50 })]);
+ expect(snapshotUsageClaims().claimedMessages).toEqual([]);
+ });
+
+ it('keeps a claim inside the retention bound, so an overlapping run still bills once', async () => {
+ await writeClaudeSession('a.jsonl', [
+ claudeAssistant({ id: 'm1', timestamp: '2026-07-01T10:05:00.000Z', output: 50 }),
+ claudeAssistant({ id: 'm2', timestamp: '2026-07-01T10:06:00.000Z', output: 25 })
+ ]);
+
+ const first = await readMeasuredUsage({ ...run, family: 'claude', home });
+ // A prune between the two reads — the second run is still pending.
+ pruneUsageClaims(Date.parse(overlapping.endTime) + CLAIM_RETENTION_MS - HOUR);
+ const second = await readMeasuredUsage({ ...overlapping, family: 'claude', home });
+
+ expect(first.tokensOut).toBe(75);
+ expect(second).toBeNull();
+ });
+
+ it('keeps a killed grok session claimed until the session itself ages out', async () => {
+ const sessionDir = join(home, '.grok', 'sessions', encodeURIComponent(WORKSPACE), 'session-aaaa');
+ await mkdir(sessionDir, { recursive: true });
+ const lastActive = Date.parse('2026-07-03T10:00:00.000Z');
+ await writeFile(join(sessionDir, 'summary.json'), JSON.stringify({
+ created_at: '2026-07-01T10:05:00.000Z',
+ last_active_at: new Date(lastActive).toISOString(),
+ current_model_id: 'example-grok-model'
+ }));
+ await writeFile(join(sessionDir, 'chat_history.jsonl'), [
+ JSON.stringify({ type: 'user', content: [{ type: 'text', text: 'a'.repeat(400) }] }),
+ JSON.stringify({ type: 'assistant', content: 'b'.repeat(200), model_id: 'example-grok-model' })
+ ].join('\n'));
+
+ const first = await readMeasuredUsage({ ...run, family: 'grok', home });
+ expect(first?.tokensOut).toBeGreaterThan(0);
+
+ // Long past the claiming run's window, but the session stayed active
+ // later, so a run overlapping its tail must still find it claimed.
+ pruneUsageClaims(lastActive + HOUR);
+ expect(await readMeasuredUsage({
+ ...run, startTime: '2026-07-03T09:00:00.000Z', endTime: '2026-07-03T09:30:00.000Z', family: 'grok', home
+ })).toBeNull();
+
+ pruneUsageClaims(lastActive + CLAIM_RETENTION_MS + 2 * HOUR);
+ expect(snapshotUsageClaims().claimedMessages).toEqual([]);
+ });
+});
+
describe('model attribution when the transcript disagrees', () => {
const run = {
providerId: 'claude-code-tui-bedrock',