Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep historical backfill from re-billing usage the live path settled (#12605)

[._.]/ Adam Eivy6261f4d05d01
diff --git a/server/services/usage.js b/server/services/usage.js
index 634e72ec75..eb543dea65 100644
--- a/server/services/usage.js
+++ b/server/services/usage.js
@@ -327,11 +327,6 @@ export function getReconciledUsageRunIds() {
return Object.keys(usageData?.reconciledRuns || {});
}
-/**
- * Run ids whose NESTED-session (sibling-family) attribution has already landed.
- * A separate marker from `reconciledRuns` on purpose: the sibling pass has to be
- * able to run on a run whose own transcript was reconciled long ago (#5831).
- */
/**
* Drop sibling markers that only guarded a single apply — the interactive scan's
* synthetic ids, which its own watermark makes redundant once the batch lands.
@@ -346,17 +341,36 @@ export async function forgetSiblingReconciledUsageRuns(runIds = []) {
if (dropped > 0) await saveUsage();
}
+/**
+ * Run ids whose NESTED-session (sibling-family) attribution has already landed.
+ * A separate marker from `reconciledRuns` on purpose: the sibling pass has to be
+ * able to run on a run whose own transcript was reconciled long ago (#5831).
+ */
export function getSiblingReconciledUsageRunIds() {
return Object.keys(usageData?.siblingReconciledRuns || {});
}
-export async function markUsageRunReconciled(runId) {
+/**
+ * Record that a completed run's accounting landed. `siblings` also marks its
+ * nested-session pass, which the live path runs whenever it has a provider list —
+ * without that marker a later backfill would bill those nested sessions again.
+ */
+export async function markUsageRunReconciled(runId, { siblings = false } = {}) {
if (!runId) return;
if (!usageData) await loadUsage();
usageData.reconciledRuns ??= {};
- if (usageData.reconciledRuns[runId]) return;
- usageData.reconciledRuns[runId] = new Date().toISOString();
- await saveUsage();
+ usageData.siblingReconciledRuns ??= {};
+ const now = new Date().toISOString();
+ let changed = false;
+ if (!usageData.reconciledRuns[runId]) {
+ usageData.reconciledRuns[runId] = now;
+ changed = true;
+ }
+ if (siblings && !usageData.siblingReconciledRuns[runId]) {
+ usageData.siblingReconciledRuns[runId] = now;
+ changed = true;
+ }
+ if (changed) await saveUsage();
}
/**
diff --git a/server/services/usageBackfillWorker.js b/server/services/usageBackfillWorker.js
index baf2cb9ef7..9ecd3637a4 100644
--- a/server/services/usageBackfillWorker.js
+++ b/server/services/usageBackfillWorker.js
@@ -3,7 +3,16 @@ import { join } from 'path';
import { readdir } from 'fs/promises';
import { estimateTokens, estimateTokensFromChars } from '../lib/contextBudget.js';
import { readJSONFile, tryReadFile } from '../lib/fileUtils.js';
-import { mergeUsageClaims, reconcileRunUsage, snapshotUsageClaims, transcriptFamily } from './usageReconciler.js';
+import {
+ mergeUsageClaims,
+ readMeasuredUsage,
+ reconcileRunUsage,
+ resolveFamilyProvider,
+ snapshotUsageClaims,
+ TRANSCRIPT_FAMILIES,
+ transcriptFamily,
+ WINDOW_SLACK_MS
+} from './usageReconciler.js';
const listRunIds = async (runsDir) => readdir(runsDir, { withFileTypes: true })
.then((entries) => entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name))
@@ -24,6 +33,68 @@ const asList = (records) => (Array.isArray(records) ? records.flat(Infinity) : [
const isWorthRecording = (records, siblings) => siblings.length > 0
|| records.some((entry) => entry?.source === 'measured' || entry?.source === 'mixed');
+const runWindow = (metadata) => ({
+ from: Date.parse(metadata.startTime) - WINDOW_SLACK_MS,
+ to: Date.parse(metadata.endTime) + WINDOW_SLACK_MS
+});
+
+// Transcript readers match a cwd or any subdirectory of it, so runs whose
+// workspaces nest inside one another can read the same session.
+const workspaceAncestors = (path) => {
+ const ancestors = [];
+ for (let cut = path.lastIndexOf('/'); cut > 0; cut = path.lastIndexOf('/', cut - 1)) {
+ ancestors.push(path.slice(0, cut));
+ }
+ return ancestors;
+};
+
+/**
+ * Re-claim what settled runs already billed, before any pending run is read.
+ *
+ * Every message inside a settled pass's window was billed — by that run or by
+ * an earlier overlapping one — so re-reading the window and discarding the
+ * counts claims exactly what is already on the books. The live ledger handed
+ * over in `claimsSeed` cannot carry that alone: it is empty after a restart
+ * and prunes claims past its retention bound, while this scan reaches runs of
+ * any age. Only settled runs that can overlap a pending one are re-read, and
+ * only for the families each settled pass actually covered.
+ */
+async function claimSettledUsage(runs, candidates, { home, providers }) {
+ const below = new Map();
+ const at = new Map();
+ const index = (map, key, window) => {
+ if (!map.has(key)) map.set(key, []);
+ map.get(key).push(window);
+ };
+ for (const { metadata } of candidates) {
+ const window = runWindow(metadata);
+ index(at, metadata.workspacePath, window);
+ index(below, metadata.workspacePath, window);
+ for (const ancestor of workspaceAncestors(metadata.workspacePath)) index(below, ancestor, window);
+ }
+ const canScanSiblings = Array.isArray(providers) && providers.length > 0;
+ for (const { metadata, parentSettled, siblingSettled } of runs) {
+ const { workspacePath } = metadata;
+ const window = runWindow(metadata);
+ const pending = [
+ ...(below.get(workspacePath) || []),
+ ...workspaceAncestors(workspacePath).flatMap((ancestor) => at.get(ancestor) || [])
+ ];
+ if (!pending.some((other) => other.from <= window.to && window.from <= other.to)) continue;
+ const parentFamily = transcriptFamily(metadata);
+ const families = parentSettled && parentFamily ? [parentFamily] : [];
+ if (siblingSettled && canScanSiblings) {
+ families.push(...TRANSCRIPT_FAMILIES.filter((family) => (
+ family !== parentFamily && resolveFamilyProvider(providers, family)
+ )));
+ }
+ for (const family of families) {
+ await readMeasuredUsage({ workspacePath, startTime: metadata.startTime, endTime: metadata.endTime, family, home })
+ .catch(() => null);
+ }
+ }
+}
+
/**
* Read historical run artifacts and produce estimate→measurement corrections.
* This function runs inside a worker in production so parsing large JSONL files
@@ -65,6 +136,7 @@ export async function scanHistoricalUsage({
const siblingReconciled = new Set(siblingReconciledRunIds);
const canScanSiblings = Array.isArray(providers) && providers.length > 0;
const candidates = [];
+ const settled = [];
for (const runId of await listRunIds(runsDir)) {
const metadataPath = join(runsDir, runId, 'metadata.json');
@@ -79,9 +151,13 @@ export async function scanHistoricalUsage({
// no transcript of its own.
const siblingPending = canScanSiblings
&& !siblingReconciled.has(runId) && !metadata.usageSiblingsReconciled;
+ const parentSettled = reconciled.has(runId) || metadata.usageReconciled === true;
+ const siblingSettled = siblingReconciled.has(runId) || metadata.usageSiblingsReconciled === true;
+ if (parentSettled || siblingSettled) settled.push({ metadata, parentSettled, siblingSettled });
if (!parentPending && !siblingPending) continue;
candidates.push({ runId, metadataPath, metadata, parentPending, siblingPending });
}
+ if (candidates.length) await claimSettledUsage(settled, candidates, { home, providers });
candidates.sort((a, b) => Date.parse(a.metadata.startTime) - Date.parse(b.metadata.startTime));
const corrections = [];
diff --git a/server/services/usageBackfillWorker.test.js b/server/services/usageBackfillWorker.test.js
index 172967f34a..3b049d0bde 100644
--- a/server/services/usageBackfillWorker.test.js
+++ b/server/services/usageBackfillWorker.test.js
@@ -117,6 +117,30 @@ describe('historical usage worker', () => {
expect(result.corrections).toEqual([]);
expect(result.claimsSnapshot.claimedMessages).toEqual(claimsSeed.claimedMessages);
});
+
+ // The live ledger is empty after a restart and prunes old claims, so a
+ // settled run on disk is the durable record of what was already billed.
+ it('does not re-bill what a settled run billed once the live claim is gone', async () => {
+ await writeRun('run-settled-live', {
+ providerId: 'claude-code-tui',
+ model: 'claude-opus-5',
+ workspacePath: workspace,
+ startTime: '2026-07-01T10:00:00.000Z',
+ endTime: '2026-07-01T10:10:00.000Z',
+ usageReconciled: true
+ });
+ await writeRun('run-overlapping-history', {
+ providerId: 'claude-code-tui',
+ model: 'claude-opus-5',
+ workspacePath: workspace,
+ startTime: '2026-07-01T10:02:00.000Z',
+ endTime: '2026-07-01T10:12:00.000Z'
+ });
+
+ const result = await scanHistoricalUsage({ runsDir: root, home, claimsSeed: snapshotUsageClaims() });
+ expect(result.total).toBe(1);
+ expect(result.corrections).toEqual([]);
+ });
});
// A nested `--review-with grok` pass leaves a grok session but never a PortOS
@@ -128,8 +152,8 @@ describe('historical sibling-family attribution', () => {
const GROK_PROVIDER = { id: 'grok-cli', type: 'cli', command: 'grok', enabled: true, defaultModel: GROK_MODEL };
const CLAUDE_PROVIDER = { id: 'claude-code-tui', type: 'tui', command: 'claude', enabled: true, defaultModel: 'claude-opus-5' };
- const writeGrokSession = async () => {
- const dir = join(home, '.grok', 'sessions', encodeURIComponent(workspace), 'session-aaaa');
+ const writeGrokSession = async (cwd = workspace) => {
+ const dir = join(home, '.grok', 'sessions', encodeURIComponent(cwd), 'session-aaaa');
await mkdir(dir, { recursive: true });
await writeFile(join(dir, 'updates.jsonl'), JSON.stringify({
timestamp: Math.round(Date.parse('2026-07-01T10:05:00.000Z') / 1000),
@@ -194,6 +218,28 @@ describe('historical sibling-family attribution', () => {
expect(result.corrections).toEqual([]);
});
+ // Readers match a cwd or any subdirectory of it, so the settled run and the
+ // pending one can sit at, above, or below each other and share one session.
+ it.each([
+ ['the same workspace', workspace, workspace],
+ ['a parent workspace', '/work', workspace],
+ ['a subdirectory workspace', `${workspace}/server`, `${workspace}/server`]
+ ])('does not re-bill a nested session a settled run already covered, from %s', async (_label, pendingWorkspace, sessionCwd) => {
+ await writeRun('run-already-scanned', { ...reconciledRun, usageSiblingsReconciled: true });
+ await writeRun('run-overlapping-history', {
+ ...reconciledRun,
+ workspacePath: pendingWorkspace,
+ startTime: '2026-07-01T10:04:00.000Z',
+ endTime: '2026-07-01T10:14:00.000Z',
+ usageReconciled: false
+ });
+ await writeGrokSession(sessionCwd);
+
+ const result = await scanHistoricalUsage({ runsDir: root, home, providers: [CLAUDE_PROVIDER, GROK_PROVIDER] });
+ expect(result.total).toBe(1);
+ expect(result.corrections).toEqual([]);
+ });
+
it('does no sibling work at all when the install has no provider list', async () => {
await writeRun('run-already-measured', reconciledRun);
await writeGrokSession();
diff --git a/server/services/usageReconciler.js b/server/services/usageReconciler.js
index f033e3505e..f2600ff940 100644
--- a/server/services/usageReconciler.js
+++ b/server/services/usageReconciler.js
@@ -80,9 +80,11 @@ export const WINDOW_SLACK_MS = 60_000;
// needed only while some run still to be reconciled can start before the
// message. Every run path is bounded far below this (the CoS max-runtime
// ceiling, the toolkit's API run cap), and the ledger never sees in-flight
-// runs, so a fixed bound stands in for "oldest pending run start". A run that
-// outlives it can re-bill a pruned neighbour's overlap — the same exposure a
-// server restart, which empties the ledger, already accepts.
+// runs, so a fixed bound stands in for "oldest pending run start". A live run
+// that outlives it can re-bill a pruned neighbour's overlap — the same exposure
+// a server restart, which empties the ledger, already accepts. The historical
+// backfill, which reaches runs of any age, does not rely on these claims for
+// old windows: it re-claims what settled runs billed from their run records.
export const CLAIM_RETENTION_MS = 48 * 60 * 60 * 1000;
// Messages already billed to a run: transcript key -> message key -> stamp.
@@ -946,20 +948,27 @@ export async function recordCompletedRunUsage(metadata, output, { home = homedir
const resolved = providers ?? await import('./providers.js')
.then((module) => module.listProviders())
.catch(() => null);
+ // Same condition `reconcileRunUsage` scans siblings under.
+ const siblingsScanned = Array.isArray(resolved) && resolved.length > 0;
// One catch for the whole chain: whatever fails — reading a transcript or
// persisting the record — usage accounting must not surface as a run failure.
await reconcileRunUsage(metadata, estimate, { home, providers: resolved, estimateSource })
.then(recordRunUsage)
.then(async () => {
if (!metadata?.id) return;
- await markUsageRunReconciled(metadata.id);
+ await markUsageRunReconciled(metadata.id, { siblings: siblingsScanned });
const metadataPath = join(PATHS.runs, metadata.id, 'metadata.json');
// Non-strict exception: missing/unreadable metadata skips the marker
// write; the usage ledger remains authoritative for the accounting.
const persisted = await readJSONFile(metadataPath, null);
if (!persisted) return;
+ const now = new Date().toISOString();
persisted.usageReconciled = true;
- persisted.usageReconciledAt = new Date().toISOString();
+ persisted.usageReconciledAt = now;
+ if (siblingsScanned) {
+ persisted.usageSiblingsReconciled = true;
+ persisted.usageSiblingsReconciledAt = now;
+ }
await atomicWrite(metadataPath, persisted);
})
.catch((err) => {
diff --git a/server/services/usageReconciler.test.js b/server/services/usageReconciler.test.js
index 48deb7c6c1..6394874abc 100644
--- a/server/services/usageReconciler.test.js
+++ b/server/services/usageReconciler.test.js
@@ -509,17 +509,24 @@ describe('recordCompletedRunUsage', () => {
});
it('durably marks a completed run so a later backfill skips it', async () => {
- await recordCompletedRunUsage({
- id: 'run-example-live',
+ const run = {
providerId: 'ollama',
model: 'llama3',
workspacePath: WORKSPACE,
promptLength: 40,
startTime: '2026-07-01T10:00:00.000Z',
endTime: '2026-07-01T10:10:00.000Z'
- }, 'captured output');
+ };
+ await recordCompletedRunUsage({ ...run, id: 'run-example-live' }, 'captured output', { home, providers: [] });
+ // With a provider list the nested-session pass ran too, so a backfill must
+ // not bill those sessions a second time.
+ await recordCompletedRunUsage({ ...run, id: 'run-example-scanned' }, 'captured output', {
+ home,
+ providers: [{ id: 'grok-cli', type: 'cli', command: 'grok', enabled: true }]
+ });
- expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-live');
+ expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-live', { siblings: false });
+ expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-scanned', { siblings: true });
});
it('records the estimate when no transcript exists rather than recording nothing', async () => {