Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep historical backfill from re-billing usage the live path settled (#12605)
[._.]/ Adam Eivy6261f4d05d01
diff --git a/server/services/usage.js b/server/services/usage.jsindex 634e72ec75..eb543dea65 100644--- a/server/services/usage.js+++ b/server/services/usage.js@@ -327,11 +327,6 @@ export function getReconciledUsageRunIds() {return Object.keys(usageData?.reconciledRuns || {});}-/**- * Run ids whose NESTED-session (sibling-family) attribution has already landed.- * A separate marker from `reconciledRuns` on purpose: the sibling pass has to be- * able to run on a run whose own transcript was reconciled long ago (#5831).- *//*** Drop sibling markers that only guarded a single apply — the interactive scan's* synthetic ids, which its own watermark makes redundant once the batch lands.@@ -346,17 +341,36 @@ export async function forgetSiblingReconciledUsageRuns(runIds = []) {if (dropped > 0) await saveUsage();}+/**+ * Run ids whose NESTED-session (sibling-family) attribution has already landed.+ * A separate marker from `reconciledRuns` on purpose: the sibling pass has to be+ * able to run on a run whose own transcript was reconciled long ago (#5831).+ */export function getSiblingReconciledUsageRunIds() {return Object.keys(usageData?.siblingReconciledRuns || {});}-export async function markUsageRunReconciled(runId) {+/**+ * Record that a completed run's accounting landed. `siblings` also marks its+ * nested-session pass, which the live path runs whenever it has a provider list —+ * without that marker a later backfill would bill those nested sessions again.+ */+export async function markUsageRunReconciled(runId, { siblings = false } = {}) {if (!runId) return;if (!usageData) await loadUsage();usageData.reconciledRuns ??= {};- if (usageData.reconciledRuns[runId]) return;- usageData.reconciledRuns[runId] = new Date().toISOString();- await saveUsage();+ usageData.siblingReconciledRuns ??= {};+ const now = new Date().toISOString();+ let changed = false;+ if (!usageData.reconciledRuns[runId]) {+ usageData.reconciledRuns[runId] = now;+ changed = true;+ }+ if (siblings && !usageData.siblingReconciledRuns[runId]) {+ usageData.siblingReconciledRuns[runId] = now;+ changed = true;+ }+ if (changed) await saveUsage();}/**diff --git a/server/services/usageBackfillWorker.js b/server/services/usageBackfillWorker.jsindex baf2cb9ef7..9ecd3637a4 100644--- a/server/services/usageBackfillWorker.js+++ b/server/services/usageBackfillWorker.js@@ -3,7 +3,16 @@ import { join } from 'path';import { readdir } from 'fs/promises';import { estimateTokens, estimateTokensFromChars } from '../lib/contextBudget.js';import { readJSONFile, tryReadFile } from '../lib/fileUtils.js';-import { mergeUsageClaims, reconcileRunUsage, snapshotUsageClaims, transcriptFamily } from './usageReconciler.js';+import {+ mergeUsageClaims,+ readMeasuredUsage,+ reconcileRunUsage,+ resolveFamilyProvider,+ snapshotUsageClaims,+ TRANSCRIPT_FAMILIES,+ transcriptFamily,+ WINDOW_SLACK_MS+} from './usageReconciler.js';const listRunIds = async (runsDir) => readdir(runsDir, { withFileTypes: true }).then((entries) => entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name))@@ -24,6 +33,68 @@ const asList = (records) => (Array.isArray(records) ? records.flat(Infinity) : [const isWorthRecording = (records, siblings) => siblings.length > 0|| records.some((entry) => entry?.source === 'measured' || entry?.source === 'mixed');+const runWindow = (metadata) => ({+ from: Date.parse(metadata.startTime) - WINDOW_SLACK_MS,+ to: Date.parse(metadata.endTime) + WINDOW_SLACK_MS+});++// Transcript readers match a cwd or any subdirectory of it, so runs whose+// workspaces nest inside one another can read the same session.+const workspaceAncestors = (path) => {+ const ancestors = [];+ for (let cut = path.lastIndexOf('/'); cut > 0; cut = path.lastIndexOf('/', cut - 1)) {+ ancestors.push(path.slice(0, cut));+ }+ return ancestors;+};++/**+ * Re-claim what settled runs already billed, before any pending run is read.+ *+ * Every message inside a settled pass's window was billed — by that run or by+ * an earlier overlapping one — so re-reading the window and discarding the+ * counts claims exactly what is already on the books. The live ledger handed+ * over in `claimsSeed` cannot carry that alone: it is empty after a restart+ * and prunes claims past its retention bound, while this scan reaches runs of+ * any age. Only settled runs that can overlap a pending one are re-read, and+ * only for the families each settled pass actually covered.+ */+async function claimSettledUsage(runs, candidates, { home, providers }) {+ const below = new Map();+ const at = new Map();+ const index = (map, key, window) => {+ if (!map.has(key)) map.set(key, []);+ map.get(key).push(window);+ };+ for (const { metadata } of candidates) {+ const window = runWindow(metadata);+ index(at, metadata.workspacePath, window);+ index(below, metadata.workspacePath, window);+ for (const ancestor of workspaceAncestors(metadata.workspacePath)) index(below, ancestor, window);+ }+ const canScanSiblings = Array.isArray(providers) && providers.length > 0;+ for (const { metadata, parentSettled, siblingSettled } of runs) {+ const { workspacePath } = metadata;+ const window = runWindow(metadata);+ const pending = [+ ...(below.get(workspacePath) || []),+ ...workspaceAncestors(workspacePath).flatMap((ancestor) => at.get(ancestor) || [])+ ];+ if (!pending.some((other) => other.from <= window.to && window.from <= other.to)) continue;+ const parentFamily = transcriptFamily(metadata);+ const families = parentSettled && parentFamily ? [parentFamily] : [];+ if (siblingSettled && canScanSiblings) {+ families.push(...TRANSCRIPT_FAMILIES.filter((family) => (+ family !== parentFamily && resolveFamilyProvider(providers, family)+ )));+ }+ for (const family of families) {+ await readMeasuredUsage({ workspacePath, startTime: metadata.startTime, endTime: metadata.endTime, family, home })+ .catch(() => null);+ }+ }+}+/*** Read historical run artifacts and produce estimate→measurement corrections.* This function runs inside a worker in production so parsing large JSONL files@@ -65,6 +136,7 @@ export async function scanHistoricalUsage({const siblingReconciled = new Set(siblingReconciledRunIds);const canScanSiblings = Array.isArray(providers) && providers.length > 0;const candidates = [];+ const settled = [];for (const runId of await listRunIds(runsDir)) {const metadataPath = join(runsDir, runId, 'metadata.json');@@ -79,9 +151,13 @@ export async function scanHistoricalUsage({// no transcript of its own.const siblingPending = canScanSiblings&& !siblingReconciled.has(runId) && !metadata.usageSiblingsReconciled;+ const parentSettled = reconciled.has(runId) || metadata.usageReconciled === true;+ const siblingSettled = siblingReconciled.has(runId) || metadata.usageSiblingsReconciled === true;+ if (parentSettled || siblingSettled) settled.push({ metadata, parentSettled, siblingSettled });if (!parentPending && !siblingPending) continue;candidates.push({ runId, metadataPath, metadata, parentPending, siblingPending });}+ if (candidates.length) await claimSettledUsage(settled, candidates, { home, providers });candidates.sort((a, b) => Date.parse(a.metadata.startTime) - Date.parse(b.metadata.startTime));const corrections = [];diff --git a/server/services/usageBackfillWorker.test.js b/server/services/usageBackfillWorker.test.jsindex 172967f34a..3b049d0bde 100644--- a/server/services/usageBackfillWorker.test.js+++ b/server/services/usageBackfillWorker.test.js@@ -117,6 +117,30 @@ describe('historical usage worker', () => {expect(result.corrections).toEqual([]);expect(result.claimsSnapshot.claimedMessages).toEqual(claimsSeed.claimedMessages);});++ // The live ledger is empty after a restart and prunes old claims, so a+ // settled run on disk is the durable record of what was already billed.+ it('does not re-bill what a settled run billed once the live claim is gone', async () => {+ await writeRun('run-settled-live', {+ providerId: 'claude-code-tui',+ model: 'claude-opus-5',+ workspacePath: workspace,+ startTime: '2026-07-01T10:00:00.000Z',+ endTime: '2026-07-01T10:10:00.000Z',+ usageReconciled: true+ });+ await writeRun('run-overlapping-history', {+ providerId: 'claude-code-tui',+ model: 'claude-opus-5',+ workspacePath: workspace,+ startTime: '2026-07-01T10:02:00.000Z',+ endTime: '2026-07-01T10:12:00.000Z'+ });++ const result = await scanHistoricalUsage({ runsDir: root, home, claimsSeed: snapshotUsageClaims() });+ expect(result.total).toBe(1);+ expect(result.corrections).toEqual([]);+ });});// A nested `--review-with grok` pass leaves a grok session but never a PortOS@@ -128,8 +152,8 @@ describe('historical sibling-family attribution', () => {const GROK_PROVIDER = { id: 'grok-cli', type: 'cli', command: 'grok', enabled: true, defaultModel: GROK_MODEL };const CLAUDE_PROVIDER = { id: 'claude-code-tui', type: 'tui', command: 'claude', enabled: true, defaultModel: 'claude-opus-5' };- const writeGrokSession = async () => {- const dir = join(home, '.grok', 'sessions', encodeURIComponent(workspace), 'session-aaaa');+ const writeGrokSession = async (cwd = workspace) => {+ const dir = join(home, '.grok', 'sessions', encodeURIComponent(cwd), 'session-aaaa');await mkdir(dir, { recursive: true });await writeFile(join(dir, 'updates.jsonl'), JSON.stringify({timestamp: Math.round(Date.parse('2026-07-01T10:05:00.000Z') / 1000),@@ -194,6 +218,28 @@ describe('historical sibling-family attribution', () => {expect(result.corrections).toEqual([]);});+ // Readers match a cwd or any subdirectory of it, so the settled run and the+ // pending one can sit at, above, or below each other and share one session.+ it.each([+ ['the same workspace', workspace, workspace],+ ['a parent workspace', '/work', workspace],+ ['a subdirectory workspace', `${workspace}/server`, `${workspace}/server`]+ ])('does not re-bill a nested session a settled run already covered, from %s', async (_label, pendingWorkspace, sessionCwd) => {+ await writeRun('run-already-scanned', { ...reconciledRun, usageSiblingsReconciled: true });+ await writeRun('run-overlapping-history', {+ ...reconciledRun,+ workspacePath: pendingWorkspace,+ startTime: '2026-07-01T10:04:00.000Z',+ endTime: '2026-07-01T10:14:00.000Z',+ usageReconciled: false+ });+ await writeGrokSession(sessionCwd);++ const result = await scanHistoricalUsage({ runsDir: root, home, providers: [CLAUDE_PROVIDER, GROK_PROVIDER] });+ expect(result.total).toBe(1);+ expect(result.corrections).toEqual([]);+ });+it('does no sibling work at all when the install has no provider list', async () => {await writeRun('run-already-measured', reconciledRun);await writeGrokSession();diff --git a/server/services/usageReconciler.js b/server/services/usageReconciler.jsindex f033e3505e..f2600ff940 100644--- a/server/services/usageReconciler.js+++ b/server/services/usageReconciler.js@@ -80,9 +80,11 @@ export const WINDOW_SLACK_MS = 60_000;// needed only while some run still to be reconciled can start before the// message. Every run path is bounded far below this (the CoS max-runtime// ceiling, the toolkit's API run cap), and the ledger never sees in-flight-// runs, so a fixed bound stands in for "oldest pending run start". A run that-// outlives it can re-bill a pruned neighbour's overlap — the same exposure a-// server restart, which empties the ledger, already accepts.+// runs, so a fixed bound stands in for "oldest pending run start". A live run+// that outlives it can re-bill a pruned neighbour's overlap — the same exposure+// a server restart, which empties the ledger, already accepts. The historical+// backfill, which reaches runs of any age, does not rely on these claims for+// old windows: it re-claims what settled runs billed from their run records.export const CLAIM_RETENTION_MS = 48 * 60 * 60 * 1000;// Messages already billed to a run: transcript key -> message key -> stamp.@@ -946,20 +948,27 @@ export async function recordCompletedRunUsage(metadata, output, { home = homedirconst resolved = providers ?? await import('./providers.js').then((module) => module.listProviders()).catch(() => null);+ // Same condition `reconcileRunUsage` scans siblings under.+ const siblingsScanned = Array.isArray(resolved) && resolved.length > 0;// One catch for the whole chain: whatever fails — reading a transcript or// persisting the record — usage accounting must not surface as a run failure.await reconcileRunUsage(metadata, estimate, { home, providers: resolved, estimateSource }).then(recordRunUsage).then(async () => {if (!metadata?.id) return;- await markUsageRunReconciled(metadata.id);+ await markUsageRunReconciled(metadata.id, { siblings: siblingsScanned });const metadataPath = join(PATHS.runs, metadata.id, 'metadata.json');// Non-strict exception: missing/unreadable metadata skips the marker// write; the usage ledger remains authoritative for the accounting.const persisted = await readJSONFile(metadataPath, null);if (!persisted) return;+ const now = new Date().toISOString();persisted.usageReconciled = true;- persisted.usageReconciledAt = new Date().toISOString();+ persisted.usageReconciledAt = now;+ if (siblingsScanned) {+ persisted.usageSiblingsReconciled = true;+ persisted.usageSiblingsReconciledAt = now;+ }await atomicWrite(metadataPath, persisted);}).catch((err) => {diff --git a/server/services/usageReconciler.test.js b/server/services/usageReconciler.test.jsindex 48deb7c6c1..6394874abc 100644--- a/server/services/usageReconciler.test.js+++ b/server/services/usageReconciler.test.js@@ -509,17 +509,24 @@ describe('recordCompletedRunUsage', () => {});it('durably marks a completed run so a later backfill skips it', async () => {- await recordCompletedRunUsage({- id: 'run-example-live',+ const run = {providerId: 'ollama',model: 'llama3',workspacePath: WORKSPACE,promptLength: 40,startTime: '2026-07-01T10:00:00.000Z',endTime: '2026-07-01T10:10:00.000Z'- }, 'captured output');+ };+ await recordCompletedRunUsage({ ...run, id: 'run-example-live' }, 'captured output', { home, providers: [] });+ // With a provider list the nested-session pass ran too, so a backfill must+ // not bill those sessions a second time.+ await recordCompletedRunUsage({ ...run, id: 'run-example-scanned' }, 'captured output', {+ home,+ providers: [{ id: 'grok-cli', type: 'cli', command: 'grok', enabled: true }]+ });- expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-live');+ expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-live', { siblings: false });+ expect(markUsageRunReconciled).toHaveBeenCalledWith('run-example-scanned', { siblings: true });});it('records the estimate when no transcript exists rather than recording nothing', async () => {