Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(fleet): handle unreadable usage ledger strictly and block destructive persist (#12111)
[._.]/d5b5e3f80d73
diff --git a/server/services/fleetLlmUsage.js b/server/services/fleetLlmUsage.jsindex 8d1f5c282d..a1591d82a9 100644--- a/server/services/fleetLlmUsage.js+++ b/server/services/fleetLlmUsage.js@@ -28,7 +28,7 @@*/import { dataPath } from '../lib/paths.js';-import { atomicWrite, readJSONFile } from '../lib/fileUtils.js';+import { atomicWrite, readJSONFileStrict } from '../lib/fileUtils.js';import { createFileWriteQueue } from '../lib/fileWriteQueue.js';import { createFleetHostUsageLedger, normalizeClientAddress } from '../lib/fleetHostUsage.js';import { getTailscaleStatus } from '../lib/tailscale.js';@@ -46,13 +46,26 @@ const queue = createFileWriteQueue();let ledger = null;let hydrating = null;let persistTimer = null;+let persistBlocked = false;+let loggedReadFailure = false;-/** The process-wide ledger, hydrated from disk exactly once. */+/** The process-wide ledger, hydrated from disk exactly once unless a read failed. */export async function getFleetHostUsageLedger() {- if (ledger) return ledger;+ if (ledger && !persistBlocked) return ledger;if (!hydrating) {- const next = createFleetHostUsageLedger();- hydrating = readJSONFile(USAGE_FILE, null, { logError: false }).then((stored) => {+ const next = ledger || createFleetHostUsageLedger();+ hydrating = readJSONFileStrict(USAGE_FILE, null, { logError: false }).then(({ ok, value: stored }) => {+ hydrating = null;+ if (!ok) {+ if (!loggedReadFailure) {+ loggedReadFailure = true;+ console.error(`❌ Could not read fleet host usage from ${USAGE_FILE}; in-memory ledger will not overwrite disk until a read succeeds`);+ }+ persistBlocked = true;+ ledger = next;+ return next;+ }+ persistBlocked = false;next.hydrate(stored);ledger = next;return next;@@ -62,7 +75,7 @@ export async function getFleetHostUsageLedger() {}async function writeLedger() {- if (!ledger) return;+ if (!ledger || persistBlocked) return;const payload = ledger.toJSON();await queue(() => atomicWrite(USAGE_FILE, JSON.stringify(payload, null, 2)))// Outside the request lifecycle (a debounce timer), so a throw here would@@ -92,6 +105,8 @@ export function __resetFleetHostUsage() {if (persistTimer) { clearTimeout(persistTimer); persistTimer = null; }ledger = null;hydrating = null;+ persistBlocked = false;+ loggedReadFailure = false;}/**diff --git a/server/services/fleetLlmUsage.test.js b/server/services/fleetLlmUsage.test.jsindex 898ce19dc6..115a375b38 100644--- a/server/services/fleetLlmUsage.test.js+++ b/server/services/fleetLlmUsage.test.js@@ -1,12 +1,12 @@import { beforeEach, describe, expect, it, vi } from 'vitest';-const state = vi.hoisted(() => ({ peers: [], tailnet: { peers: [] }, stored: null, written: [] }));+const state = vi.hoisted(() => ({ peers: [], tailnet: { peers: [] }, stored: { ok: true, value: null }, written: [] }));vi.mock('./instances.js', () => ({ getPeers: async () => state.peers }));vi.mock('../lib/tailscale.js', () => ({ getTailscaleStatus: async () => state.tailnet }));vi.mock('../lib/paths.js', () => ({ dataPath: (...parts) => `/example/data/${parts.join('/')}` }));vi.mock('../lib/fileUtils.js', () => ({- readJSONFile: async () => state.stored,+ readJSONFileStrict: async () => state.stored,atomicWrite: async (path, value) => state.written.push({ path, value }),}));@@ -15,7 +15,7 @@ const { __resetFleetHostUsage, flushFleetHostUsage, getFleetHostUsageLedger, getbeforeEach(() => {state.peers = [];state.tailnet = { peers: [] };- state.stored = null;+ state.stored = { ok: true, value: null };state.written = [];__resetFleetHostUsage();});@@ -49,22 +49,25 @@ describe('fleet host usage report', () => {it('serves a report from disk after a restart, without a phantom in-flight request', async () => {state.stored = {- version: 1,- since: Date.parse('2026-09-01T00:00:00Z'),- clients: [{- key: '192.0.2.10',- firstSeen: Date.parse('2026-09-19T00:00:00Z'),- lastSeen: Date.now(),- requests: 4,- errors: 0,- tokenReports: 4,- promptTokens: 400,- completionTokens: 900,- activeRequests: 3,- models: ['qwen3.8-27b'],- days: {},- }],- recent: [],+ ok: true,+ value: {+ version: 1,+ since: Date.parse('2026-09-01T00:00:00Z'),+ clients: [{+ key: '192.0.2.10',+ firstSeen: Date.parse('2026-09-19T00:00:00Z'),+ lastSeen: Date.now(),+ requests: 4,+ errors: 0,+ tokenReports: 4,+ promptTokens: 400,+ completionTokens: 900,+ activeRequests: 3,+ models: ['qwen3.8-27b'],+ days: {},+ }],+ recent: [],+ },};const report = await getFleetHostUsageReport();@@ -91,4 +94,79 @@ describe('fleet host usage report', () => {'clientKey', 'completionTokens', 'durationMs', 'finishedAt', 'id', 'model', 'path', 'promptTokens', 'startedAt', 'status',]);});++ it('skips persisting and logs error once when usage file is unreadable, then resumes on successful read', async () => {+ const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});++ // First attempt: unreadable file (e.g. EACCES, torn write)+ state.stored = { ok: false, value: null };++ const ledger = await getFleetHostUsageLedger();+ expect(errorSpy).toHaveBeenCalledTimes(1);+ expect(errorSpy.mock.calls[0][0]).toContain('Could not read fleet host usage');++ // Second touch while still unreadable: logs no additional error+ await getFleetHostUsageLedger();+ expect(errorSpy).toHaveBeenCalledTimes(1);++ // Record request and try to flush+ ledger.endRequest(+ ledger.beginRequest({ address: '192.0.2.10', path: '/v1/chat/completions' }),+ { status: 200, usage: { promptTokens: 5, completionTokens: 6 } },+ );+ await flushFleetHostUsage();++ // Persist must be skipped to avoid overwriting unreadable file+ expect(state.written).toHaveLength(0);++ // File becomes readable (e.g. lock released or permissions fixed)+ state.stored = {+ ok: true,+ value: {+ version: 1,+ since: Date.parse('2026-09-01T00:00:00Z'),+ clients: [{+ key: '192.0.2.10',+ firstSeen: Date.parse('2026-09-01T00:00:00Z'),+ lastSeen: Date.now(),+ requests: 10,+ errors: 0,+ tokenReports: 10,+ promptTokens: 1000,+ completionTokens: 2000,+ activeRequests: 0,+ models: [],+ days: {},+ }],+ recent: [],+ },+ };++ const recoveredLedger = await getFleetHostUsageLedger();+ expect(recoveredLedger).toBe(ledger);++ // Now flush should proceed and persist loaded history+ await flushFleetHostUsage();+ expect(state.written).toHaveLength(1);+ const payload = JSON.parse(state.written[0].value);+ expect(payload.clients[0].requests).toBe(10);+ expect(payload.clients[0].completionTokens).toBe(2000);++ errorSpy.mockRestore();+ });++ it('absent file hydrates empty and persists normally', async () => {+ state.stored = { ok: true, value: null };++ const ledger = await getFleetHostUsageLedger();+ ledger.endRequest(+ ledger.beginRequest({ address: '192.0.2.10', path: '/v1/chat/completions' }),+ { status: 200, usage: { promptTokens: 1, completionTokens: 1 } },+ );+ await flushFleetHostUsage();++ expect(state.written).toHaveLength(1);+ const payload = JSON.parse(state.written[0].value);+ expect(payload.clients[0].requests).toBe(1);+ });});