Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: skip stray Grok session files during usage repair (#13435)
[._.]/9992c79eda11
diff --git a/server/services/grokUsageBackfill.js b/server/services/grokUsageBackfill.jsindex 479a67ded2..6e3f90c62c 100644--- a/server/services/grokUsageBackfill.js+++ b/server/services/grokUsageBackfill.js@@ -30,22 +30,60 @@ import { addDatedUsage, flushUsageLedger, getUsage, loadUsage } from './usage.jsconst yieldToEventLoop = () => new Promise((done) => setImmediate(done));-async function readJson(path, fallback) {+// Node's filesystem errors put the absolute path in `message`. Log `code` for+// those so a home directory never lands in the process log. A parse failure+// has no code; its message does not include a path.+const faultLabel = (err) => {+ if (err?.code) return String(err.code);+ return String(err?.message || err?.name || 'error').replace(/[\r\n]+/g, ' ');+};++const oneLine = (value) => String(value ?? '').replace(/[\r\n]+/g, ' ');++/**+ * Read JSON, or `fallback` when the file is missing.+ *+ * A present-but-unreadable file is also `fallback` unless `strict`. Session+ * summaries, gallery sidecars, video history, the scan watermark, and provider+ * config are per-entry inputs: one bad file must not abort the repair.+ * `usage.json` stays strict so a corrupt ledger is not planned as an empty+ * repair-id list.+ */+async function readJson(path, fallback, { strict = false, label = 'file' } = {}) {try {return JSON.parse(await readFile(path, 'utf8'));} catch (err) {if (err.code === 'ENOENT') return fallback;- throw err;+ if (strict) throw err;+ console.warn(`⚠️ Grok usage repair skipped unreadable ${label} (${faultLabel(err)})`);+ return fallback;}}-async function listDir(path) {+/**+ * Entry names in `dir`, or [] when `dir` is missing.+ *+ * `directories` keeps only real directories. Grok writes a flat+ * `prompt_history.jsonl` beside the session-id directories inside each cwd+ * folder, and other files can sit next to the cwd folders themselves. Opening+ * `<file>/summary.json` throws ENOTDIR and aborts the repair — the layout+ * `listSubdirs` in usageReconciler.js already filters (#6218). This copy stays+ * local so the repair does not import the reconciler's claim ledger and+ * transcript readers. Any other directory error is skipped for the same+ * reason: one unreadable folder must not abort the repair.+ */+async function listNames(dir, { directories = false } = {}) {+ let entries;try {- return await readdir(path);+ entries = await readdir(dir, { withFileTypes: true });} catch (err) {- if (err.code === 'ENOENT') return [];- throw err;+ if (err.code !== 'ENOENT') {+ console.warn(`⚠️ Grok usage repair skipped a directory (${faultLabel(err)})`);+ }+ return [];}+ const wanted = directories ? entries.filter((entry) => entry.isDirectory()) : entries;+ return wanted.map((entry) => entry.name);}async function fileMtime(path) {@@ -60,28 +98,36 @@ async function collectSessions(home) {const root = join(home, '.grok', 'sessions');const turns = [];const media = [];- for (const dirName of await listDir(root)) {+ for (const dirName of await listNames(root, { directories: true })) {let cwd = dirName;try { cwd = decodeURIComponent(dirName); } catch { /* keep the raw directory name */ }const cwdDir = join(root, dirName);- for (const sessionId of await listDir(cwdDir)) {- const sessionDir = join(cwdDir, sessionId);- const updatesPath = join(sessionDir, 'updates.jsonl');- const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');- if (updatesText) {- const parsed = parseGrokTurns(updatesText);- for (const turn of parsed.longTurns || []) turns.push(turn);+ for (const sessionId of await listNames(cwdDir, { directories: true })) {+ try {+ const sessionDir = join(cwdDir, sessionId);+ const updatesPath = join(sessionDir, 'updates.jsonl');+ const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');+ if (updatesText) {+ const parsed = parseGrokTurns(updatesText);+ for (const turn of parsed.longTurns || []) turns.push(turn);+ }+ const summary = await readJson(join(sessionDir, 'summary.json'), null, {+ label: `summary.json for session ${oneLine(sessionId)}`,+ });+ const chatPath = join(sessionDir, 'chat_history.jsonl');+ const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')+ || await fileMtime(chatPath)+ || await fileMtime(updatesPath);+ const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;+ const chatText = await readFile(chatPath, 'utf8').catch(() => '');+ media.push(...imagineEventsFromGrokSession({+ chatText, updatesText, sessionId, cwd, day,+ }));+ } catch (err) {+ // One bad session must not drop the gallery, video history, or the+ // sessions already read.+ console.warn(`⚠️ Grok usage repair skipped session ${oneLine(sessionId)} (${faultLabel(err)})`);}- const summary = await readJson(join(sessionDir, 'summary.json'), null);- const chatPath = join(sessionDir, 'chat_history.jsonl');- const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')- || await fileMtime(chatPath)- || await fileMtime(updatesPath);- const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;- const chatText = await readFile(chatPath, 'utf8').catch(() => '');- media.push(...imagineEventsFromGrokSession({- chatText, updatesText, sessionId, cwd, day,- }));await yieldToEventLoop();}}@@ -91,22 +137,34 @@ async function collectSessions(home) {async function collectGallery(dataDir) {const dir = join(dataDir, 'images');const media = [];- for (const name of await listDir(dir)) {+ for (const name of await listNames(dir)) {if (!name.endsWith('.metadata.json')) continue;- const meta = await readJson(join(dir, name), null);- const event = galleryImageUsageEvent(meta);- if (event) media.push(event);+ try {+ const meta = await readJson(join(dir, name), null, { label: oneLine(name) });+ const event = galleryImageUsageEvent(meta);+ if (event) media.push(event);+ } catch (err) {+ console.warn(`⚠️ Grok usage repair skipped gallery file ${oneLine(name)} (${faultLabel(err)})`);+ }}return media;}async function collectVideos(dataDir) {- const history = await readJson(join(dataDir, 'video-history.json'), []);- return (Array.isArray(history) ? history : []).flatMap(videoHistoryUsageEvents);+ const history = await readJson(join(dataDir, 'video-history.json'), [], { label: 'video-history.json' });+ const media = [];+ for (const entry of Array.isArray(history) ? history : []) {+ try {+ media.push(...videoHistoryUsageEvents(entry));+ } catch (err) {+ console.warn(`⚠️ Grok usage repair skipped a video history entry (${faultLabel(err)})`);+ }+ }+ return media;}async function providerIdFromFile(dataDir) {- const stored = await readJson(join(dataDir, 'providers.json'), null);+ const stored = await readJson(join(dataDir, 'providers.json'), null, { label: 'providers.json' });const list = Array.isArray(stored) ? stored : (stored?.providers || []);const grok = list.filter((provider) => provider && familyForProvider(provider) === 'grok');return grokUsageProviderIdFromList(grok.length ? list : []);@@ -123,9 +181,11 @@ export async function planGrokUsageRepairFromDisk({repairIds = null,providerId = null,} = {}) {- const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} });+ const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} }, {+ label: 'usage-interactive-scan.json',+ });const watermarkMs = typeof scan?.watermarks?.grok === 'number' ? scan.watermarks.grok : null;- const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}))?.grokRepairIds;+ const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}, { strict: true }))?.grokRepairIds;const sessions = await collectSessions(home);return planGrokUsageRepair({turns: sessions.turns,diff --git a/server/services/grokUsageBackfill.test.js b/server/services/grokUsageBackfill.test.jsindex 9599ce72c7..0af0fc2d36 100644--- a/server/services/grokUsageBackfill.test.js+++ b/server/services/grokUsageBackfill.test.js@@ -29,6 +29,28 @@ const { flushUsageLedger } = await import('./usage.js');let home;let dataDir;+const IMAGE_CHAT = `${JSON.stringify({+ type: 'assistant',+ tool_calls: [{ name: 'image_gen', arguments: '{}' }],+})}\n`;++async function writeGrokSession({+ cwd = '/tmp/example-repo',+ sessionId = 'session-1',+ summary = { created_at: '2026-10-04T12:00:00.000Z' },+ chat = IMAGE_CHAT,+} = {}) {+ const cwdDir = join(home, '.grok', 'sessions', encodeURIComponent(cwd));+ const sessionDir = join(cwdDir, sessionId);+ await mkdir(sessionDir, { recursive: true });+ if (summary !== null) {+ const body = typeof summary === 'string' ? summary : JSON.stringify(summary);+ await writeFile(join(sessionDir, 'summary.json'), body);+ }+ if (chat !== null) await writeFile(join(sessionDir, 'chat_history.jsonl'), chat);+ return cwdDir;+}+beforeEach(async () => {home = await mkdtemp(join(tmpdir(), 'grok-repair-home-'));dataDir = await mkdtemp(join(tmpdir(), 'grok-repair-data-'));@@ -78,4 +100,63 @@ describe('applyGrokUsageRepair', () => {.rejects.toThrow(/usage\.json/);expect(written).toHaveLength(0);});++ it('repairs the real session when a cwd folder also holds prompt_history.jsonl', async () => {+ // Grok writes that file beside the session-id directories, and other files+ // can sit next to the cwd folders. Treating either name as a directory+ // threw ENOTDIR on `<file>/summary.json` and aborted Reconcile now.+ const cwdDir = await writeGrokSession();+ await writeFile(join(cwdDir, 'prompt_history.jsonl'), '{"prompt":"not a session"}\n');+ await writeFile(join(home, '.grok', 'sessions', 'not-a-cwd.txt'), 'stray');+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const result = await applyGrokUsageRepair({ home, dataDir });+ expect(result).toMatchObject({ added: 2, images: 2, turns: 0, videos: 0, backedUp: true });+ expect(written.map((row) => row.repairId).sort()).toEqual([+ 'gallery:job-1',+ 'session-image:session-1:0',+ ]);+ expect(warn).not.toHaveBeenCalled();+ } finally {+ warn.mockRestore();+ }+ });+});++describe('planGrokUsageRepairFromDisk bad inputs', () => {+ it('skips an unreadable session file, gallery sidecar, video history, and scan without dropping the rest', async () => {+ await writeGrokSession({ sessionId: 'session-good' });+ await writeGrokSession({ sessionId: 'session-bad', summary: '{not-json' });+ await writeFile(join(dataDir, 'images', 'broken.metadata.json'), '{not-json');+ await writeFile(join(dataDir, 'video-history.json'), '{not-json');+ await writeFile(join(dataDir, 'usage-interactive-scan.json'), '{not-json');+ await writeFile(join(dataDir, 'providers.json'), '{not-json');+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const plan = await planGrokUsageRepairFromDisk({ home, dataDir });+ expect(plan.additions.map((item) => item.repairId).sort()).toEqual([+ 'gallery:job-1',+ 'session-image:session-bad:0',+ 'session-image:session-good:0',+ ]);+ expect(plan.additions.every((item) => item.record.providerId === 'grok')).toBe(true);+ expect(plan.notes[0]).toMatch(/no Grok usage scan has run yet/);+ expect(warn).toHaveBeenCalledTimes(5);+ expect(warn.mock.calls.map((call) => String(call[0]))).toEqual(expect.arrayContaining([+ expect.stringContaining('summary.json for session session-bad'),+ expect.stringContaining('broken.metadata.json'),+ expect.stringContaining('video-history.json'),+ expect.stringContaining('usage-interactive-scan.json'),+ expect.stringContaining('providers.json'),+ ]));+ for (const [line] of warn.mock.calls) expect(String(line)).not.toContain(home);+ } finally {+ warn.mockRestore();+ }+ });++ it('does not treat an unreadable usage ledger as an empty repair list', async () => {+ await writeFile(join(dataDir, 'usage.json'), '{not-json');+ await expect(planGrokUsageRepairFromDisk({ home, dataDir })).rejects.toThrow(SyntaxError);+ });});