Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: skip stray Grok session files during usage repair (#13435)

[._.]/9992c79eda11
diff --git a/server/services/grokUsageBackfill.js b/server/services/grokUsageBackfill.js
index 479a67ded2..6e3f90c62c 100644
--- a/server/services/grokUsageBackfill.js
+++ b/server/services/grokUsageBackfill.js
@@ -30,22 +30,60 @@ import { addDatedUsage, flushUsageLedger, getUsage, loadUsage } from './usage.js
const yieldToEventLoop = () => new Promise((done) => setImmediate(done));
-async function readJson(path, fallback) {
+// Node's filesystem errors put the absolute path in `message`. Log `code` for
+// those so a home directory never lands in the process log. A parse failure
+// has no code; its message does not include a path.
+const faultLabel = (err) => {
+ if (err?.code) return String(err.code);
+ return String(err?.message || err?.name || 'error').replace(/[\r\n]+/g, ' ');
+};
+
+const oneLine = (value) => String(value ?? '').replace(/[\r\n]+/g, ' ');
+
+/**
+ * Read JSON, or `fallback` when the file is missing.
+ *
+ * A present-but-unreadable file is also `fallback` unless `strict`. Session
+ * summaries, gallery sidecars, video history, the scan watermark, and provider
+ * config are per-entry inputs: one bad file must not abort the repair.
+ * `usage.json` stays strict so a corrupt ledger is not planned as an empty
+ * repair-id list.
+ */
+async function readJson(path, fallback, { strict = false, label = 'file' } = {}) {
try {
return JSON.parse(await readFile(path, 'utf8'));
} catch (err) {
if (err.code === 'ENOENT') return fallback;
- throw err;
+ if (strict) throw err;
+ console.warn(`⚠️ Grok usage repair skipped unreadable ${label} (${faultLabel(err)})`);
+ return fallback;
}
}
-async function listDir(path) {
+/**
+ * Entry names in `dir`, or [] when `dir` is missing.
+ *
+ * `directories` keeps only real directories. Grok writes a flat
+ * `prompt_history.jsonl` beside the session-id directories inside each cwd
+ * folder, and other files can sit next to the cwd folders themselves. Opening
+ * `<file>/summary.json` throws ENOTDIR and aborts the repair — the layout
+ * `listSubdirs` in usageReconciler.js already filters (#6218). This copy stays
+ * local so the repair does not import the reconciler's claim ledger and
+ * transcript readers. Any other directory error is skipped for the same
+ * reason: one unreadable folder must not abort the repair.
+ */
+async function listNames(dir, { directories = false } = {}) {
+ let entries;
try {
- return await readdir(path);
+ entries = await readdir(dir, { withFileTypes: true });
} catch (err) {
- if (err.code === 'ENOENT') return [];
- throw err;
+ if (err.code !== 'ENOENT') {
+ console.warn(`⚠️ Grok usage repair skipped a directory (${faultLabel(err)})`);
+ }
+ return [];
}
+ const wanted = directories ? entries.filter((entry) => entry.isDirectory()) : entries;
+ return wanted.map((entry) => entry.name);
}
async function fileMtime(path) {
@@ -60,28 +98,36 @@ async function collectSessions(home) {
const root = join(home, '.grok', 'sessions');
const turns = [];
const media = [];
- for (const dirName of await listDir(root)) {
+ for (const dirName of await listNames(root, { directories: true })) {
let cwd = dirName;
try { cwd = decodeURIComponent(dirName); } catch { /* keep the raw directory name */ }
const cwdDir = join(root, dirName);
- for (const sessionId of await listDir(cwdDir)) {
- const sessionDir = join(cwdDir, sessionId);
- const updatesPath = join(sessionDir, 'updates.jsonl');
- const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');
- if (updatesText) {
- const parsed = parseGrokTurns(updatesText);
- for (const turn of parsed.longTurns || []) turns.push(turn);
+ for (const sessionId of await listNames(cwdDir, { directories: true })) {
+ try {
+ const sessionDir = join(cwdDir, sessionId);
+ const updatesPath = join(sessionDir, 'updates.jsonl');
+ const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');
+ if (updatesText) {
+ const parsed = parseGrokTurns(updatesText);
+ for (const turn of parsed.longTurns || []) turns.push(turn);
+ }
+ const summary = await readJson(join(sessionDir, 'summary.json'), null, {
+ label: `summary.json for session ${oneLine(sessionId)}`,
+ });
+ const chatPath = join(sessionDir, 'chat_history.jsonl');
+ const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')
+ || await fileMtime(chatPath)
+ || await fileMtime(updatesPath);
+ const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;
+ const chatText = await readFile(chatPath, 'utf8').catch(() => '');
+ media.push(...imagineEventsFromGrokSession({
+ chatText, updatesText, sessionId, cwd, day,
+ }));
+ } catch (err) {
+ // One bad session must not drop the gallery, video history, or the
+ // sessions already read.
+ console.warn(`⚠️ Grok usage repair skipped session ${oneLine(sessionId)} (${faultLabel(err)})`);
}
- const summary = await readJson(join(sessionDir, 'summary.json'), null);
- const chatPath = join(sessionDir, 'chat_history.jsonl');
- const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')
- || await fileMtime(chatPath)
- || await fileMtime(updatesPath);
- const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;
- const chatText = await readFile(chatPath, 'utf8').catch(() => '');
- media.push(...imagineEventsFromGrokSession({
- chatText, updatesText, sessionId, cwd, day,
- }));
await yieldToEventLoop();
}
}
@@ -91,22 +137,34 @@ async function collectSessions(home) {
async function collectGallery(dataDir) {
const dir = join(dataDir, 'images');
const media = [];
- for (const name of await listDir(dir)) {
+ for (const name of await listNames(dir)) {
if (!name.endsWith('.metadata.json')) continue;
- const meta = await readJson(join(dir, name), null);
- const event = galleryImageUsageEvent(meta);
- if (event) media.push(event);
+ try {
+ const meta = await readJson(join(dir, name), null, { label: oneLine(name) });
+ const event = galleryImageUsageEvent(meta);
+ if (event) media.push(event);
+ } catch (err) {
+ console.warn(`⚠️ Grok usage repair skipped gallery file ${oneLine(name)} (${faultLabel(err)})`);
+ }
}
return media;
}
async function collectVideos(dataDir) {
- const history = await readJson(join(dataDir, 'video-history.json'), []);
- return (Array.isArray(history) ? history : []).flatMap(videoHistoryUsageEvents);
+ const history = await readJson(join(dataDir, 'video-history.json'), [], { label: 'video-history.json' });
+ const media = [];
+ for (const entry of Array.isArray(history) ? history : []) {
+ try {
+ media.push(...videoHistoryUsageEvents(entry));
+ } catch (err) {
+ console.warn(`⚠️ Grok usage repair skipped a video history entry (${faultLabel(err)})`);
+ }
+ }
+ return media;
}
async function providerIdFromFile(dataDir) {
- const stored = await readJson(join(dataDir, 'providers.json'), null);
+ const stored = await readJson(join(dataDir, 'providers.json'), null, { label: 'providers.json' });
const list = Array.isArray(stored) ? stored : (stored?.providers || []);
const grok = list.filter((provider) => provider && familyForProvider(provider) === 'grok');
return grokUsageProviderIdFromList(grok.length ? list : []);
@@ -123,9 +181,11 @@ export async function planGrokUsageRepairFromDisk({
repairIds = null,
providerId = null,
} = {}) {
- const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} });
+ const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} }, {
+ label: 'usage-interactive-scan.json',
+ });
const watermarkMs = typeof scan?.watermarks?.grok === 'number' ? scan.watermarks.grok : null;
- const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}))?.grokRepairIds;
+ const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}, { strict: true }))?.grokRepairIds;
const sessions = await collectSessions(home);
return planGrokUsageRepair({
turns: sessions.turns,
diff --git a/server/services/grokUsageBackfill.test.js b/server/services/grokUsageBackfill.test.js
index 9599ce72c7..0af0fc2d36 100644
--- a/server/services/grokUsageBackfill.test.js
+++ b/server/services/grokUsageBackfill.test.js
@@ -29,6 +29,28 @@ const { flushUsageLedger } = await import('./usage.js');
let home;
let dataDir;
+const IMAGE_CHAT = `${JSON.stringify({
+ type: 'assistant',
+ tool_calls: [{ name: 'image_gen', arguments: '{}' }],
+})}\n`;
+
+async function writeGrokSession({
+ cwd = '/tmp/example-repo',
+ sessionId = 'session-1',
+ summary = { created_at: '2026-10-04T12:00:00.000Z' },
+ chat = IMAGE_CHAT,
+} = {}) {
+ const cwdDir = join(home, '.grok', 'sessions', encodeURIComponent(cwd));
+ const sessionDir = join(cwdDir, sessionId);
+ await mkdir(sessionDir, { recursive: true });
+ if (summary !== null) {
+ const body = typeof summary === 'string' ? summary : JSON.stringify(summary);
+ await writeFile(join(sessionDir, 'summary.json'), body);
+ }
+ if (chat !== null) await writeFile(join(sessionDir, 'chat_history.jsonl'), chat);
+ return cwdDir;
+}
+
beforeEach(async () => {
home = await mkdtemp(join(tmpdir(), 'grok-repair-home-'));
dataDir = await mkdtemp(join(tmpdir(), 'grok-repair-data-'));
@@ -78,4 +100,63 @@ describe('applyGrokUsageRepair', () => {
.rejects.toThrow(/usage\.json/);
expect(written).toHaveLength(0);
});
+
+ it('repairs the real session when a cwd folder also holds prompt_history.jsonl', async () => {
+ // Grok writes that file beside the session-id directories, and other files
+ // can sit next to the cwd folders. Treating either name as a directory
+ // threw ENOTDIR on `<file>/summary.json` and aborted Reconcile now.
+ const cwdDir = await writeGrokSession();
+ await writeFile(join(cwdDir, 'prompt_history.jsonl'), '{"prompt":"not a session"}\n');
+ await writeFile(join(home, '.grok', 'sessions', 'not-a-cwd.txt'), 'stray');
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const result = await applyGrokUsageRepair({ home, dataDir });
+ expect(result).toMatchObject({ added: 2, images: 2, turns: 0, videos: 0, backedUp: true });
+ expect(written.map((row) => row.repairId).sort()).toEqual([
+ 'gallery:job-1',
+ 'session-image:session-1:0',
+ ]);
+ expect(warn).not.toHaveBeenCalled();
+ } finally {
+ warn.mockRestore();
+ }
+ });
+});
+
+describe('planGrokUsageRepairFromDisk bad inputs', () => {
+ it('skips an unreadable session file, gallery sidecar, video history, and scan without dropping the rest', async () => {
+ await writeGrokSession({ sessionId: 'session-good' });
+ await writeGrokSession({ sessionId: 'session-bad', summary: '{not-json' });
+ await writeFile(join(dataDir, 'images', 'broken.metadata.json'), '{not-json');
+ await writeFile(join(dataDir, 'video-history.json'), '{not-json');
+ await writeFile(join(dataDir, 'usage-interactive-scan.json'), '{not-json');
+ await writeFile(join(dataDir, 'providers.json'), '{not-json');
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const plan = await planGrokUsageRepairFromDisk({ home, dataDir });
+ expect(plan.additions.map((item) => item.repairId).sort()).toEqual([
+ 'gallery:job-1',
+ 'session-image:session-bad:0',
+ 'session-image:session-good:0',
+ ]);
+ expect(plan.additions.every((item) => item.record.providerId === 'grok')).toBe(true);
+ expect(plan.notes[0]).toMatch(/no Grok usage scan has run yet/);
+ expect(warn).toHaveBeenCalledTimes(5);
+ expect(warn.mock.calls.map((call) => String(call[0]))).toEqual(expect.arrayContaining([
+ expect.stringContaining('summary.json for session session-bad'),
+ expect.stringContaining('broken.metadata.json'),
+ expect.stringContaining('video-history.json'),
+ expect.stringContaining('usage-interactive-scan.json'),
+ expect.stringContaining('providers.json'),
+ ]));
+ for (const [line] of warn.mock.calls) expect(String(line)).not.toContain(home);
+ } finally {
+ warn.mockRestore();
+ }
+ });
+
+ it('does not treat an unreadable usage ledger as an empty repair list', async () => {
+ await writeFile(join(dataDir, 'usage.json'), '{not-json');
+ await expect(planGrokUsageRepairFromDisk({ home, dataDir })).rejects.toThrow(SyntaxError);
+ });
});