Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Apply the Grok usage repair to the server's in-memory ledger.
*
* Reconcile on the Usage page calls this from the historical-backfill job, on
* the same thread that owns usage.json. A separate process writing that file
* would lose to the server's next save. Repair ids already on the ledger make
* a repeat a no-op. The on-disk ledger is copied before the first write of a
* batch that has something to add.
*
* The CLI script is a dry-run and stopped-server wrapper around these two
* functions. Cache-rate corrections are not part of either path: the report
* prices tokens when it is built.
*/
import { copyFile, mkdir, readdir, readFile, stat } from 'fs/promises';
import { homedir } from 'os';
import { join, resolve } from 'path';
import { PATHS } from '../lib/fileUtils.js';
import { familyForProvider } from '../lib/providerFamilies.js';
import { parseGrokTurns } from '../lib/providerTranscriptUsage.js';
import { grokUsageProviderIdFromList } from './grokMediaUsage.js';
import {
galleryImageUsageEvent,
imagineEventsFromGrokSession,
planGrokUsageRepair,
summarizeGrokRepairAdditions,
videoHistoryUsageEvents,
} from './grokUsageRepair.js';
import { addDatedUsage, flushUsageLedger, getUsage, loadUsage } from './usage.js';
const yieldToEventLoop = () => new Promise((done) => setImmediate(done));
// Node's filesystem errors put the absolute path in `message`. Log `code` for
// those so a home directory never lands in the process log. A parse failure
// has no code; its message does not include a path.
const faultLabel = (err) => {
if (err?.code) return String(err.code);
return String(err?.message || err?.name || 'error').replace(/[\r\n]+/g, ' ');
};
const oneLine = (value) => String(value ?? '').replace(/[\r\n]+/g, ' ');
/**
* Read JSON, or `fallback` when the file is missing.
*
* A present-but-unreadable file is also `fallback` unless `strict`. Session
* summaries, gallery sidecars, video history, the scan watermark, and provider
* config are per-entry inputs: one bad file must not abort the repair.
* `usage.json` stays strict so a corrupt ledger is not planned as an empty
* repair-id list.
*/
async function readJson(path, fallback, { strict = false, label = 'file' } = {}) {
try {
return JSON.parse(await readFile(path, 'utf8'));
} catch (err) {
if (err.code === 'ENOENT') return fallback;
if (strict) throw err;
console.warn(`⚠️ Grok usage repair skipped unreadable ${label} (${faultLabel(err)})`);
return fallback;
}
}
/**
* Entry names in `dir`, or [] when `dir` is missing.
*
* `directories` keeps only real directories. Grok writes a flat
* `prompt_history.jsonl` beside the session-id directories inside each cwd
* folder, and other files can sit next to the cwd folders themselves. Opening
* `<file>/summary.json` throws ENOTDIR and aborts the repair — the layout
* `listSubdirs` in usageReconciler.js already filters (#6218). This copy stays
* local so the repair does not import the reconciler's claim ledger and
* transcript readers. Any other directory error is skipped for the same
* reason: one unreadable folder must not abort the repair.
*/
async function listNames(dir, { directories = false } = {}) {
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch (err) {
if (err.code !== 'ENOENT') {
console.warn(`⚠️ Grok usage repair skipped a directory (${faultLabel(err)})`);
}
return [];
}
const wanted = directories ? entries.filter((entry) => entry.isDirectory()) : entries;
return wanted.map((entry) => entry.name);
}
async function fileMtime(path) {
try {
return (await stat(path)).mtimeMs;
} catch {
return null;
}
}
async function collectSessions(home) {
const root = join(home, '.grok', 'sessions');
const turns = [];
const media = [];
for (const dirName of await listNames(root, { directories: true })) {
let cwd = dirName;
try { cwd = decodeURIComponent(dirName); } catch { /* keep the raw directory name */ }
const cwdDir = join(root, dirName);
for (const sessionId of await listNames(cwdDir, { directories: true })) {
try {
const sessionDir = join(cwdDir, sessionId);
const updatesPath = join(sessionDir, 'updates.jsonl');
const updatesText = await readFile(updatesPath, 'utf8').catch(() => '');
if (updatesText) {
const parsed = parseGrokTurns(updatesText);
for (const turn of parsed.longTurns || []) turns.push(turn);
}
const summary = await readJson(join(sessionDir, 'summary.json'), null, {
label: `summary.json for session ${oneLine(sessionId)}`,
});
const chatPath = join(sessionDir, 'chat_history.jsonl');
const stamp = Date.parse(summary?.updated_at || summary?.created_at || '')
|| await fileMtime(chatPath)
|| await fileMtime(updatesPath);
const day = Number.isFinite(stamp) ? new Date(stamp).toISOString().slice(0, 10) : null;
const chatText = await readFile(chatPath, 'utf8').catch(() => '');
media.push(...imagineEventsFromGrokSession({
chatText, updatesText, sessionId, cwd, day,
}));
} catch (err) {
// One bad session must not drop the gallery, video history, or the
// sessions already read.
console.warn(`⚠️ Grok usage repair skipped session ${oneLine(sessionId)} (${faultLabel(err)})`);
}
await yieldToEventLoop();
}
}
return { turns, media };
}
async function collectGallery(dataDir) {
const dir = join(dataDir, 'images');
const media = [];
for (const name of await listNames(dir)) {
if (!name.endsWith('.metadata.json')) continue;
try {
const meta = await readJson(join(dir, name), null, { label: oneLine(name) });
const event = galleryImageUsageEvent(meta);
if (event) media.push(event);
} catch (err) {
console.warn(`⚠️ Grok usage repair skipped gallery file ${oneLine(name)} (${faultLabel(err)})`);
}
}
return media;
}
async function collectVideos(dataDir) {
const history = await readJson(join(dataDir, 'video-history.json'), [], { label: 'video-history.json' });
const media = [];
for (const entry of Array.isArray(history) ? history : []) {
try {
media.push(...videoHistoryUsageEvents(entry));
} catch (err) {
console.warn(`⚠️ Grok usage repair skipped a video history entry (${faultLabel(err)})`);
}
}
return media;
}
async function providerIdFromFile(dataDir) {
const stored = await readJson(join(dataDir, 'providers.json'), null, { label: 'providers.json' });
const list = Array.isArray(stored)
? stored
: (Array.isArray(stored?.providers) ? stored.providers : Object.values(stored?.providers || {}));
const grok = list.filter((provider) => provider && familyForProvider(provider) === 'grok');
return grokUsageProviderIdFromList(grok.length ? list : []);
}
/**
* Read sessions, the gallery, and video history without touching the ledger.
* `repairIds` default to the on-disk list so a dry run does not load the
* server's in-memory copy.
*/
export async function planGrokUsageRepairFromDisk({
home = homedir(),
dataDir = PATHS.data,
repairIds = null,
providerId = null,
} = {}) {
const scan = await readJson(join(dataDir, 'usage-interactive-scan.json'), { watermarks: {} }, {
label: 'usage-interactive-scan.json',
});
const watermarkMs = typeof scan?.watermarks?.grok === 'number' ? scan.watermarks.grok : null;
const storedIds = repairIds ?? (await readJson(join(dataDir, 'usage.json'), {}, { strict: true }))?.grokRepairIds;
const sessions = await collectSessions(home);
return planGrokUsageRepair({
turns: sessions.turns,
media: [
...sessions.media,
...await collectGallery(dataDir),
...await collectVideos(dataDir),
],
repairIds: Array.isArray(storedIds) ? storedIds : [],
watermarkMs,
providerId: providerId || await providerIdFromFile(dataDir),
});
}
async function backupLedger(dataDir) {
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
const dest = join(dataDir, 'backups', `grok-usage-${stamp}`);
await mkdir(dest, { recursive: true });
for (const name of ['usage.json', 'usage-interactive-scan.json']) {
const source = join(dataDir, name);
try {
await copyFile(source, join(dest, name));
} catch (err) {
if (err.code !== 'ENOENT') throw err;
}
}
return dest;
}
/**
* Plan against the in-memory repair ids and add any missing rows there.
* Writes only PATHS.data, which is the file the running server will save.
* @returns {Promise<{ added: number, turns: number, images: number, videos: number,
* backedUp: boolean, note: string|null, backupDir: string|null }>}
*/
export async function applyGrokUsageRepair({
home = homedir(),
dataDir = PATHS.data,
providers = null,
providerId = null,
} = {}) {
if (resolve(dataDir) !== resolve(PATHS.data)) {
throw new Error(`Grok usage repair writes ${PATHS.data}/usage.json`);
}
await loadUsage();
const resolvedProviderId = providerId
|| (providers ? grokUsageProviderIdFromList(providers) : await providerIdFromFile(dataDir));
const plan = await planGrokUsageRepairFromDisk({
home,
dataDir,
repairIds: getUsage().grokRepairIds || [],
providerId: resolvedProviderId,
});
const note = plan.notes[0] || null;
if (plan.additions.length === 0) {
return { ...summarizeGrokRepairAdditions([]), backedUp: false, note, backupDir: null };
}
const backupDir = await backupLedger(dataDir);
const written = [];
for (const item of plan.additions) {
const wrote = await addDatedUsage(item.day, item.record, { repairId: item.repairId, flush: false });
if (wrote) written.push(item);
}
if (written.length > 0) await flushUsageLedger();
return { ...summarizeGrokRepairAdditions(written), backedUp: true, note, backupDir };
}