Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix(usage): account for Codex image render sessions
[._.]/ Adam Eivy118690d7c2d1
diff --git a/client/src/pages/UsagePage.jsx b/client/src/pages/UsagePage.jsxindex 4fdbabcfe4..d297868d51 100644--- a/client/src/pages/UsagePage.jsx+++ b/client/src/pages/UsagePage.jsx@@ -23,6 +23,7 @@ import { USAGE_PERIOD_OPTIONS, DEFAULT_USAGE_PERIOD } from '../lib/usagePeriods'const BACKFILL_EVENTS = ['usage-backfill:updated'];+const IMAGE_USAGE_EVENTS = ['image-gen:completed'];// A subscription is one account across every federated instance, but each// instance can only read its own CLI's panel. When peers have contributed a@@ -597,6 +598,8 @@ function InternalUsageMetrics() {return () => { requestRef.current += 1; };}, [fetchUsage]);+ useSocketResource(fetchUsage, { events: IMAGE_USAGE_EVENTS, immediate: false });+const [startBackfill, startingBackfill] = useAsyncAction(async () => {const status = await api.startUsageBackfill({ silent: true });setBackfill(status);@@ -702,6 +705,30 @@ function InternalUsageMetrics() {</button></div>+ {usage.imageRenders && (+ <section className="bg-port-card border border-port-border rounded-xl p-3 sm:p-4 space-y-2" aria-label="Codex image usage">+ <h3 className="text-sm font-medium text-white">Codex image usage · this machine</h3>+ <p className="text-sm text-gray-300">+ {formatCount(usage.imageRenders.renderSessions)} render sessions · {formatCount(usage.imageRenders.savedOutputs)} saved outputs in this period+ </p>+ <p className="text-xs text-gray-400">+ Image tokens, image cost and subscription allowance attribution: unknown.+ Any driver tokens already captured by text accounting stay in the text report. Image costs are excluded from the API estimate.+ Saved outputs show observed activity; they do not measure every image-tool call or failed attempt.+ Historical saved records do not establish which machine executed a render.+ </p>+ <p className="text-xs text-gray-400">+ {usage.imageRenders.driverTokens+ ? `${formatCount(usage.imageRenders.driverTokens.reportedTotal)} reported driver tokens across ${formatCount(usage.imageRenders.driverTokens.sessionsMeasured)} sessions; no input/output split or image attribution.`+ : 'Driver tokens: unknown.'}+ {usage.imageRenders.driverTokens && ` Driver tokens are unknown for ${formatCount(usage.imageRenders.driverSessionsUnknown)} other sessions.`}+ </p>+ {usage.imageRenders.jobIdentitySessions > 0 && (+ <p className="text-xs text-gray-400">{formatCount(usage.imageRenders.jobIdentitySessions)} sessions use a saved job identity because the provider session ID is unavailable.</p>+ )}+ </section>+ )}+{/* Summary Stats (all-time) */}<div className="grid grid-cols-1 sm:grid-cols-3 gap-3 sm:gap-4"><div className="bg-port-card border border-port-border rounded-xl p-3 sm:p-4 text-center">diff --git a/client/src/pages/UsagePage.test.jsx b/client/src/pages/UsagePage.test.jsxindex 79843a389f..923959477e 100644--- a/client/src/pages/UsagePage.test.jsx+++ b/client/src/pages/UsagePage.test.jsx@@ -657,3 +657,19 @@ describe('UsagePage report range freshness (#10687)', () => {expect(screen.queryByRole('alert')).not.toBeInTheDocument();});});+++describe('UsagePage image accounting', () => {+ it('shows saved render activity separately from unknown image cost and allowance', async () => {+ api.getUsage.mockResolvedValue({ ...usage, imageRenders: { renderSessions: 8, savedOutputs: 10, jobIdentitySessions: 2, imageCostUsd: null } });+ render(<MemoryRouter><UsagePage /></MemoryRouter>);+ const card = await screen.findByRole('region', { name: 'Codex image usage' });+ expect(within(card).getByText(/8 render sessions · 10 saved outputs/)).toBeInTheDocument();+ expect(within(card).getByText(/Image tokens, image cost and subscription allowance attribution: unknown/)).toBeInTheDocument();+ expect(within(card).queryByText(/\$0/)).not.toBeInTheDocument();+ expect(within(card).getByText(/2 sessions use a saved job identity/)).toBeInTheDocument();+ const reads = api.getUsage.mock.calls.length;+ await act(async () => { socket.emit('image-gen:completed', { mode: 'codex' }); });+ await waitFor(() => expect(api.getUsage.mock.calls.length).toBe(reads + 1));+ });+});diff --git a/docs/STORAGE.md b/docs/STORAGE.mdindex cbed33c48b..f90e123561 100644--- a/docs/STORAGE.md+++ b/docs/STORAGE.md@@ -1233,3 +1233,58 @@ recovery evidence prevents an interrupted journal replay from becoming freshretirement evidence after the display row has aged out. It has no TTL purge.FableLoom shot-planning run checkpoints live in machine-local `data/fableloom-shot-runs/` JSON journals (atomic writes, one latest run per story/episode). They preserve drafts, review verdicts and feedback across restarts; they do not federate. Reading an interrupted run marks it failed without starting a provider call. Explicit resume uses the saved provider/guidance and rejects changed source revisions. Story records remain PostgreSQL-primary.++### Codex image render accounting++`data/image-render-usage/<sha256>.json` is a machine-local `file-primary`+accounting receipt ledger (`server/services/imageRenderUsage.js`). Each immutable+receipt describes one saved original output; `renderKey` groups outputs from the+same Codex session. The Usage page counts sessions separately from saved outputs.+A missing provider session ID uses the original PortOS job identity and is+reported as such. These are observed saved render sessions, not a count of every+image-tool invocation, attempt or billable image.++This is an accounting log alongside `usage.json`, with no relational queries,+record editing or cross-domain joins. Receipts publish complete files with an+exclusive hard link, so a live completion and an explicit backfill cannot replace+one another or double-count an output across processes. Hidden temporary files+are ignored after an interrupted write. A damaged receipt fails the read rather+than reporting a trustworthy zero. Rsync backs up this directory by default; it+is not a peer-usage digest or a media-library asset and does not federate.++Receipts contain hashed identities, the original saved timestamp, driver model+and accounting evidence only. They contain no prompts, image bytes, filenames or+session contents. Codex's human-mode `tokens used` footer, when present, is stored+as a reported driver total without an input/output breakdown. The summary counts+it once per session and states measurement coverage. It is never added to the+text-token ledger, which may already include the session. Image tokens, image+cost and subscription allowance remain `null` (unknown); text API prices and+average allowance comparisons are not image conversion rates. The API cost+estimate excludes unknown image cost.++Historical import is explicit and never runs at boot. From the repository root:++```sh+node scripts/backfill-codex-image-usage.js --data-dir <install>/data --report <private-plan.json>+node scripts/backfill-codex-image-usage.js --data-dir <install>/data --apply <private-plan.json> --journal <private-journal.json>+```++Review the dry-run coverage before applying. The importer reads only PortOS+image sidecars and completed queue metadata, excludes known derived copies,+groups multiple outputs per session, and uses saved original timestamps rather+than filesystem mtimes. It never reads provider sessions. Unattributed sidecars,+unsaved/failed calls, unavailable token/cost/allowance data and historical+execution-machine provenance remain unknown. A changed inventory requires a new+dry run. The plan and journal must use new paths; existing files are never+replaced. Neither operation mutates `usage.json`, sidecars or queue history.++Before writing receipts, apply saves a journal with a unique batch ID. Every+inserted receipt carries that `backfillBatch`; pre-existing/live receipts retain+their original bytes. An interrupted batch can be rerun with a new journal path+and the same plan when the source inventory is unchanged. For recovery, retain+the journal and move only receipts whose `backfillBatch` matches that journal to+a private archive outside the ledger, verifying preservation before removal.+This also identifies additions made before a crash prevented the final journal+update. Never restore an old `usage.json` over newer usage to undo this import.+No migration or seed is required: this is a new optional ledger, absent until a+live completion or an explicit import publishes its first receipt.diff --git a/scripts/backfill-codex-image-usage.js b/scripts/backfill-codex-image-usage.jsnew file mode 100644index 0000000000..80a523cdf7--- /dev/null+++ b/scripts/backfill-codex-image-usage.js@@ -0,0 +1,32 @@+#!/usr/bin/env node+import { readFile } from 'node:fs/promises';+import { resolve } from 'node:path';+import { pathToFileURL } from 'node:url';+import { publishAccountingFile } from '../server/services/imageRenderUsage.js';+import { planImageRenderBackfill, applyImageRenderBackfill } from '../server/services/imageRenderBackfill.js';++export async function main(args) {+ const allowed = new Set(['--data-dir', '--report', '--apply', '--journal']);+ const options = {};+ for (let i = 0; i < args.length; i += 2) {+ if (!allowed.has(args[i]) || !args[i + 1] || args[i + 1].startsWith('--') || options[args[i]]) throw new Error('Usage: --data-dir <data> --report <plan.json> OR --data-dir <data> --apply <plan.json> --journal <journal.json>');+ options[args[i]] = resolve(args[i + 1]);+ }+ if (!options['--data-dir']) throw new Error('An explicit --data-dir is required');+ if (options['--apply']) {+ if (!options['--journal'] || options['--report']) throw new Error('--apply requires --journal and cannot use --report');+ const reviewedPlan = JSON.parse(await readFile(options['--apply'], 'utf8'));+ return applyImageRenderBackfill(options['--data-dir'], reviewedPlan, options['--journal']);+ }+ if (!options['--report'] || options['--journal']) throw new Error('Dry run requires --report');+ const plan = await planImageRenderBackfill(options['--data-dir']);+ if (!await publishAccountingFile(options['--report'], plan)) throw new Error('Report already exists; choose a new report path');+ return { dryRun: true, inventoryHash: plan.inventoryHash, coverage: plan.coverage, unknown: plan.unknown };+}++if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {+ main(process.argv.slice(2)).then(result => console.log(JSON.stringify(result, null, 2))).catch(err => {+ console.error(`❌ Codex image usage backfill failed: ${err.message}`);+ process.exitCode = 1;+ });+}diff --git a/server/lib/fileCore.js b/server/lib/fileCore.jsindex 1138f62b57..f040eee879 100644--- a/server/lib/fileCore.js+++ b/server/lib/fileCore.js@@ -56,7 +56,7 @@ const unsyncableDirs = new Set();* guarantee with one warning rather than failing a write that succeeded.* Windows cannot open a directory for fsync, so it is skipped there.*/-async function syncDirectory(dir) {+export async function syncDirectory(dir) {if (isWindows()) return;try {const handle = await open(dir, 'r');diff --git a/server/routes/usage.js b/server/routes/usage.jsindex a2d08c4d59..7d248cdaf8 100644--- a/server/routes/usage.js+++ b/server/routes/usage.js@@ -1,5 +1,6 @@import { Router } from 'express';import * as usage from '../services/usage.js';+import { getImageRenderUsage } from '../services/imageRenderUsage.js';import { getClaudeCodeUsage } from '../services/claudeCodeUsage.js';import { refreshLocalTranscriptUsage } from '../services/claudeCodeTranscriptUsage.js';import { getProviderQuotas } from '../services/providerUsage.js';@@ -41,7 +42,7 @@ router.get('/', asyncHandler(async (req, res) => {// (or before a peer's first usage sync) — the UI hides that section rather// than showing a one-row "fleet".// - monthly: fixed calendar months with the rolling chart's fleet coverage.- const [subscriptionSavings, fleet, monthly] = await Promise.all([+ const [subscriptionSavings, fleet, monthly, imageRenders] = await Promise.all([getSubscriptionSavings({report: summary.report,providers,@@ -54,9 +55,10 @@ router.get('/', asyncHandler(async (req, res) => {getApiBilledInstanceIds().then((apiBilledInstanceIds) =>getFleetUsage({ from, to, providers, apiBilledInstanceIds })),getFleetMonthlyUsage({ providers }),+ getImageRenderUsage({ from, to }),]);res.json({- ...summary, subscriptionSavings, fleet,+ ...summary, subscriptionSavings, fleet, imageRenders,monthlyHistory: monthly.history,monthlySeries: monthly.series,monthlyInstanceCount: monthly.instanceCount,diff --git a/server/routes/usage.test.js b/server/routes/usage.test.jsindex 1c7bf2ac7e..327622db43 100644--- a/server/routes/usage.test.js+++ b/server/routes/usage.test.js@@ -3,6 +3,8 @@ import express from 'express';import { request } from '../lib/testHelper.js';import { errorMiddleware } from '../lib/errorHandler.js';+vi.mock('../services/imageRenderUsage.js', () => ({ getImageRenderUsage: vi.fn(async () => ({ renderSessions: 2, savedOutputs: 3, imageCostUsd: null })) }));+vi.mock('../services/usage.js', () => ({getUsageSummary: vi.fn(),getUsage: vi.fn(),@@ -55,6 +57,7 @@ vi.mock('../services/usageBackfill.js', () => ({}));import * as usage from '../services/usage.js';+import { getImageRenderUsage } from '../services/imageRenderUsage.js';import { getAllProviders } from '../services/providers.js';import { getProviderQuotas } from '../services/providerUsage.js';import { getHistoricalUsageBackfillStatus, startHistoricalUsageBackfill } from '../services/usageBackfill.js';@@ -77,6 +80,15 @@ describe('usage routes', () => {vi.clearAllMocks();});+ it('GET /api/usage includes local image activity in the selected range without changing the text cost report', async () => {+ usage.getUsageSummary.mockReturnValue({ report: { totals: { estimatedCost: 12 } } });+ const response = await request(buildApp()).get('/api/usage?from=2026-07-01&to=2026-07-31');+ expect(response.status).toBe(200);+ expect(getImageRenderUsage).toHaveBeenCalledWith({ from: '2026-07-01', to: '2026-07-31' });+ expect(response.body.imageRenders).toEqual({ renderSessions: 2, savedOutputs: 3, imageCostUsd: null });+ expect(response.body.report.totals.estimatedCost).toBe(12);+ });+it('GET /api/usage/hourly projects only counters and bypasses all reporting work', async () => {const hourlyActivity = Array.from({ length: 24 }, (_, hour) => hour);// Synthetic extra records must never escape in the hourly payload. Reporting@@ -134,6 +146,7 @@ describe('usage routes', () => {providers: ['anthropic'],subscriptionSavings: { configured: false, families: [] },fleet: { instances: [], totals: null },+ imageRenders: { renderSessions: 2, savedOutputs: 3, imageCostUsd: null },monthlyHistory: [],monthlySeries: [],monthlyInstanceCount: 1,diff --git a/server/services/imageGen/codex.js b/server/services/imageGen/codex.jsindex cf8e52c385..c3267c1627 100644--- a/server/services/imageGen/codex.js+++ b/server/services/imageGen/codex.js@@ -48,6 +48,7 @@ import { autoCleanGeneratedImage } from '../../lib/imageClean.js';import { imageGenEvents } from '../imageGenEvents.js';import { broadcastSse, attachSseClient as attachSse, closeJobAfterDelay, createJobFailureFinalizer, dispatchTerminalEvent } from '../../lib/sseUtils.js';import { killWithEscalation } from '../../lib/killWithEscalation.js';+import { codexDriverReportedTokens, recordCodexRender } from '../imageRenderUsage.js';import { renderTimingFields } from '../../lib/renderTiming.js';import { buildCodexStartupArgs, buildEffortArgs, resolveCliEffort } from '../../lib/providerModels.js';import {@@ -532,12 +533,17 @@ function runCodex(job, jobId, bin, args, outputPath, filename, meta, { cleanC2PA// `job.renderStartedAtMs` is the queue-ingestion instant generateImage// captured, so the spread measures the render itself — not the time the// job spent queued behind other renders.- await atomicWrite(sidecar, { ...meta, codexSessionId: sessionId, ...(aspectConformed ? { aspectConformed } : {}), ...renderTimingFields(job.renderStartedAtMs) });+ const renderMeta = { ...meta, codexSessionId: sessionId, driverReportedTokens: codexDriverReportedTokens(stderrTail), ...(aspectConformed ? { aspectConformed } : {}), ...renderTimingFields(job.renderStartedAtMs) };+ await atomicWrite(sidecar, renderMeta);// Cleaners run BEFORE the SSE complete + completed events so subscribers// see the cleaned bytes. codex output is the highest-value target for// C2PA stripping because gpt-image is the one provider that embeds// provenance metadata.await autoCleanGeneratedImage({ cleanC2PA, denoise, pngPath: outputPath, sidecarPath: sidecar, mode: IMAGE_GEN_MODE.CODEX });+ // Accounting failure must preserve the image and its recoverable sidecar.+ await recordCodexRender(renderMeta).catch(err => {+ console.error(`❌ Failed to record Codex image usage: ${err.message}`);+ });job.status = 'complete';if (activeProcs.get(jobId) === proc) activeProcs.delete(jobId);activeJobs.delete(jobId);diff --git a/server/services/imageGen/codex.test.js b/server/services/imageGen/codex.test.jsindex 9a41bdac8a..ebab0f2b6f 100644--- a/server/services/imageGen/codex.test.js+++ b/server/services/imageGen/codex.test.js@@ -80,6 +80,7 @@ vi.mock('../../lib/fileUtils.js', async () => {// a spread-only override would leave the original PATHS.images intact// for any function closing over the actual module's binding.actual.PATHS.images = FAKE_IMAGES_DIR;+ actual.PATHS.data = join(TEST_HOME, 'data');// NOTE: do not invent PATHS keys here. An earlier revision set `PATHS.tmp`,// a key the real object has never had, so the suite exercised a property// that exists only under test while every production render threw@@ -454,7 +455,7 @@ describe('codex provider — image harvest', () => {const child = spawnCalls[0].child;// Drive the session id banner on stderr (codex prints it there).- child.stderr.emit('data', Buffer.from(`session id: ${sessionId}\n`));+ child.stderr.emit('data', Buffer.from(`session id: ${sessionId}\ntokens used\n1,234\n`));// Then close cleanly.child.exitCode = 0;const releaseCut = await acquireBackupSnapshotCut();@@ -477,6 +478,9 @@ describe('codex provider — image harvest', () => {const written = await readFile(finalPath);expect(Buffer.compare(written, fakePngBytes)).toBe(0);+ const { getImageRenderUsage } = await import('../imageRenderUsage.js');+ expect(await getImageRenderUsage()).toMatchObject({ renderSessions: 1, savedOutputs: 1, imageCostUsd: null, driverTokens: { reportedTotal: 1234, sessionsMeasured: 1 } });+// Sidecar metadata exists too.const sidecar = join(FAKE_IMAGES_DIR, `${job.generationId}.metadata.json`);expect(existsSync(sidecar)).toBe(true);@@ -486,6 +490,29 @@ describe('codex provider — image harvest', () => {});});+ it('preserves the completed image and sidecar when receipt persistence fails', async () => {+ const sessionId = '44444444-4444-4444-8444-444444444444';+ const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);+ await mkdir(codexDir, { recursive: true });+ await writeFile(join(codexDir, 'exec-output.png'), Buffer.from('fakepngbytes'));+ await mkdir(PATHS.data, { recursive: true });+ await writeFile(join(PATHS.data, 'image-render-usage'), 'blocked-directory');+ const logged = vi.spyOn(console, 'error').mockImplementation(() => {});+ try {+ const completed = vi.fn();+ imageGenEvents.on('completed', completed);+ const job = await codex.generateImage({ prompt: 'a fox' });+ const child = spawnCalls[0].child;+ child.stderr.emit('data', Buffer.from(`session id: ${sessionId}\n`));+ child.exitCode = 0;+ child.emit('close', 0, null);+ await vi.waitFor(() => expect(completed).toHaveBeenCalledTimes(1));+ expect(existsSync(join(FAKE_IMAGES_DIR, job.filename))).toBe(true);+ expect(JSON.parse(await readFile(join(FAKE_IMAGES_DIR, `${job.generationId}.metadata.json`), 'utf8'))).toMatchObject({ codexSessionId: sessionId });+ expect(logged).toHaveBeenCalledWith(expect.stringContaining('Failed to record Codex image usage'));+ } finally { logged.mockRestore(); }+ });+it('crops an off-aspect frame to the requested ratio and records it in the sidecar', async () => {const sessionId = '019dd59e-a8da-7bd2-a7d8-4ac6f46e7b99';const codexDir = join(TEST_HOME, '.codex', 'generated_images', sessionId);diff --git a/server/services/imageRenderBackfill.js b/server/services/imageRenderBackfill.jsnew file mode 100644index 0000000000..e711c591c4--- /dev/null+++ b/server/services/imageRenderBackfill.js@@ -0,0 +1,85 @@+import { createHash, randomUUID } from 'node:crypto';+import { readFile, readdir } from 'node:fs/promises';+import { join } from 'node:path';+import { atomicWrite } from '../lib/fileCore.js';+import { codexRenderReceipt, readRenderReceipts, recordRenderReceipt, publishAccountingFile } from './imageRenderUsage.js';++const hash = value => createHash('sha256').update(JSON.stringify(value)).digest('hex');++// Reads only PortOS's authoritative sidecars and completed queue metadata.+// Never consults provider session directories, filesystem mtimes, or prompts.+export async function planImageRenderBackfill(dataDir) {+ const directory = join(dataDir, 'image-render-usage');+ const existing = new Map((await readRenderReceipts({ directory })).map(row => [row.id, row]));+ const candidates = new Map();+ const coverage = { sidecars: 0, codexSidecars: 0, unattributedSidecars: 0, derived: 0, unrecoverable: 0, unreadable: 0, queueOnly: 0, duplicates: 0 };+ const representedFiles = new Set();+ const add = meta => {+ const receipt = codexRenderReceipt(meta);+ if (!receipt) { coverage.unrecoverable++; return; }+ if (candidates.has(receipt.id)) { coverage.duplicates++; return; }+ candidates.set(receipt.id, receipt);+ };+ const entries = await readdir(join(dataDir, 'images'), { withFileTypes: true });+ for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {+ if (!entry.isFile() || !entry.name.endsWith('.metadata.json')) continue;+ coverage.sidecars++;+ let meta;+ try { meta = JSON.parse(await readFile(join(dataDir, 'images', entry.name), 'utf8')); }+ catch (err) {+ // Permission failures stop this inventory; they are never an empty source.+ if (!(err instanceof SyntaxError)) throw err;+ coverage.unreadable++;+ continue;+ }+ representedFiles.add(entry.name.replace(/\.metadata\.json$/, '.png'));+ if (meta?.filename) representedFiles.add(meta.filename);+ if (!meta?.mode) coverage.unattributedSidecars++;+ if (meta?.mode !== 'codex') continue;+ coverage.codexSidecars++;+ if (meta.cleanedFrom || meta.filmLookFrom || meta.sourceFilename) { coverage.derived++; continue; }+ add(meta);+ }+ const queue = await readFile(join(dataDir, 'media-jobs.json'), 'utf8').then(JSON.parse).catch(err => {+ if (err.code === 'ENOENT') return { jobs: [] };+ throw err;+ });+ if (!Array.isArray(queue.jobs)) throw new Error('Invalid media queue history');+ for (const job of queue.jobs) {+ if (job.kind !== 'image' || job.status !== 'completed' || job.params?.mode !== 'codex' || !job.result?.filename || representedFiles.has(job.result.filename)) continue;+ coverage.queueOnly++;+ add({ mode: 'codex', id: job.result.generationId, model: job.params.model, createdAt: job.startedAt || job.completedAt });+ }+ const receipts = [...candidates.values()].sort((a, b) => a.id.localeCompare(b.id));+ const pending = receipts.filter(row => !existing.has(row.id));+ return {+ version: 1, inventoryHash: hash(receipts), receipts,+ coverage: {+ ...coverage, eligibleOutputs: receipts.length,+ renderSessions: new Set(receipts.map(row => row.renderKey)).size,+ jobIdentityOutputs: receipts.filter(row => row.identitySource === 'portos-job').length,+ alreadyRecorded: receipts.length - pending.length, additiveOutputs: pending.length,+ earliest: receipts.map(row => row.occurredAt).sort()[0] || null,+ latest: receipts.map(row => row.occurredAt).sort().at(-1) || null,+ },+ unknown: ['image tokens', 'image cost', 'subscription allowance units', 'driver tokens', 'unsaved/failed image-tool calls', 'execution machine for historical sidecars without local queue evidence'],+ };+}++export async function applyImageRenderBackfill(dataDir, reviewedPlan, journalPath) {+ const current = await planImageRenderBackfill(dataDir);+ if (reviewedPlan.version !== 1 || current.inventoryHash !== reviewedPlan.inventoryHash || hash(reviewedPlan.receipts) !== current.inventoryHash) {+ throw new Error('Image inventory changed; create and review a new dry-run plan');+ }+ const batch = randomUUID();+ // Written BEFORE receipts: after interruption the batch marker identifies+ // precisely our additions, even if the final journal update did not land.+ const journal = { version: 1, batch, inventoryHash: current.inventoryHash, status: 'applying', plannedIds: current.receipts.map(row => row.id), inserted: 0 };+ if (!await publishAccountingFile(journalPath, journal)) throw new Error('Journal already exists; choose a new journal path');+ for (const receipt of current.receipts) {+ if (await recordRenderReceipt({ ...receipt, backfillBatch: batch }, { directory: join(dataDir, 'image-render-usage') })) journal.inserted++;+ }+ journal.status = 'complete';+ await atomicWrite(journalPath, journal);+ return { ...current.coverage, inserted: journal.inserted, batch };+}diff --git a/server/services/imageRenderBackfill.test.js b/server/services/imageRenderBackfill.test.jsnew file mode 100644index 0000000000..1fc06fd811--- /dev/null+++ b/server/services/imageRenderBackfill.test.js@@ -0,0 +1,87 @@+import { afterEach, beforeEach, describe, expect, it } from 'vitest';+import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { planImageRenderBackfill, applyImageRenderBackfill } from './imageRenderBackfill.js';+import { codexRenderReceipt, getImageRenderUsage, readRenderReceipts, recordRenderReceipt, codexDriverReportedTokens } from './imageRenderUsage.js';++let root, directory;+const session = '11111111-1111-4111-8111-111111111111';+const id = '22222222-2222-4222-8222-222222222222';+const otherId = '33333333-3333-4333-8333-333333333333';+const meta = (extra = {}) => ({ mode: 'codex', id, codexSessionId: session, model: 'example-driver', createdAt: '2026-07-01T12:00:00Z', prompt: 'Private example prompt', ...extra });+const sidecar = (name, row) => writeFile(join(root, 'images', `${name}.metadata.json`), JSON.stringify(row));++beforeEach(async () => {+ root = await mkdtemp(join(tmpdir(), 'image-usage-'));+ directory = join(root, 'image-render-usage');+ await mkdir(join(root, 'images'));+ await writeFile(join(root, 'usage.json'), '{"existing":"preserve exactly"}');+});+afterEach(async () => { await rm(root, { recursive: true, force: true }); });++describe('saved Codex image usage recovery', () => {+ it('deduplicates repeated outputs and derived copies, groups sessions, and preserves token history across repeated applies', async () => {+ await sidecar('original', meta());+ await sidecar('duplicate', meta());+ await sidecar('second-output', meta({ id: otherId }));+ await sidecar('derived', meta({ cleanedFrom: 'original.png' }));+ await sidecar('import', meta({ sourceFilename: 'original.png', codexSessionId: null }));+ await sidecar('missing-date', meta({ createdAt: null }));+ await sidecar('other-provider', { mode: 'agy' });+ await writeFile(join(root, 'media-jobs.json'), JSON.stringify({ jobs: [{ kind: 'image', status: 'completed', params: { mode: 'codex' }, result: { filename: 'original.png', generationId: id }, startedAt: '2026-07-01T12:00:00Z' }] }));+ const plan = await planImageRenderBackfill(root);+ expect(plan.coverage).toMatchObject({ eligibleOutputs: 2, renderSessions: 1, duplicates: 1, derived: 2, unrecoverable: 1, queueOnly: 0 });+ expect(JSON.stringify(plan)).not.toContain('Private example prompt');+ expect(await readdir(root)).not.toContain('image-render-usage');+ const applied = await applyImageRenderBackfill(root, plan, join(root, 'journal.json'));+ expect(applied.inserted).toBe(2);+ expect((await applyImageRenderBackfill(root, plan, join(root, 'second-journal.json'))).inserted).toBe(0);+ expect(await getImageRenderUsage({ directory, from: '2026-07-01', to: '2026-07-01' })).toMatchObject({ renderSessions: 1, savedOutputs: 2, imageTokens: null, imageCostUsd: null, allowanceUnits: null, driverTokens: null });+ expect(await getImageRenderUsage({ directory, from: '2026-07-02' })).toMatchObject({ renderSessions: 0, savedOutputs: 0 });+ expect(await readFile(join(root, 'usage.json'), 'utf8')).toBe('{"existing":"preserve exactly"}');+ const journal = JSON.parse(await readFile(join(root, 'journal.json'), 'utf8'));+ expect((await readRenderReceipts({ directory })).every(row => row.backfillBatch === journal.batch)).toBe(true);+ });++ it('counts a session spanning midnight only in its original period and refuses changed dry-run inputs', async () => {+ await sidecar('first', meta({ createdAt: '2026-07-01T23:59:59Z' }));+ const oldPlan = await planImageRenderBackfill(root);+ await sidecar('next', meta({ id: otherId, createdAt: '2026-07-02T00:00:01Z' }));+ await expect(applyImageRenderBackfill(root, oldPlan, join(root, 'journal.json'))).rejects.toThrow('inventory changed');+ const plan = await planImageRenderBackfill(root);+ await applyImageRenderBackfill(root, plan, join(root, 'journal.json'));+ expect(await getImageRenderUsage({ directory, from: '2026-07-01', to: '2026-07-01' })).toMatchObject({ renderSessions: 1, savedOutputs: 2 });+ expect(await getImageRenderUsage({ directory, from: '2026-07-02' })).toMatchObject({ renderSessions: 0, savedOutputs: 0 });+ });++ it('publishes complete receipts exactly once across simultaneous writers and does not replace a live receipt during backfill', async () => {+ const receipt = codexRenderReceipt(meta({ driverReportedTokens: codexDriverReportedTokens('result\ntokens used\n1,234\n') }));+ expect((await Promise.all(Array.from({ length: 8 }, () => recordRenderReceipt(receipt, { directory })))).filter(Boolean)).toHaveLength(1);+ await sidecar('original', meta());+ const plan = await planImageRenderBackfill(root);+ expect(plan.coverage.alreadyRecorded).toBe(1);+ expect((await applyImageRenderBackfill(root, plan, join(root, 'journal.json'))).inserted).toBe(0);+ expect(await readRenderReceipts({ directory })).toEqual([receipt]);+ expect(await getImageRenderUsage({ directory })).toMatchObject({ driverTokens: { reportedTotal: 1234, sessionsMeasured: 1 }, driverSessionsUnknown: 0, imageCostUsd: null });+ expect(codexDriverReportedTokens('no footer')).toBeNull();+ });++ it('uses completed queue evidence for deleted sidecars and keeps missing identities or timestamps unrecoverable', async () => {+ await writeFile(join(root, 'media-jobs.json'), JSON.stringify({ jobs: [+ { kind: 'image', status: 'completed', params: { mode: 'codex' }, result: { filename: 'deleted.png', generationId: id }, startedAt: '2026-06-01T01:02:03Z' },+ { kind: 'image', status: 'failed', params: { mode: 'codex' }, result: { filename: 'failed.png', generationId: otherId } },+ ] }));+ await sidecar('no-identity', meta({ id: null, codexSessionId: null }));+ const plan = await planImageRenderBackfill(root);+ expect(plan.coverage).toMatchObject({ eligibleOutputs: 1, queueOnly: 1, unrecoverable: 1, jobIdentityOutputs: 1, earliest: '2026-06-01T01:02:03.000Z' });+ });++ it('fails closed on a damaged ledger and reports malformed sidecars without inventing usage', async () => {+ await writeFile(join(root, 'images', 'bad.metadata.json'), '{');+ expect((await planImageRenderBackfill(root)).coverage.unreadable).toBe(1);+ await recordRenderReceipt(codexRenderReceipt(meta()), { directory });+ await writeFile(join(directory, `${codexRenderReceipt(meta()).id}.json`), '{');+ await expect(planImageRenderBackfill(root)).rejects.toThrow();+ });+});diff --git a/server/services/imageRenderUsage.js b/server/services/imageRenderUsage.jsnew file mode 100644index 0000000000..df5ed1d361--- /dev/null+++ b/server/services/imageRenderUsage.js@@ -0,0 +1,132 @@+import { createHash, randomUUID } from 'node:crypto';+import { link, readFile, readdir } from 'node:fs/promises';+import { dirname, join } from 'node:path';+import { atomicWrite, ensureDir, unlinkGuarded, syncDirectory } from '../lib/fileCore.js';+import { PATHS } from '../lib/paths.js';++const ledgerDir = () => join(PATHS.data, 'image-render-usage');+const uuid = /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i;+const timestamp = value => typeof value === 'string' && Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null;+const digest = value => createHash('sha256').update(value).digest('hex');++// Project only accounting evidence. Never persist prompts, filenames, paths or+// session contents. A driver session may produce several image-tool outputs.+export function codexRenderReceipt(meta) {+ if (meta?.mode !== 'codex' || meta.cleanedFrom || meta.filmLookFrom || meta.sourceFilename) return null;+ const session = uuid.test(meta.codexSessionId) ? meta.codexSessionId : null;+ const job = [meta.id, meta.jobId].find(value => typeof value === 'string' && uuid.test(value));+ if (!session && !job) return null;+ const occurredAt = timestamp(meta.renderStartedAt) || timestamp(meta.createdAt);+ if (!occurredAt) return null;+ const renderKey = digest(`codex:${session || job}`);+ // Without a job id, a recovered tool call supplies a stable output identity.+ const outputKey = digest(`${renderKey}:${job || meta.codexCallId || 'saved-output'}`);+ return {+ version: 1, id: outputKey, renderKey, occurredAt,+ timestampSource: timestamp(meta.renderStartedAt) ? 'renderStartedAt' : 'createdAt',+ identitySource: session ? 'codex-session' : 'portos-job',+ driverModel: typeof meta.model === 'string' ? meta.model : null,+ driverTokens: Number.isSafeInteger(meta.driverReportedTokens) && meta.driverReportedTokens >= 0+ ? { reportedTotal: meta.driverReportedTokens, source: 'codex-cli-footer' } : null,+ imageTokens: null, imageCostUsd: null, allowanceUnits: null,+ source: 'portos-saved-render',+ };+}++export async function readRenderReceipts({ directory = ledgerDir() } = {}) {+ const entries = await readdir(directory, { withFileTypes: true }).catch(err => {+ if (err.code === 'ENOENT') return [];+ throw err;+ });+ const receipts = [];+ for (const entry of entries) {+ if (!entry.isFile() || !/^[a-f0-9]{64}\.json$/.test(entry.name)) continue;+ const receipt = JSON.parse(await readFile(join(directory, entry.name), 'utf8'));+ if (receipt.version !== 1 || receipt.id !== entry.name.slice(0, -5) || !timestamp(receipt.occurredAt) || !/^[a-f0-9]{64}$/.test(receipt.renderKey)) {+ throw new Error('Invalid image render usage receipt');+ }+ receipts.push(receipt);+ }+ return receipts;+}++// Publish a complete inode without replacing an existing receipt. Unlike a+// read/modify/write ledger this also serializes live renders against a separate+// backfill process. A crash leaves at most an ignored staging file.+export async function publishAccountingFile(path, value) {+ await ensureDir(dirname(path));+ const staging = join(dirname(path), `.${randomUUID()}.tmp`);+ await atomicWrite(staging, value);+ try {+ await link(staging, path);+ await syncDirectory(dirname(path));+ return true;+ } catch (err) {+ if (err.code !== 'EEXIST') throw err;+ return false;+ } finally {+ await unlinkGuarded(staging);+ }+}++export async function recordRenderReceipt(receipt, { directory = ledgerDir() } = {}) {+ if (!receipt || !/^[a-f0-9]{64}$/.test(receipt.id)) throw new Error('Invalid image render receipt identity');+ const path = join(directory, `${receipt.id}.json`);+ let existing = await readFile(path, 'utf8').then(JSON.parse).catch(err => {+ if (err.code === 'ENOENT') return undefined;+ throw err;+ });+ if (existing === undefined) {+ if (await publishAccountingFile(path, receipt)) return true;+ existing = JSON.parse(await readFile(path, 'utf8'));+ }+ if (existing?.id !== receipt.id || existing.renderKey !== receipt.renderKey || existing.version !== 1) throw new Error('Conflicting image render receipt');+ return false;+}++// Human-mode Codex prints a token footer, not image usage or a directional+// breakdown. Retain exactly that reported driver total without pricing it or+// adding it again to the independently reconciled text-token ledger.+export function codexDriverReportedTokens(output) {+ const match = output.match(/(?:^|\n)tokens used\r?\n([\d,]+)\s*$/);+ const value = match ? Number(match[1].replaceAll(',', '')) : NaN;+ return Number.isSafeInteger(value) && value >= 0 ? value : null;+}++export async function recordCodexRender(meta) {+ const receipt = codexRenderReceipt(meta);+ if (!receipt) throw new Error('Codex render lacks accounting identity or original timestamp');+ return recordRenderReceipt(receipt);+}++export async function getImageRenderUsage({ from = null, to = null, directory } = {}) {+ const receipts = await readRenderReceipts({ directory });+ // Filter sessions by their first observed output so a session spanning UTC+ // midnight is not counted twice in adjacent reporting windows.+ const sessions = new Map();+ for (const receipt of receipts) {+ const session = sessions.get(receipt.renderKey) || { occurredAt: receipt.occurredAt, outputs: 0, fallback: false, driverTotal: null };+ if (receipt.occurredAt < session.occurredAt) session.occurredAt = receipt.occurredAt;+ session.outputs++;+ if (receipt.driverTokens?.source === 'codex-cli-footer' && Number.isSafeInteger(receipt.driverTokens.reportedTotal)) {+ session.driverTotal = Math.max(session.driverTotal ?? 0, receipt.driverTokens.reportedTotal);+ }+ session.fallback ||= receipt.identitySource === 'portos-job';+ sessions.set(receipt.renderKey, session);+ }+ let renderSessions = 0, savedOutputs = 0, jobIdentitySessions = 0, driverSessionsMeasured = 0, driverReportedTotal = 0;+ for (const session of sessions.values()) {+ const day = session.occurredAt.slice(0, 10);+ if ((from && day < from) || (to && day > to)) continue;+ renderSessions++;+ savedOutputs += session.outputs;+ if (session.fallback) jobIdentitySessions++;+ if (session.driverTotal !== null) { driverSessionsMeasured++; driverReportedTotal += session.driverTotal; }+ }+ return {+ scope: 'local-codex-saved-renders', renderSessions, savedOutputs, jobIdentitySessions,+ driverTokens: driverSessionsMeasured ? { reportedTotal: driverReportedTotal, sessionsMeasured: driverSessionsMeasured } : null,+ driverSessionsUnknown: renderSessions - driverSessionsMeasured,+ imageTokens: null, imageCostUsd: null, allowanceUnits: null,+ };+}