Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve staged image inputs when reference scans are incomplete
[._.]/ Adam Eivy00e3472e2797
diff --git a/server/services/imageRefsGc.js b/server/services/imageRefsGc.jsindex b6d06311a7..698435abed 100644--- a/server/services/imageRefsGc.js+++ b/server/services/imageRefsGc.js@@ -11,17 +11,14 @@** This periodic sweep deletes `init-`/`ref-`-prefixed files in `data/image-refs`* that are BOTH:- * (a) not referenced by any gallery sidecar's `initImageFilename` /- * `referenceImageFilenames` basenames, AND+ * (a) not referenced by any gallery sidecar's applied, requested or omitted+ * input filenames, AND* (b) older than a grace window.** Why BOTH gates (not either):- * - The reference check alone is unsafe — the codex backend does NOT record an- * `initImageFilename` in its sidecar, so a still-wanted init upload behind a- * codex render would look unreferenced. The age gate is the backstop: a file- * younger than the window is spared regardless, covering codex lineage and- * renders still in flight (the staged file exists before its sidecar is- * written).+ * - The reference check alone is unsafe — newly staged inputs exist before a+ * render publishes its sidecar. The age gate spares those in-flight uploads+ * and gives legacy uploads without recorded lineage a grace window.* - The age check alone is unsafe — a long-lived gallery image that still* points at an old init upload would lose its lineage source. The reference* check protects those: any basename a kept sidecar names is never swept,@@ -38,7 +35,9 @@import { readdir, stat } from 'fs/promises';import { join, basename } from 'path';-import { PATHS, tryReadFile, safeJSONParse, unlinkGuarded } from '../lib/fileUtils.js';+import { PATHS, unlinkGuarded } from '../lib/fileUtils.js';+import { readJSONFileStrict, unreadableStoreError } from '../lib/jsonIo.js';+import { isPlainObject } from '../lib/objects.js';import { createSweepScheduler } from './sweepScheduler.js';// `init-<uuid>` / `ref-<uuid>` with a decodable image extension — the exact@@ -49,7 +48,7 @@ const STAGED_UPLOAD_RE = /^(?:init|ref)-[0-9a-f-]+\.(?:png|jpe?g|webp)$/i;// Grace window before an unreferenced staged upload is eligible for deletion.// Measured against the file's mtime. Generous on purpose: the cost of a stale// upload lingering an extra week is trivial next to deleting an init/reference-// source a render (e.g. a sidecar-less codex edit) still depends on.+// source before its render publishes the sidecar.export const ORPHAN_REF_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days// How often the sweep runs once started. These files are cheap to leave around,@@ -58,8 +57,9 @@ const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours/*** Scan every gallery sidecar and collect the set of `data/image-refs` basenames- * any kept image still references (`initImageFilename` + `referenceImageFilenames`).- * A basename in this set is never swept regardless of age.+ * any kept image still references, including requested or omitted provider inputs.+ * A basename in this set is never swept regardless of age. An incomplete scan+ * rejects before the sweep deletes anything; only a missing gallery is empty.** Reads the `*.metadata.json` sidecars directly rather than going through* `listGallery()` so the dir is injectable for tests and we don't pay for the@@ -67,21 +67,35 @@ const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours*/export async function collectReferencedRefBasenames(imagesDir = PATHS.images) {const referenced = new Set();- // A missing/unreadable images dir yields [] (no sidecars → nothing referenced).- const entries = await readdir(imagesDir).catch(() => []);+ const entries = await readdir(imagesDir).catch(err => {+ if (err.code === 'ENOENT') return [];+ throw err;+ });for (const name of entries) {if (!name.endsWith('.metadata.json')) continue;- const raw = await tryReadFile(join(imagesDir, name));- if (raw == null) continue;- const meta = safeJSONParse(raw, {});- if (typeof meta.initImageFilename === 'string' && meta.initImageFilename) {- referenced.add(basename(meta.initImageFilename));+ const path = join(imagesDir, name);+ const { ok, value: meta } = await readJSONFileStrict(path, null, { logError: false });+ // Even ENOENT here is an incomplete scan: a listed sidecar disappearing+ // can be an in-flight replacement, not proof that its inputs are orphaned.+ if (!ok || !isPlainObject(meta)) throw unreadableStoreError(path);+ const filenames = [];+ if (meta.initImageFilename != null) filenames.push(meta.initImageFilename);+ if (meta.referenceImageFilenames != null) {+ if (!Array.isArray(meta.referenceImageFilenames)) throw unreadableStoreError(path);+ filenames.push(...meta.referenceImageFilenames);}- if (Array.isArray(meta.referenceImageFilenames)) {- for (const f of meta.referenceImageFilenames) {- if (typeof f === 'string' && f) referenced.add(basename(f));+ for (const field of ['requestedInputImages', 'omittedInputImages']) {+ if (meta[field] == null) continue;+ if (!Array.isArray(meta[field])) throw unreadableStoreError(path);+ for (const input of meta[field]) {+ if (!isPlainObject(input)) throw unreadableStoreError(path);+ filenames.push(input.filename);}}+ for (const filename of filenames) {+ if (typeof filename !== 'string') throw unreadableStoreError(path);+ if (filename) referenced.add(basename(filename));+ }}return referenced;}@@ -119,7 +133,7 @@ export async function sweepOrphanRefImages({continue;}// mtime backstop — spare anything younger than the grace window so a- // sidecar-less codex render or an in-flight render isn't swept early. A+ // legacy sidecar-less render or an in-flight render isn't swept early. A// stat failure means the file vanished under us; skip it silently.const info = await stat(join(refsDir, name)).catch(() => null);if (!info) continue;diff --git a/server/services/imageRefsGc.test.js b/server/services/imageRefsGc.test.jsindex e992e546b4..fb39395fe8 100644--- a/server/services/imageRefsGc.test.js+++ b/server/services/imageRefsGc.test.js@@ -1,4 +1,5 @@-import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import * as fsPromises from 'fs/promises';import { mkdir, writeFile, rm, utimes, readdir } from 'fs/promises';import { existsSync } from 'fs';import { join } from 'path';@@ -9,6 +10,11 @@ import {ORPHAN_REF_MAX_AGE_MS,} from './imageRefsGc.js';+vi.mock('fs/promises', async importOriginal => {+ const fs = await importOriginal();+ return { ...fs, readFile: vi.fn(fs.readFile), readdir: vi.fn(fs.readdir) };+});+// Pin a clock so age math is deterministic.const NOW = Date.parse('2026-06-12T00:00:00Z');const OLD = NOW - ORPHAN_REF_MAX_AGE_MS - 60_000; // just past the grace window@@ -43,6 +49,7 @@ beforeEach(async () => {});afterEach(async () => {+ vi.resetAllMocks();await rm(root, { recursive: true, force: true });});@@ -57,7 +64,6 @@ describe('collectReferencedRefBasenames', () => {it('normalizes any path to a basename and tolerates absent/empty fields', async () => {await sidecar('job1', { init: '/abs/path/init-ddd.png', refs: [] });await sidecar('job2', {}); // both fields null/empty- await writeFile(join(imagesDir, 'job3.metadata.json'), '{ not json'); // toleratedconst set = await collectReferencedRefBasenames(imagesDir);expect([...set]).toEqual(['init-ddd.png']);});@@ -77,6 +83,71 @@ describe('sweepOrphanRefImages', () => {const KEPT_REF = 'ref-dddddddd-4444-4444-8444-dddddddddddd.png';const YOUNG_INIT = 'init-eeeeeeee-5555-4555-8555-eeeeeeeeeeee.png';+ it('preserves requested peer inputs and omitted local inputs through repeated sweeps', async () => {+ await stageRef(KEPT_INIT);+ await stageRef(KEPT_REF);+ await stageRef(ORPHAN_A);+ await writeFile(join(imagesDir, 'peer.metadata.json'), JSON.stringify({+ federatedPeerId: 'example-peer',+ requestedInputImages: [{ role: 'initImage', filename: KEPT_INIT }],+ }));+ await writeFile(join(imagesDir, 'local.metadata.json'), JSON.stringify({+ omittedInputImages: [{ role: 'referenceImages', filename: KEPT_REF, reason: 'unsupported-render-path' }],+ }));+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))+ .toEqual({ deleted: 1, keptReferenced: 2, keptYoung: 0 });+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))+ .toEqual({ deleted: 0, keptReferenced: 2, keptYoung: 0 });+ expect((await readdir(refsDir)).sort()).toEqual([KEPT_INIT, KEPT_REF].sort());+ });++ it.each([+ ['truncated JSON', '{ not json'],+ ['null', 'null'],+ ['array', '[]'],+ ['scalar', '42'],+ ['invalid legacy references', JSON.stringify({ referenceImageFilenames: 'ref-aaa.png' })],+ ['invalid requested reference', JSON.stringify({ requestedInputImages: [{ filename: 42 }] })],+ ['invalid omitted references', JSON.stringify({ omittedInputImages: {} })],+ ])('does not delete any upload on %s metadata, and retries after repair', async (_label, raw) => {+ await stageRef(ORPHAN_A);+ await stageRef(KEPT_INIT);+ await sidecar('valid', { init: KEPT_INIT });+ const brokenPath = join(imagesDir, 'broken.metadata.json');+ await writeFile(brokenPath, raw);+ await expect(sweepOrphanRefImages({ now: NOW, refsDir, imagesDir })).rejects.toThrow();+ expect((await readdir(refsDir)).sort()).toEqual([ORPHAN_A, KEPT_INIT].sort());+ await writeFile(brokenPath, '{}');+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))+ .toEqual({ deleted: 1, keptReferenced: 1, keptYoung: 0 });+ });++ it.each(['gallery listing', 'sidecar read', 'sidecar vanished'])('stops on failed %s and succeeds on retry', async (failure) => {+ await stageRef(ORPHAN_A);+ await stageRef(KEPT_INIT);+ await sidecar('valid', { init: KEPT_INIT });+ const method = failure === 'gallery listing' ? 'readdir' : 'readFile';+ const original = (await vi.importActual('fs/promises'))[method];+ const target = failure === 'gallery listing' ? imagesDir : join(imagesDir, 'valid.metadata.json');+ const spy = fsPromises[method].mockImplementation((path, ...args) => {+ if (path === target) return Promise.reject(Object.assign(new Error('Fixture read failure'), {+ code: failure === 'sidecar vanished' ? 'ENOENT' : 'EACCES',+ }));+ return original(path, ...args);+ });+ await expect(sweepOrphanRefImages({ now: NOW, refsDir, imagesDir })).rejects.toThrow();+ expect((await readdir(refsDir)).sort()).toEqual([ORPHAN_A, KEPT_INIT].sort());+ spy.mockReset();+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))+ .toEqual({ deleted: 1, keptReferenced: 1, keptYoung: 0 });+ });++ it('cleans proven orphans when the gallery directory has never been created', async () => {+ await stageRef(ORPHAN_A);+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir: join(root, 'absent-gallery') }))+ .toEqual({ deleted: 1, keptReferenced: 0, keptYoung: 0 });+ });+it('deletes old, unreferenced staged uploads', async () => {await stageRef(ORPHAN_A, OLD);await stageRef(ORPHAN_B, OLD);