Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve staged image inputs when reference scans are incomplete

[._.]/ Adam Eivy00e3472e2797
diff --git a/server/services/imageRefsGc.js b/server/services/imageRefsGc.js
index b6d06311a7..698435abed 100644
--- a/server/services/imageRefsGc.js
+++ b/server/services/imageRefsGc.js
@@ -11,17 +11,14 @@
*
* This periodic sweep deletes `init-`/`ref-`-prefixed files in `data/image-refs`
* that are BOTH:
- * (a) not referenced by any gallery sidecar's `initImageFilename` /
- * `referenceImageFilenames` basenames, AND
+ * (a) not referenced by any gallery sidecar's applied, requested or omitted
+ * input filenames, AND
* (b) older than a grace window.
*
* Why BOTH gates (not either):
- * - The reference check alone is unsafe — the codex backend does NOT record an
- * `initImageFilename` in its sidecar, so a still-wanted init upload behind a
- * codex render would look unreferenced. The age gate is the backstop: a file
- * younger than the window is spared regardless, covering codex lineage and
- * renders still in flight (the staged file exists before its sidecar is
- * written).
+ * - The reference check alone is unsafe — newly staged inputs exist before a
+ * render publishes its sidecar. The age gate spares those in-flight uploads
+ * and gives legacy uploads without recorded lineage a grace window.
* - The age check alone is unsafe — a long-lived gallery image that still
* points at an old init upload would lose its lineage source. The reference
* check protects those: any basename a kept sidecar names is never swept,
@@ -38,7 +35,9 @@
import { readdir, stat } from 'fs/promises';
import { join, basename } from 'path';
-import { PATHS, tryReadFile, safeJSONParse, unlinkGuarded } from '../lib/fileUtils.js';
+import { PATHS, unlinkGuarded } from '../lib/fileUtils.js';
+import { readJSONFileStrict, unreadableStoreError } from '../lib/jsonIo.js';
+import { isPlainObject } from '../lib/objects.js';
import { createSweepScheduler } from './sweepScheduler.js';
// `init-<uuid>` / `ref-<uuid>` with a decodable image extension — the exact
@@ -49,7 +48,7 @@ const STAGED_UPLOAD_RE = /^(?:init|ref)-[0-9a-f-]+\.(?:png|jpe?g|webp)$/i;
// Grace window before an unreferenced staged upload is eligible for deletion.
// Measured against the file's mtime. Generous on purpose: the cost of a stale
// upload lingering an extra week is trivial next to deleting an init/reference
-// source a render (e.g. a sidecar-less codex edit) still depends on.
+// source before its render publishes the sidecar.
export const ORPHAN_REF_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
// How often the sweep runs once started. These files are cheap to leave around,
@@ -58,8 +57,9 @@ const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
/**
* Scan every gallery sidecar and collect the set of `data/image-refs` basenames
- * any kept image still references (`initImageFilename` + `referenceImageFilenames`).
- * A basename in this set is never swept regardless of age.
+ * any kept image still references, including requested or omitted provider inputs.
+ * A basename in this set is never swept regardless of age. An incomplete scan
+ * rejects before the sweep deletes anything; only a missing gallery is empty.
*
* Reads the `*.metadata.json` sidecars directly rather than going through
* `listGallery()` so the dir is injectable for tests and we don't pay for the
@@ -67,21 +67,35 @@ const SWEEP_INTERVAL_MS = 24 * 60 * 60 * 1000; // 24 hours
*/
export async function collectReferencedRefBasenames(imagesDir = PATHS.images) {
const referenced = new Set();
- // A missing/unreadable images dir yields [] (no sidecars → nothing referenced).
- const entries = await readdir(imagesDir).catch(() => []);
+ const entries = await readdir(imagesDir).catch(err => {
+ if (err.code === 'ENOENT') return [];
+ throw err;
+ });
for (const name of entries) {
if (!name.endsWith('.metadata.json')) continue;
- const raw = await tryReadFile(join(imagesDir, name));
- if (raw == null) continue;
- const meta = safeJSONParse(raw, {});
- if (typeof meta.initImageFilename === 'string' && meta.initImageFilename) {
- referenced.add(basename(meta.initImageFilename));
+ const path = join(imagesDir, name);
+ const { ok, value: meta } = await readJSONFileStrict(path, null, { logError: false });
+ // Even ENOENT here is an incomplete scan: a listed sidecar disappearing
+ // can be an in-flight replacement, not proof that its inputs are orphaned.
+ if (!ok || !isPlainObject(meta)) throw unreadableStoreError(path);
+ const filenames = [];
+ if (meta.initImageFilename != null) filenames.push(meta.initImageFilename);
+ if (meta.referenceImageFilenames != null) {
+ if (!Array.isArray(meta.referenceImageFilenames)) throw unreadableStoreError(path);
+ filenames.push(...meta.referenceImageFilenames);
}
- if (Array.isArray(meta.referenceImageFilenames)) {
- for (const f of meta.referenceImageFilenames) {
- if (typeof f === 'string' && f) referenced.add(basename(f));
+ for (const field of ['requestedInputImages', 'omittedInputImages']) {
+ if (meta[field] == null) continue;
+ if (!Array.isArray(meta[field])) throw unreadableStoreError(path);
+ for (const input of meta[field]) {
+ if (!isPlainObject(input)) throw unreadableStoreError(path);
+ filenames.push(input.filename);
}
}
+ for (const filename of filenames) {
+ if (typeof filename !== 'string') throw unreadableStoreError(path);
+ if (filename) referenced.add(basename(filename));
+ }
}
return referenced;
}
@@ -119,7 +133,7 @@ export async function sweepOrphanRefImages({
continue;
}
// mtime backstop — spare anything younger than the grace window so a
- // sidecar-less codex render or an in-flight render isn't swept early. A
+ // legacy sidecar-less render or an in-flight render isn't swept early. A
// stat failure means the file vanished under us; skip it silently.
const info = await stat(join(refsDir, name)).catch(() => null);
if (!info) continue;
diff --git a/server/services/imageRefsGc.test.js b/server/services/imageRefsGc.test.js
index e992e546b4..fb39395fe8 100644
--- a/server/services/imageRefsGc.test.js
+++ b/server/services/imageRefsGc.test.js
@@ -1,4 +1,5 @@
-import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import * as fsPromises from 'fs/promises';
import { mkdir, writeFile, rm, utimes, readdir } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
@@ -9,6 +10,11 @@ import {
ORPHAN_REF_MAX_AGE_MS,
} from './imageRefsGc.js';
+vi.mock('fs/promises', async importOriginal => {
+ const fs = await importOriginal();
+ return { ...fs, readFile: vi.fn(fs.readFile), readdir: vi.fn(fs.readdir) };
+});
+
// Pin a clock so age math is deterministic.
const NOW = Date.parse('2026-06-12T00:00:00Z');
const OLD = NOW - ORPHAN_REF_MAX_AGE_MS - 60_000; // just past the grace window
@@ -43,6 +49,7 @@ beforeEach(async () => {
});
afterEach(async () => {
+ vi.resetAllMocks();
await rm(root, { recursive: true, force: true });
});
@@ -57,7 +64,6 @@ describe('collectReferencedRefBasenames', () => {
it('normalizes any path to a basename and tolerates absent/empty fields', async () => {
await sidecar('job1', { init: '/abs/path/init-ddd.png', refs: [] });
await sidecar('job2', {}); // both fields null/empty
- await writeFile(join(imagesDir, 'job3.metadata.json'), '{ not json'); // tolerated
const set = await collectReferencedRefBasenames(imagesDir);
expect([...set]).toEqual(['init-ddd.png']);
});
@@ -77,6 +83,71 @@ describe('sweepOrphanRefImages', () => {
const KEPT_REF = 'ref-dddddddd-4444-4444-8444-dddddddddddd.png';
const YOUNG_INIT = 'init-eeeeeeee-5555-4555-8555-eeeeeeeeeeee.png';
+ it('preserves requested peer inputs and omitted local inputs through repeated sweeps', async () => {
+ await stageRef(KEPT_INIT);
+ await stageRef(KEPT_REF);
+ await stageRef(ORPHAN_A);
+ await writeFile(join(imagesDir, 'peer.metadata.json'), JSON.stringify({
+ federatedPeerId: 'example-peer',
+ requestedInputImages: [{ role: 'initImage', filename: KEPT_INIT }],
+ }));
+ await writeFile(join(imagesDir, 'local.metadata.json'), JSON.stringify({
+ omittedInputImages: [{ role: 'referenceImages', filename: KEPT_REF, reason: 'unsupported-render-path' }],
+ }));
+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))
+ .toEqual({ deleted: 1, keptReferenced: 2, keptYoung: 0 });
+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))
+ .toEqual({ deleted: 0, keptReferenced: 2, keptYoung: 0 });
+ expect((await readdir(refsDir)).sort()).toEqual([KEPT_INIT, KEPT_REF].sort());
+ });
+
+ it.each([
+ ['truncated JSON', '{ not json'],
+ ['null', 'null'],
+ ['array', '[]'],
+ ['scalar', '42'],
+ ['invalid legacy references', JSON.stringify({ referenceImageFilenames: 'ref-aaa.png' })],
+ ['invalid requested reference', JSON.stringify({ requestedInputImages: [{ filename: 42 }] })],
+ ['invalid omitted references', JSON.stringify({ omittedInputImages: {} })],
+ ])('does not delete any upload on %s metadata, and retries after repair', async (_label, raw) => {
+ await stageRef(ORPHAN_A);
+ await stageRef(KEPT_INIT);
+ await sidecar('valid', { init: KEPT_INIT });
+ const brokenPath = join(imagesDir, 'broken.metadata.json');
+ await writeFile(brokenPath, raw);
+ await expect(sweepOrphanRefImages({ now: NOW, refsDir, imagesDir })).rejects.toThrow();
+ expect((await readdir(refsDir)).sort()).toEqual([ORPHAN_A, KEPT_INIT].sort());
+ await writeFile(brokenPath, '{}');
+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))
+ .toEqual({ deleted: 1, keptReferenced: 1, keptYoung: 0 });
+ });
+
+ it.each(['gallery listing', 'sidecar read', 'sidecar vanished'])('stops on failed %s and succeeds on retry', async (failure) => {
+ await stageRef(ORPHAN_A);
+ await stageRef(KEPT_INIT);
+ await sidecar('valid', { init: KEPT_INIT });
+ const method = failure === 'gallery listing' ? 'readdir' : 'readFile';
+ const original = (await vi.importActual('fs/promises'))[method];
+ const target = failure === 'gallery listing' ? imagesDir : join(imagesDir, 'valid.metadata.json');
+ const spy = fsPromises[method].mockImplementation((path, ...args) => {
+ if (path === target) return Promise.reject(Object.assign(new Error('Fixture read failure'), {
+ code: failure === 'sidecar vanished' ? 'ENOENT' : 'EACCES',
+ }));
+ return original(path, ...args);
+ });
+ await expect(sweepOrphanRefImages({ now: NOW, refsDir, imagesDir })).rejects.toThrow();
+ expect((await readdir(refsDir)).sort()).toEqual([ORPHAN_A, KEPT_INIT].sort());
+ spy.mockReset();
+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir }))
+ .toEqual({ deleted: 1, keptReferenced: 1, keptYoung: 0 });
+ });
+
+ it('cleans proven orphans when the gallery directory has never been created', async () => {
+ await stageRef(ORPHAN_A);
+ expect(await sweepOrphanRefImages({ now: NOW, refsDir, imagesDir: join(root, 'absent-gallery') }))
+ .toEqual({ deleted: 1, keptReferenced: 0, keptYoung: 0 });
+ });
+
it('deletes old, unreferenced staged uploads', async () => {
await stageRef(ORPHAN_A, OLD);
await stageRef(ORPHAN_B, OLD);