Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: keep share-bucket media-job imports through the media queue's saves (#12782)

Adam Eivy0b7354629e16
diff --git a/server/services/mediaJobQueue/importedJob.js b/server/services/mediaJobQueue/importedJob.js
new file mode 100644
index 0000000000..ec771bdbd6
--- /dev/null
+++ b/server/services/mediaJobQueue/importedJob.js
@@ -0,0 +1,34 @@
+import { isSafeRecordId } from '../../lib/validation.js';
+import { isPlainObject } from '../../lib/objects.js';
+import { isStr } from '../../lib/textUtils.js';
+import { PARAM_ALLOWLIST } from './sanitizeJob.js';
+
+// Re-render metadata a share bucket carries for a gallery asset. The record is
+// peer-authored, so only completed image/video rows survive, and params are
+// cut to the re-render knobs: worker paths, upload cleanup targets and the
+// `remotePeerReservation` marker (which exempts a row from every archive trim)
+// must never arrive from a bucket.
+const IMPORTABLE_KINDS = new Set(['image', 'video']);
+const RESULT_FILE_KEYS = ['filename', 'videoPath', 'thumbnail'];
+const isBareFilename = (value) => isStr(value) && value.length > 0 && !/[\\/]/.test(value) && value !== '..' && value !== '.';
+
+export function importedArchiveRow(raw) {
+ if (!isPlainObject(raw) || !isSafeRecordId(raw.id)) return null;
+ if (!IMPORTABLE_KINDS.has(raw.kind) || raw.status !== 'completed') return null;
+ const params = Object.fromEntries(Object.entries(isPlainObject(raw.params) ? raw.params : {})
+ .filter(([key]) => PARAM_ALLOWLIST.has(key)));
+ const result = Object.fromEntries(RESULT_FILE_KEYS
+ .filter((key) => isBareFilename(raw.result?.[key]))
+ .map((key) => [key, raw.result[key]]));
+ const completedAt = isStr(raw.completedAt) && Number.isFinite(Date.parse(raw.completedAt)) ? raw.completedAt : null;
+ return {
+ id: raw.id,
+ kind: raw.kind,
+ owner: isStr(raw.owner) ? raw.owner : null,
+ status: 'completed',
+ completedAt,
+ params,
+ result,
+ imported: true,
+ };
+}
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index 8d5dda3097..3447c587b6 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -53,6 +53,7 @@ import { parseQuotaDeferral, formatDeferral } from '../../lib/imageQuotaSignal.j
import { VIDEO_GEN_MODE, CLOUD_VIDEO_GEN_MODES, mediaJobExecutionLane } from '../../lib/generationModes.js';
import { REMOTE_MEDIA_MODULES, isRemoteMediaJob } from './remoteMediaJob.js';
import { createVideoHolds } from './videoHolds.js';
+import { importedArchiveRow } from './importedJob.js';
import {
canReceiveProgress, canRequestCancellation, claimTerminalOutcome,
snapshotCancellationIntent, applyCancellationIntent, restoreCancellationIntent,
@@ -113,6 +114,9 @@ const RESERVATION_RECONCILE_RETRY_MS = 30_000;
const JOBS_FILE = join(PATHS.data, 'media-jobs.json');
const COMPLETED_TTL_MS = 24 * 60 * 60 * 1000;
const MAX_PERSISTED_ARCHIVE = 500;
+// Share-bucket re-render records outlive the 24h TTL (they are old by nature) but
+// stay bounded; the oldest imports drop first.
+const MAX_IMPORTED_ARCHIVE = 2000;
// Defaults (env-overridable). `Number(non-numeric)` → NaN, and
// `setTimeout(NaN)` fires immediately — that would fail every job at boot
// if MEDIA_JOB_WATCHDOG_*_MS were set to garbage. Fall back to the default
@@ -639,12 +643,14 @@ async function persistImpl() {
function trimArchiveForSnapshot() {
const cutoff = Date.now() - COMPLETED_TTL_MS;
const pendingArchive = archive.filter((job) => job.durabilityPending);
- const trimmedArchive = archive
- .filter((j) => {
- const ts = j.completedAt ? new Date(j.completedAt).getTime() : Date.now();
- return !j.durabilityPending && (j.params?.remotePeerReservation === true || ts > cutoff);
- })
- .filter((job) => job.params?.remotePeerReservation !== true).slice(-MAX_PERSISTED_ARCHIVE)
+ const isImported = (job) => job.imported === true && !job.durabilityPending && job.params?.remotePeerReservation !== true;
+ const trimmedArchive = archive.filter(isImported).slice(-MAX_IMPORTED_ARCHIVE)
+ .concat(archive
+ .filter((j) => {
+ const ts = j.completedAt ? new Date(j.completedAt).getTime() : Date.now();
+ return !isImported(j) && !j.durabilityPending && (j.params?.remotePeerReservation === true || ts > cutoff);
+ })
+ .filter((job) => job.params?.remotePeerReservation !== true).slice(-MAX_PERSISTED_ARCHIVE))
.concat(archive.filter((job) => !job.durabilityPending && job.params?.remotePeerReservation === true), pendingArchive);
// Mutate `archive` in place so subsequent reads see the trim too.
archive.length = 0;
@@ -757,8 +763,9 @@ function captureQueueSnapshot() {
const canWriteOutcome = (job) => completionAdmitted || job.durabilityPending?.status !== 'completed';
const hasDeferredCompletion = [...pendingTransitions.keys()].some((job) => !canWriteOutcome(job));
// Strip non-serializable bits.
- const serializable = live.map((job) => (canWriteOutcome(job) ? { ...job, ...job.durabilityPending } : job)).map(({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs, durabilityPending }) =>
+ const serializable = live.map((job) => (canWriteOutcome(job) ? { ...job, ...job.durabilityPending } : job)).map(({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs, durabilityPending, imported }) =>
({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs,
+ ...(imported === true ? { imported } : {}),
...(status !== 'completed' && durabilityPending?.status === 'completed' ? { durabilityPending } : {}) }),
);
const transitions = [...pendingTransitions].filter(([job]) => canWriteOutcome(job));
@@ -1355,6 +1362,26 @@ export function quiesceMediaJobQueue() {
return true;
}
+// Share-bucket import entry point. The queue owns media-jobs.json: a writer
+// outside it is overwritten by the next persist() (the snapshot is built from
+// memory alone), so imported re-render records go through the in-memory archive
+// and are saved with the queue's own snapshot. Ids already known to the queue
+// win; returns the number of rows added.
+export async function importArchivedJobs(records) {
+ if (!initPromise) throw new Error('Media queue is not initialized; imported jobs cannot be saved');
+ await initPromise;
+ if (persistBlocked) throw new Error('Media queue storage is blocked until its snapshot is repaired and the server restarts');
+ let added = 0;
+ for (const record of Array.isArray(records) ? records : []) {
+ const row = importedArchiveRow(record);
+ if (!row || findJob(row.id)) continue;
+ archive.push(row);
+ added += 1;
+ }
+ if (added) await persist();
+ return added;
+}
+
// Drop a job from the archive (e.g. after a successful retry — the old failed
// row shouldn't keep cluttering the recent-reel UI now that a fresh job has
// inherited its work). Returns true if removed, false if the id wasn't found
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 94305d41c2..eea633e05c 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -2460,6 +2460,62 @@ describe('mediaJobQueue unreadable snapshot (#4115)', () => {
});
});
+describe('importArchivedJobs (#12782)', () => {
+ const file = () => join(tempDataDir, 'media-jobs.json');
+ const readJobs = () => JSON.parse(readFileSync(file(), 'utf8')).jobs;
+ const OLD = '2026-01-01T00:00:00.000Z';
+ const record = (n, patch = {}) => ({
+ id: `00000000-0000-4000-8000-0000000000${n}`, kind: 'image', status: 'completed', completedAt: OLD,
+ params: { prompt: 'a harbour', seed: 7 }, result: { filename: `${n}.png` }, ...patch,
+ });
+
+ it('makes an imported record visible without a restart and survives a later queue persist and the 24h trim', async () => {
+ await mediaJobQueue.initMediaJobQueue();
+ const imported = record('11');
+ expect(await mediaJobQueue.importArchivedJobs([imported])).toBe(1);
+ expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ params: { prompt: 'a harbour', seed: 7 }, result: { filename: '11.png' } });
+
+ // The regression: an unrelated queue mutation rewrites the file from memory
+ // and trims archive rows older than 24h — the imported row must outlive both.
+ stubs.generateVideo.mockImplementation(() => new Promise(() => {}));
+ await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'later job' } });
+ await flush();
+ expect(readJobs().map((j) => j.id)).toContain(imported.id);
+ expect(mediaJobQueue.getJob(imported.id)).toBeTruthy();
+
+ await importFresh();
+ await mediaJobQueue.initMediaJobQueue();
+ expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ imported: true, status: 'completed' });
+ });
+
+ it('keeps local rows on an id collision and refuses peer-supplied runtime state', async () => {
+ writeFileSync(file(), JSON.stringify({ jobs: [record('21', { params: { prompt: 'local' }, completedAt: new Date().toISOString() })] }));
+ await importFresh();
+ await mediaJobQueue.initMediaJobQueue();
+ const hostile = record('22', {
+ params: { prompt: 'p', remotePeerReservation: true, uploadedTempPath: '/etc/passwd', initImagePath: '/etc/hosts' },
+ result: { filename: '../../escape.png' },
+ });
+ const skipped = [record('23', { status: 'running' }), record('24', { kind: 'training' }), record('../25')];
+ expect(await mediaJobQueue.importArchivedJobs([record('21', { params: { prompt: 'peer' } }), hostile, ...skipped])).toBe(1);
+ expect(mediaJobQueue.getJob(record('21').id).params.prompt).toBe('local');
+ const stored = mediaJobQueue.getJob(hostile.id);
+ expect(stored.params).toEqual({ prompt: 'p' });
+ expect(stored.result).toEqual({});
+ expect(skipped.map((j) => mediaJobQueue.getJob(j.id))).toEqual([null, null, null]);
+ });
+
+ it('refuses to import into a queue whose snapshot was unreadable, leaving the file alone', async () => {
+ const CORRUPT = '{"jobs": [{"id": "00000000-0000-4000-8000-0000000000aa",';
+ writeFileSync(file(), CORRUPT);
+ await importFresh();
+ await mediaJobQueue.initMediaJobQueue();
+ await expect(mediaJobQueue.importArchivedJobs([record('31')])).rejects.toThrow(/blocked/);
+ expect(mediaJobQueue.getJob(record('31').id)).toBeNull();
+ expect(readFileSync(file(), 'utf8')).toBe(CORRUPT);
+ });
+});
+
// Polling helper for the async settles this suite waits on (a debounced disk
// write, a worker tick). A predicate that THROWS counts as "not true yet" and
// keeps polling: several predicates readFileSync the persisted snapshot, which
diff --git a/server/services/mediaJobQueue/sanitizeJob.js b/server/services/mediaJobQueue/sanitizeJob.js
index bf01d5a34e..b2756ac172 100644
--- a/server/services/mediaJobQueue/sanitizeJob.js
+++ b/server/services/mediaJobQueue/sanitizeJob.js
@@ -5,7 +5,7 @@ import { mediaJobExecutionLane } from '../../lib/generationModes.js';
// Public projection of a media job. Keep worker-only paths and subprocess
// details out of both the queue API and the processing dashboard.
-const PARAM_ALLOWLIST = new Set([
+export const PARAM_ALLOWLIST = new Set([
'prompt', 'negativePrompt', 'modelId', 'model', 'effort',
'width', 'height', 'numFrames', 'fps', 'steps', 'guidanceScale',
'batchSize', 'seed', 'tiling', 'disableAudio', 'mode', 'imageStrength',
diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.js
index 4110b320be..9fed31eaa6 100644
--- a/server/services/sharing/importer.js
+++ b/server/services/sharing/importer.js
@@ -17,8 +17,8 @@
* local `data/{images,videos}/` directories (skip-if-present). v2+ manifests
* use a content-addressed source at `assets/blobs/<hash>`; legacy v1 manifests
* fall back to `assets/{images,videos}/<filename>`. Media-job records bundled
- * in `records/media/` are merged into `data/media-jobs.json` so re-render
- * workflows have the prompt + params.
+ * in `records/media/` are handed to the media queue (which owns
+ * `data/media-jobs.json`) so re-render workflows have the prompt + params.
*/
import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';
@@ -50,6 +50,7 @@ import { maybeJournalBeforeOverwrite, flushBaseHashes, setSyncBaseHash, contentH
import { isStr } from '../../lib/textUtils.js';
import { detectImageFormat, detectMediaManifest } from '../../lib/mimeTypes.js';
import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';
+import { importArchivedJobs } from '../mediaJobQueue/index.js';
import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';
// Record kinds that participate in the non-blocking conflict journal. Universe,
@@ -443,32 +444,19 @@ function importableMediaJob(raw, recordId) {
};
}
-/** Merge bucket-bundled media-job records into local data/media-jobs.json. */
+/** Hand bucket-bundled media-job records to the media queue, which owns data/media-jobs.json. */
async function mergeMediaJobRecords(bucketPath, recordIds) {
const mediaDir = bucketRecordsDir(bucketPath, 'media');
if (!existsSync(mediaDir)) return;
- const persistedPath = join(PATHS.data, 'media-jobs.json');
- const [persisted, incoming] = await Promise.all([
- readJSONFile(persistedPath, { jobs: [] }, { logError: false, strict: true }),
- Promise.all((recordIds || []).map(async (id) => {
- if (!isSafeRecordId(id)) return null;
- const recordPath = bucketRecordPath(bucketPath, 'media', id);
- if (!existsSync(recordPath)) return null;
- // Input-only bucket record: unreadable data is skipped/deferred, never written back here.
- return readJSONFile(recordPath, null, { logError: false });
- })),
- ]);
- const byId = new Map((persisted.jobs || []).map((j) => [j.id, j]));
- let changed = false;
- for (const [index, raw] of incoming.entries()) {
- const job = importableMediaJob(raw, recordIds[index]);
- if (!job || byId.has(job.id)) continue;
- byId.set(job.id, job);
- changed = true;
- }
- if (changed) {
- await atomicWrite(persistedPath, { ...persisted, jobs: Array.from(byId.values()) });
- }
+ const incoming = await Promise.all((recordIds || []).map(async (id) => {
+ if (!isSafeRecordId(id)) return null;
+ const recordPath = bucketRecordPath(bucketPath, 'media', id);
+ if (!existsSync(recordPath)) return null;
+ // Input-only bucket record: unreadable data is skipped/deferred, never written back here.
+ return importableMediaJob(await readJSONFile(recordPath, null, { logError: false }), id);
+ }));
+ const records = incoming.filter(Boolean);
+ if (records.length) await importArchivedJobs(records);
}
/**
diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.js
index b646037cbf..eb9044766d 100644
--- a/server/services/sharing/integration.test.js
+++ b/server/services/sharing/integration.test.js
@@ -74,8 +74,12 @@ vi.mock('./peerSync.js', () => mockNoPeerSync());
// Stub mediaJobQueue.getJob — exporter uses it for full-fidelity gen
// metadata on each shared imageJobId. Returns null so the exporter falls
// through to direct asset copy via imageRefs.
+// importArchivedJobs is the importer's only write path into the queue; the
+// media-job import test points it at the real queue to pin the persisted file.
+const mediaQueue = vi.hoisted(() => ({ actual: null }));
vi.mock('../mediaJobQueue/index.js', () => ({
getJob: () => null,
+ importArchivedJobs: (...args) => mediaQueue.actual.importArchivedJobs(...args),
}));
const buckets = await import('./buckets.js');
@@ -326,20 +330,31 @@ describe('sharing round-trip', () => {
it('preserves local video holds while importing bundled media-job records', async () => {
const bucket = await buckets.createBucket({ name: 'Example Bucket', path: tempBucket, mode: 'inbox' });
- const local = { id: 'retained-local', kind: 'video', status: 'queued', params: { modelId: 'example-video' } };
+ const local = { id: 'retained-local', kind: 'video', status: 'completed', completedAt: new Date().toISOString(), params: { prompt: 'local' } };
const videoHolds = [{ id: '00000000-0000-4000-8000-000000000001', modelId: 'example-video', runtime: 'mlx_video',
classification: 'runtimeerror', cause: 'shader failed', heldAt: '2026-09-05T00:00:00.000Z' }];
const file = join(tempData, 'media-jobs.json');
- writeFileSync(file, JSON.stringify({ jobs: [local], videoHolds, futureEnvelopeField: 'preserved' }));
- const incoming = { id: '00000000-0000-4000-8000-000000000002', kind: 'video', status: 'completed', params: {} };
+ writeFileSync(file, JSON.stringify({ jobs: [local], videoHolds }));
+ const incoming = { id: '00000000-0000-4000-8000-000000000002', kind: 'video', status: 'completed',
+ completedAt: '2026-01-01T00:00:00.000Z', params: { prompt: 'bundled' }, result: { filename: 'bundled.mp4' } };
mkdirSync(join(tempBucket, 'records', 'media'), { recursive: true });
writeFileSync(join(tempBucket, 'records', 'media', `${incoming.id}.json`), JSON.stringify(incoming));
writeFileSync(join(tempBucket, 'manifests', 'example-media.json'), JSON.stringify({
id: 'example-manifest', senderInstanceId: 'example-peer', kind: 'media', recordIds: [incoming.id], assetRefs: [],
}));
- expect((await importer.processManifest(bucket.id, 'example-media.json')).processed).toBe(true);
- expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual({ jobs: [local, { ...incoming, owner: null, completedAt: null, result: null }],
- videoHolds, futureEnvelopeField: 'preserved' });
+ const queue = await vi.importActual('../mediaJobQueue/index.js');
+ mediaQueue.actual = queue;
+ await queue.initMediaJobQueue();
+ try {
+ expect((await importer.processManifest(bucket.id, 'example-media.json')).processed).toBe(true);
+ // Visible without a restart, and older than the queue's 24h trim yet kept.
+ expect(queue.getJob(incoming.id)).toMatchObject({ params: { prompt: 'bundled' }, result: { filename: 'bundled.mp4' } });
+ const saved = JSON.parse(readFileSync(file, 'utf8'));
+ expect(saved.jobs.map((j) => j.id).sort()).toEqual([local.id, incoming.id].sort());
+ expect(saved.videoHolds).toEqual(videoHolds);
+ } finally {
+ await queue.__resetForTests();
+ }
});
it('merges bundled media-job records only as completed re-render metadata under the id they were read by', async () => {
@@ -364,11 +379,18 @@ describe('sharing round-trip', () => {
writeFileSync(join(tempBucket, 'manifests', 'example-media.json'), JSON.stringify({
id: 'example-manifest', senderInstanceId: 'example-peer', kind: 'media', recordIds: Object.keys(records), assetRefs: [],
}));
- await importer.processManifest(bucket.id, 'example-media.json');
- expect(JSON.parse(readFileSync(file, 'utf8')).jobs).toEqual([{
- id: id(4), kind: 'image', owner: null, status: 'completed', completedAt: '2026-09-05T00:00:00.000Z',
- params: { prompt: 'example' }, result: { filename: `${id(4)}.png` },
- }]);
+ const queue = await vi.importActual('../mediaJobQueue/index.js');
+ mediaQueue.actual = queue;
+ await queue.initMediaJobQueue();
+ try {
+ await importer.processManifest(bucket.id, 'example-media.json');
+ expect(JSON.parse(readFileSync(file, 'utf8')).jobs).toEqual([{
+ id: id(4), kind: 'image', owner: null, status: 'completed', completedAt: '2026-09-05T00:00:00.000Z',
+ params: { prompt: 'example' }, result: { filename: `${id(4)}.png` }, imported: true,
+ }]);
+ } finally {
+ await queue.__resetForTests();
+ }
});
it('names an exported media record by its job id, never by an imported sidecar id', async () => {
diff --git a/server/services/sharingDurability.test.js b/server/services/sharingDurability.test.js
index ce7aec9734..ee82f0f3e1 100644
--- a/server/services/sharingDurability.test.js
+++ b/server/services/sharingDurability.test.js
@@ -27,7 +27,8 @@ vi.mock('./instanceIdentity.js', () => mockTestIdentity());
vi.mock('./sharing/peerSync.js', () => mockNoPeerSync({}, {
getOutboundCoverageForPeer: async () => ({ universe: new Set(), pipeline: new Set(), mediaCollections: new Set() }),
}));
-vi.mock('./mediaJobQueue/index.js', () => ({ getJob: () => null }));
+// The queue owns data/media-jobs.json (its unreadable-snapshot latch is pinned in mediaJobQueue/index.test.js).
+vi.mock('./mediaJobQueue/index.js', () => ({ getJob: () => null, importArchivedJobs: async () => 0 }));
vi.mock('./sharing/annotationIdentity.js', () => ({ resolveBucketSourceName: async () => 'Test' }));
vi.mock('../lib/peerHttpClient.js', () => ({ peerFetch: vi.fn(() => { throw new Error('Unexpected network request'); }) }));
@@ -100,8 +101,6 @@ const cases = [
() => manifests.markProcessed(bucket.id, 'new', 'new-id'), saved => { expect(saved.processedById).toEqual({ old: 'old-id', new: 'new-id' }); expect(saved.processed).toEqual(['legacy']); }],
['import inbox', () => join(tempRoot, 'sharing/inbox', `${bucket.id}.json`), { items: [{ manifestId: 'old' }] },
() => importer.processManifest(bucket.id, 'incoming.json'), saved => expect(saved.items.map(i => i.manifestId)).toEqual(['old', 'incoming'])],
- ['import media jobs', () => join(tempRoot, 'media-jobs.json'), { jobs: [{ id: 'old' }], custom: true },
- () => importer.processManifest(bucket.id, 'incoming.json'), saved => { expect(saved.jobs.map(j => j.id)).toEqual(['old', 'job-new']); expect(saved.custom).toBe(true); }],
['annotation tombstones', () => join(bucketPath, 'records/media-annotations/test-instance.json'), { annotations: { 'image:note.png': { note: 'old' } } },
() => annotations.exportAnnotationsToBucket({ ...bucket, mode: 'auto-merge' }, { 'image:new.png': { note: 'new' } }, 'test-instance'),
saved => { expect(saved.annotations['image:note.png'].note).toBe(''); expect(saved.annotations['image:new.png'].note).toBe('new'); }],