Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: keep share-bucket media-job imports through the media queue's saves (#12782)
Adam Eivy0b7354629e16
diff --git a/server/services/mediaJobQueue/importedJob.js b/server/services/mediaJobQueue/importedJob.jsnew file mode 100644index 0000000000..ec771bdbd6--- /dev/null+++ b/server/services/mediaJobQueue/importedJob.js@@ -0,0 +1,34 @@+import { isSafeRecordId } from '../../lib/validation.js';+import { isPlainObject } from '../../lib/objects.js';+import { isStr } from '../../lib/textUtils.js';+import { PARAM_ALLOWLIST } from './sanitizeJob.js';++// Re-render metadata a share bucket carries for a gallery asset. The record is+// peer-authored, so only completed image/video rows survive, and params are+// cut to the re-render knobs: worker paths, upload cleanup targets and the+// `remotePeerReservation` marker (which exempts a row from every archive trim)+// must never arrive from a bucket.+const IMPORTABLE_KINDS = new Set(['image', 'video']);+const RESULT_FILE_KEYS = ['filename', 'videoPath', 'thumbnail'];+const isBareFilename = (value) => isStr(value) && value.length > 0 && !/[\\/]/.test(value) && value !== '..' && value !== '.';++export function importedArchiveRow(raw) {+ if (!isPlainObject(raw) || !isSafeRecordId(raw.id)) return null;+ if (!IMPORTABLE_KINDS.has(raw.kind) || raw.status !== 'completed') return null;+ const params = Object.fromEntries(Object.entries(isPlainObject(raw.params) ? raw.params : {})+ .filter(([key]) => PARAM_ALLOWLIST.has(key)));+ const result = Object.fromEntries(RESULT_FILE_KEYS+ .filter((key) => isBareFilename(raw.result?.[key]))+ .map((key) => [key, raw.result[key]]));+ const completedAt = isStr(raw.completedAt) && Number.isFinite(Date.parse(raw.completedAt)) ? raw.completedAt : null;+ return {+ id: raw.id,+ kind: raw.kind,+ owner: isStr(raw.owner) ? raw.owner : null,+ status: 'completed',+ completedAt,+ params,+ result,+ imported: true,+ };+}diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex 8d5dda3097..3447c587b6 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -53,6 +53,7 @@ import { parseQuotaDeferral, formatDeferral } from '../../lib/imageQuotaSignal.jimport { VIDEO_GEN_MODE, CLOUD_VIDEO_GEN_MODES, mediaJobExecutionLane } from '../../lib/generationModes.js';import { REMOTE_MEDIA_MODULES, isRemoteMediaJob } from './remoteMediaJob.js';import { createVideoHolds } from './videoHolds.js';+import { importedArchiveRow } from './importedJob.js';import {canReceiveProgress, canRequestCancellation, claimTerminalOutcome,snapshotCancellationIntent, applyCancellationIntent, restoreCancellationIntent,@@ -113,6 +114,9 @@ const RESERVATION_RECONCILE_RETRY_MS = 30_000;const JOBS_FILE = join(PATHS.data, 'media-jobs.json');const COMPLETED_TTL_MS = 24 * 60 * 60 * 1000;const MAX_PERSISTED_ARCHIVE = 500;+// Share-bucket re-render records outlive the 24h TTL (they are old by nature) but+// stay bounded; the oldest imports drop first.+const MAX_IMPORTED_ARCHIVE = 2000;// Defaults (env-overridable). `Number(non-numeric)` → NaN, and// `setTimeout(NaN)` fires immediately — that would fail every job at boot// if MEDIA_JOB_WATCHDOG_*_MS were set to garbage. Fall back to the default@@ -639,12 +643,14 @@ async function persistImpl() {function trimArchiveForSnapshot() {const cutoff = Date.now() - COMPLETED_TTL_MS;const pendingArchive = archive.filter((job) => job.durabilityPending);- const trimmedArchive = archive- .filter((j) => {- const ts = j.completedAt ? new Date(j.completedAt).getTime() : Date.now();- return !j.durabilityPending && (j.params?.remotePeerReservation === true || ts > cutoff);- })- .filter((job) => job.params?.remotePeerReservation !== true).slice(-MAX_PERSISTED_ARCHIVE)+ const isImported = (job) => job.imported === true && !job.durabilityPending && job.params?.remotePeerReservation !== true;+ const trimmedArchive = archive.filter(isImported).slice(-MAX_IMPORTED_ARCHIVE)+ .concat(archive+ .filter((j) => {+ const ts = j.completedAt ? new Date(j.completedAt).getTime() : Date.now();+ return !isImported(j) && !j.durabilityPending && (j.params?.remotePeerReservation === true || ts > cutoff);+ })+ .filter((job) => job.params?.remotePeerReservation !== true).slice(-MAX_PERSISTED_ARCHIVE)).concat(archive.filter((job) => !job.durabilityPending && job.params?.remotePeerReservation === true), pendingArchive);// Mutate `archive` in place so subsequent reads see the trim too.archive.length = 0;@@ -757,8 +763,9 @@ function captureQueueSnapshot() {const canWriteOutcome = (job) => completionAdmitted || job.durabilityPending?.status !== 'completed';const hasDeferredCompletion = [...pendingTransitions.keys()].some((job) => !canWriteOutcome(job));// Strip non-serializable bits.- const serializable = live.map((job) => (canWriteOutcome(job) ? { ...job, ...job.durabilityPending } : job)).map(({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs, durabilityPending }) =>+ const serializable = live.map((job) => (canWriteOutcome(job) ? { ...job, ...job.durabilityPending } : job)).map(({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs, durabilityPending, imported }) =>({ id, kind, owner, status, queuedAt, startedAt, completedAt, params, result, error, position, progress, statusMsg, etaMs,+ ...(imported === true ? { imported } : {}),...(status !== 'completed' && durabilityPending?.status === 'completed' ? { durabilityPending } : {}) }),);const transitions = [...pendingTransitions].filter(([job]) => canWriteOutcome(job));@@ -1355,6 +1362,26 @@ export function quiesceMediaJobQueue() {return true;}+// Share-bucket import entry point. The queue owns media-jobs.json: a writer+// outside it is overwritten by the next persist() (the snapshot is built from+// memory alone), so imported re-render records go through the in-memory archive+// and are saved with the queue's own snapshot. Ids already known to the queue+// win; returns the number of rows added.+export async function importArchivedJobs(records) {+ if (!initPromise) throw new Error('Media queue is not initialized; imported jobs cannot be saved');+ await initPromise;+ if (persistBlocked) throw new Error('Media queue storage is blocked until its snapshot is repaired and the server restarts');+ let added = 0;+ for (const record of Array.isArray(records) ? records : []) {+ const row = importedArchiveRow(record);+ if (!row || findJob(row.id)) continue;+ archive.push(row);+ added += 1;+ }+ if (added) await persist();+ return added;+}+// Drop a job from the archive (e.g. after a successful retry — the old failed// row shouldn't keep cluttering the recent-reel UI now that a fresh job has// inherited its work). Returns true if removed, false if the id wasn't founddiff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 94305d41c2..eea633e05c 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -2460,6 +2460,62 @@ describe('mediaJobQueue unreadable snapshot (#4115)', () => {});});+describe('importArchivedJobs (#12782)', () => {+ const file = () => join(tempDataDir, 'media-jobs.json');+ const readJobs = () => JSON.parse(readFileSync(file(), 'utf8')).jobs;+ const OLD = '2026-01-01T00:00:00.000Z';+ const record = (n, patch = {}) => ({+ id: `00000000-0000-4000-8000-0000000000${n}`, kind: 'image', status: 'completed', completedAt: OLD,+ params: { prompt: 'a harbour', seed: 7 }, result: { filename: `${n}.png` }, ...patch,+ });++ it('makes an imported record visible without a restart and survives a later queue persist and the 24h trim', async () => {+ await mediaJobQueue.initMediaJobQueue();+ const imported = record('11');+ expect(await mediaJobQueue.importArchivedJobs([imported])).toBe(1);+ expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ params: { prompt: 'a harbour', seed: 7 }, result: { filename: '11.png' } });++ // The regression: an unrelated queue mutation rewrites the file from memory+ // and trims archive rows older than 24h — the imported row must outlive both.+ stubs.generateVideo.mockImplementation(() => new Promise(() => {}));+ await mediaJobQueue.enqueueJob({ kind: 'video', params: { prompt: 'later job' } });+ await flush();+ expect(readJobs().map((j) => j.id)).toContain(imported.id);+ expect(mediaJobQueue.getJob(imported.id)).toBeTruthy();++ await importFresh();+ await mediaJobQueue.initMediaJobQueue();+ expect(mediaJobQueue.getJob(imported.id)).toMatchObject({ imported: true, status: 'completed' });+ });++ it('keeps local rows on an id collision and refuses peer-supplied runtime state', async () => {+ writeFileSync(file(), JSON.stringify({ jobs: [record('21', { params: { prompt: 'local' }, completedAt: new Date().toISOString() })] }));+ await importFresh();+ await mediaJobQueue.initMediaJobQueue();+ const hostile = record('22', {+ params: { prompt: 'p', remotePeerReservation: true, uploadedTempPath: '/etc/passwd', initImagePath: '/etc/hosts' },+ result: { filename: '../../escape.png' },+ });+ const skipped = [record('23', { status: 'running' }), record('24', { kind: 'training' }), record('../25')];+ expect(await mediaJobQueue.importArchivedJobs([record('21', { params: { prompt: 'peer' } }), hostile, ...skipped])).toBe(1);+ expect(mediaJobQueue.getJob(record('21').id).params.prompt).toBe('local');+ const stored = mediaJobQueue.getJob(hostile.id);+ expect(stored.params).toEqual({ prompt: 'p' });+ expect(stored.result).toEqual({});+ expect(skipped.map((j) => mediaJobQueue.getJob(j.id))).toEqual([null, null, null]);+ });++ it('refuses to import into a queue whose snapshot was unreadable, leaving the file alone', async () => {+ const CORRUPT = '{"jobs": [{"id": "00000000-0000-4000-8000-0000000000aa",';+ writeFileSync(file(), CORRUPT);+ await importFresh();+ await mediaJobQueue.initMediaJobQueue();+ await expect(mediaJobQueue.importArchivedJobs([record('31')])).rejects.toThrow(/blocked/);+ expect(mediaJobQueue.getJob(record('31').id)).toBeNull();+ expect(readFileSync(file(), 'utf8')).toBe(CORRUPT);+ });+});+// Polling helper for the async settles this suite waits on (a debounced disk// write, a worker tick). A predicate that THROWS counts as "not true yet" and// keeps polling: several predicates readFileSync the persisted snapshot, whichdiff --git a/server/services/mediaJobQueue/sanitizeJob.js b/server/services/mediaJobQueue/sanitizeJob.jsindex bf01d5a34e..b2756ac172 100644--- a/server/services/mediaJobQueue/sanitizeJob.js+++ b/server/services/mediaJobQueue/sanitizeJob.js@@ -5,7 +5,7 @@ import { mediaJobExecutionLane } from '../../lib/generationModes.js';// Public projection of a media job. Keep worker-only paths and subprocess// details out of both the queue API and the processing dashboard.-const PARAM_ALLOWLIST = new Set([+export const PARAM_ALLOWLIST = new Set(['prompt', 'negativePrompt', 'modelId', 'model', 'effort','width', 'height', 'numFrames', 'fps', 'steps', 'guidanceScale','batchSize', 'seed', 'tiling', 'disableAudio', 'mode', 'imageStrength',diff --git a/server/services/sharing/importer.js b/server/services/sharing/importer.jsindex 4110b320be..9fed31eaa6 100644--- a/server/services/sharing/importer.js+++ b/server/services/sharing/importer.js@@ -17,8 +17,8 @@* local `data/{images,videos}/` directories (skip-if-present). v2+ manifests* use a content-addressed source at `assets/blobs/<hash>`; legacy v1 manifests* fall back to `assets/{images,videos}/<filename>`. Media-job records bundled- * in `records/media/` are merged into `data/media-jobs.json` so re-render- * workflows have the prompt + params.+ * in `records/media/` are handed to the media queue (which owns+ * `data/media-jobs.json`) so re-render workflows have the prompt + params.*/import { createKeyCachedQueue } from '../../lib/createKeyCachedQueue.js';@@ -50,6 +50,7 @@ import { maybeJournalBeforeOverwrite, flushBaseHashes, setSyncBaseHash, contentHimport { isStr } from '../../lib/textUtils.js';import { detectImageFormat, detectMediaManifest } from '../../lib/mimeTypes.js';import { ASSET_PULL_MAX_BYTES, SIDECAR_MAX_BYTES } from './assetLimits.js';+import { importArchivedJobs } from '../mediaJobQueue/index.js';import { mapWithConcurrency } from '../../lib/mapWithConcurrency.js';// Record kinds that participate in the non-blocking conflict journal. Universe,@@ -443,32 +444,19 @@ function importableMediaJob(raw, recordId) {};}-/** Merge bucket-bundled media-job records into local data/media-jobs.json. */+/** Hand bucket-bundled media-job records to the media queue, which owns data/media-jobs.json. */async function mergeMediaJobRecords(bucketPath, recordIds) {const mediaDir = bucketRecordsDir(bucketPath, 'media');if (!existsSync(mediaDir)) return;- const persistedPath = join(PATHS.data, 'media-jobs.json');- const [persisted, incoming] = await Promise.all([- readJSONFile(persistedPath, { jobs: [] }, { logError: false, strict: true }),- Promise.all((recordIds || []).map(async (id) => {- if (!isSafeRecordId(id)) return null;- const recordPath = bucketRecordPath(bucketPath, 'media', id);- if (!existsSync(recordPath)) return null;- // Input-only bucket record: unreadable data is skipped/deferred, never written back here.- return readJSONFile(recordPath, null, { logError: false });- })),- ]);- const byId = new Map((persisted.jobs || []).map((j) => [j.id, j]));- let changed = false;- for (const [index, raw] of incoming.entries()) {- const job = importableMediaJob(raw, recordIds[index]);- if (!job || byId.has(job.id)) continue;- byId.set(job.id, job);- changed = true;- }- if (changed) {- await atomicWrite(persistedPath, { ...persisted, jobs: Array.from(byId.values()) });- }+ const incoming = await Promise.all((recordIds || []).map(async (id) => {+ if (!isSafeRecordId(id)) return null;+ const recordPath = bucketRecordPath(bucketPath, 'media', id);+ if (!existsSync(recordPath)) return null;+ // Input-only bucket record: unreadable data is skipped/deferred, never written back here.+ return importableMediaJob(await readJSONFile(recordPath, null, { logError: false }), id);+ }));+ const records = incoming.filter(Boolean);+ if (records.length) await importArchivedJobs(records);}/**diff --git a/server/services/sharing/integration.test.js b/server/services/sharing/integration.test.jsindex b646037cbf..eb9044766d 100644--- a/server/services/sharing/integration.test.js+++ b/server/services/sharing/integration.test.js@@ -74,8 +74,12 @@ vi.mock('./peerSync.js', () => mockNoPeerSync());// Stub mediaJobQueue.getJob — exporter uses it for full-fidelity gen// metadata on each shared imageJobId. Returns null so the exporter falls// through to direct asset copy via imageRefs.+// importArchivedJobs is the importer's only write path into the queue; the+// media-job import test points it at the real queue to pin the persisted file.+const mediaQueue = vi.hoisted(() => ({ actual: null }));vi.mock('../mediaJobQueue/index.js', () => ({getJob: () => null,+ importArchivedJobs: (...args) => mediaQueue.actual.importArchivedJobs(...args),}));const buckets = await import('./buckets.js');@@ -326,20 +330,31 @@ describe('sharing round-trip', () => {it('preserves local video holds while importing bundled media-job records', async () => {const bucket = await buckets.createBucket({ name: 'Example Bucket', path: tempBucket, mode: 'inbox' });- const local = { id: 'retained-local', kind: 'video', status: 'queued', params: { modelId: 'example-video' } };+ const local = { id: 'retained-local', kind: 'video', status: 'completed', completedAt: new Date().toISOString(), params: { prompt: 'local' } };const videoHolds = [{ id: '00000000-0000-4000-8000-000000000001', modelId: 'example-video', runtime: 'mlx_video',classification: 'runtimeerror', cause: 'shader failed', heldAt: '2026-09-05T00:00:00.000Z' }];const file = join(tempData, 'media-jobs.json');- writeFileSync(file, JSON.stringify({ jobs: [local], videoHolds, futureEnvelopeField: 'preserved' }));- const incoming = { id: '00000000-0000-4000-8000-000000000002', kind: 'video', status: 'completed', params: {} };+ writeFileSync(file, JSON.stringify({ jobs: [local], videoHolds }));+ const incoming = { id: '00000000-0000-4000-8000-000000000002', kind: 'video', status: 'completed',+ completedAt: '2026-01-01T00:00:00.000Z', params: { prompt: 'bundled' }, result: { filename: 'bundled.mp4' } };mkdirSync(join(tempBucket, 'records', 'media'), { recursive: true });writeFileSync(join(tempBucket, 'records', 'media', `${incoming.id}.json`), JSON.stringify(incoming));writeFileSync(join(tempBucket, 'manifests', 'example-media.json'), JSON.stringify({id: 'example-manifest', senderInstanceId: 'example-peer', kind: 'media', recordIds: [incoming.id], assetRefs: [],}));- expect((await importer.processManifest(bucket.id, 'example-media.json')).processed).toBe(true);- expect(JSON.parse(readFileSync(file, 'utf8'))).toEqual({ jobs: [local, { ...incoming, owner: null, completedAt: null, result: null }],- videoHolds, futureEnvelopeField: 'preserved' });+ const queue = await vi.importActual('../mediaJobQueue/index.js');+ mediaQueue.actual = queue;+ await queue.initMediaJobQueue();+ try {+ expect((await importer.processManifest(bucket.id, 'example-media.json')).processed).toBe(true);+ // Visible without a restart, and older than the queue's 24h trim yet kept.+ expect(queue.getJob(incoming.id)).toMatchObject({ params: { prompt: 'bundled' }, result: { filename: 'bundled.mp4' } });+ const saved = JSON.parse(readFileSync(file, 'utf8'));+ expect(saved.jobs.map((j) => j.id).sort()).toEqual([local.id, incoming.id].sort());+ expect(saved.videoHolds).toEqual(videoHolds);+ } finally {+ await queue.__resetForTests();+ }});it('merges bundled media-job records only as completed re-render metadata under the id they were read by', async () => {@@ -364,11 +379,18 @@ describe('sharing round-trip', () => {writeFileSync(join(tempBucket, 'manifests', 'example-media.json'), JSON.stringify({id: 'example-manifest', senderInstanceId: 'example-peer', kind: 'media', recordIds: Object.keys(records), assetRefs: [],}));- await importer.processManifest(bucket.id, 'example-media.json');- expect(JSON.parse(readFileSync(file, 'utf8')).jobs).toEqual([{- id: id(4), kind: 'image', owner: null, status: 'completed', completedAt: '2026-09-05T00:00:00.000Z',- params: { prompt: 'example' }, result: { filename: `${id(4)}.png` },- }]);+ const queue = await vi.importActual('../mediaJobQueue/index.js');+ mediaQueue.actual = queue;+ await queue.initMediaJobQueue();+ try {+ await importer.processManifest(bucket.id, 'example-media.json');+ expect(JSON.parse(readFileSync(file, 'utf8')).jobs).toEqual([{+ id: id(4), kind: 'image', owner: null, status: 'completed', completedAt: '2026-09-05T00:00:00.000Z',+ params: { prompt: 'example' }, result: { filename: `${id(4)}.png` }, imported: true,+ }]);+ } finally {+ await queue.__resetForTests();+ }});it('names an exported media record by its job id, never by an imported sidecar id', async () => {diff --git a/server/services/sharingDurability.test.js b/server/services/sharingDurability.test.jsindex ce7aec9734..ee82f0f3e1 100644--- a/server/services/sharingDurability.test.js+++ b/server/services/sharingDurability.test.js@@ -27,7 +27,8 @@ vi.mock('./instanceIdentity.js', () => mockTestIdentity());vi.mock('./sharing/peerSync.js', () => mockNoPeerSync({}, {getOutboundCoverageForPeer: async () => ({ universe: new Set(), pipeline: new Set(), mediaCollections: new Set() }),}));-vi.mock('./mediaJobQueue/index.js', () => ({ getJob: () => null }));+// The queue owns data/media-jobs.json (its unreadable-snapshot latch is pinned in mediaJobQueue/index.test.js).+vi.mock('./mediaJobQueue/index.js', () => ({ getJob: () => null, importArchivedJobs: async () => 0 }));vi.mock('./sharing/annotationIdentity.js', () => ({ resolveBucketSourceName: async () => 'Test' }));vi.mock('../lib/peerHttpClient.js', () => ({ peerFetch: vi.fn(() => { throw new Error('Unexpected network request'); }) }));@@ -100,8 +101,6 @@ const cases = [() => manifests.markProcessed(bucket.id, 'new', 'new-id'), saved => { expect(saved.processedById).toEqual({ old: 'old-id', new: 'new-id' }); expect(saved.processed).toEqual(['legacy']); }],['import inbox', () => join(tempRoot, 'sharing/inbox', `${bucket.id}.json`), { items: [{ manifestId: 'old' }] },() => importer.processManifest(bucket.id, 'incoming.json'), saved => expect(saved.items.map(i => i.manifestId)).toEqual(['old', 'incoming'])],- ['import media jobs', () => join(tempRoot, 'media-jobs.json'), { jobs: [{ id: 'old' }], custom: true },- () => importer.processManifest(bucket.id, 'incoming.json'), saved => { expect(saved.jobs.map(j => j.id)).toEqual(['old', 'job-new']); expect(saved.custom).toBe(true); }],['annotation tombstones', () => join(bucketPath, 'records/media-annotations/test-instance.json'), { annotations: { 'image:note.png': { note: 'old' } } },() => annotations.exportAnnotationsToBucket({ ...bucket, mode: 'auto-merge' }, { 'image:new.png': { note: 'new' } }, 'test-instance'),saved => { expect(saved.annotations['image:note.png'].note).toBe(''); expect(saved.annotations['image:new.png'].note).toBe('new'); }],