Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12619]): fail renders a restart left queued (#12747)
[._.]/cc38e7daeeb2
diff --git a/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx b/client/src/components/musicVideo/YoutubeThumbnailPanel.jsxindex 9fa82cce04..63b6158150 100644--- a/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx+++ b/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx@@ -105,6 +105,9 @@ export default function YoutubeThumbnailPanel({ project, publishKit }) {const tests = Array.isArray(kit.youtubeThumbnailTests) ? kit.youtubeThumbnailTests.filter((t) => t?.filename) : [];const inTest = Boolean(thumb.filename) && tests.some((t) => t.filename === thumb.filename);const crop = wide ? focusX : focusY;+ // `pending` survives a read only while the paint is live. A restart clears a+ // dead one and sets lastError, which leaves Paint enabled. The click itself+ // stays disabled until that request returns.const painting = Boolean(publishKit.paintingThumbnail || thumb.pending);const paintTitle = title.trim();// The exact prompt the paint would send, from the server's own builder, refreshed as the title or the committed notes change.diff --git a/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx b/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsxindex 7c97062296..1dfbe24297 100644--- a/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx+++ b/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx@@ -56,4 +56,31 @@ describe('YoutubeThumbnailPanel', () => {fireEvent.click(screen.getByRole('button', { name: /Paint the title into this image/ }));expect(k.paintThumbnail).toHaveBeenCalledWith({ source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', notes: 'chrome letters' });});++ const openPaint = async (kit, publishKit) => {+ render(<YoutubeThumbnailPanel project={project({ youtubeThumbnail: kit })} publishKit={publishKit} />);+ fireEvent.click(screen.getByRole('button', { name: /YouTube thumbnail/ }));+ fireEvent.click(screen.getByText('Paint the title with the image model'));+ await waitFor(() => expect(screen.getByRole('button', { name: '+ YouTube thumbnail brief' })).toBeTruthy());+ };++ it('disables Paint while the server still reports a live paint', async () => {+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true, pending: { requestId: 'r1', jobId: 'job-live', mode: 'codex' } };+ await openPaint(made, hook());+ expect(screen.getByRole('button', { name: 'Painting…' })).toBeDisabled();+ expect(screen.getByRole('status').textContent).toContain('Painting on codex');+ });++ it('enables Paint once a restart has cleared the pending paint', async () => {+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true, pending: null, lastError: 'interrupted by restart' };+ await openPaint(made, hook());+ expect(screen.getByRole('button', { name: 'Paint the title into this image' })).toBeEnabled();+ expect(screen.getByRole('alert').textContent).toBe('interrupted by restart');+ });++ it('disables Paint while the click is still queuing', async () => {+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true };+ await openPaint(made, hook({ paintingThumbnail: true }));+ expect(screen.getByRole('button', { name: 'Queuing…' })).toBeDisabled();+ });});diff --git a/server/routes/decks.db.test.js b/server/routes/decks.db.test.jsindex c095b91c1f..eda097dba4 100644--- a/server/routes/decks.db.test.js+++ b/server/routes/decks.db.test.js@@ -1,5 +1,5 @@/** Deck designer over HTTP and PostgreSQL. Runs only against portos_test. */-import { afterAll, describe, expect, it } from 'vitest';+import { afterAll, describe, expect, it, vi } from 'vitest';import express from 'express';import { randomUUID } from 'node:crypto';import { checkHealth, ensureSchema, query, close } from '../lib/db.js';@@ -9,6 +9,7 @@ import { errorMiddleware } from '../lib/errorHandler.js';import deckRoutes from './decks.js';import { attachCardRender, markCardsRenderQueued, markCardRenderTerminal } from '../services/decks.js';import { cardStatus } from '../lib/deckTemplates.js';+import * as mediaJobs from '../services/mediaJobQueue/index.js';const health = await checkHealth().catch((error) => ({ connected: false, error: error.message }));const ready = requireDbOrSkip('routes/decks.db.test', health.connected, health.error);@@ -101,11 +102,27 @@ describe.skipIf(!ready)('decks over HTTP and PostgreSQL', () => {await patch(`/${deck.id}/cards/${card.id}`, { prompt: 'a queen' });await markCardsRenderQueued(deck.id, [{ cardId: card.id, render: { jobId: 'job-1', status: 'queued', queuedAt: new Date().toISOString() } }]);const readCard = async () => (await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id);- expect(cardStatus(await readCard())).toBe('queued');+ // job-1 is not in the media queue. A detail read treats that as interrupted+ // unless the job is still queued or running.+ const live = vi.spyOn(mediaJobs, 'getJob').mockImplementation((jobId) => (+ jobId === 'job-1' ? { id: jobId, status: 'queued' } : null+ ));+ try {+ expect(cardStatus(await readCard())).toBe('queued');+ } finally {+ live.mockRestore();+ }// A failure event for a superseded job must not flip the card.expect(await markCardRenderTerminal(deck.id, card.id, { jobId: 'job-0', status: 'failed' })).toBeNull();- expect(cardStatus(await readCard())).toBe('queued');+ const stillQueued = vi.spyOn(mediaJobs, 'getJob').mockImplementation((jobId) => (+ jobId === 'job-1' ? { id: jobId, status: 'queued' } : null+ ));+ try {+ expect(cardStatus(await readCard())).toBe('queued');+ } finally {+ stillQueued.mockRestore();+ }const attached = await attachCardRender({ deckId: deck.id, cardId: card.id, filename: 'job-1.png', jobId: 'job-1' });expect(attached).toMatchObject({ imageRefs: ['job-1.png'], primaryImageRef: 'job-1.png' });@@ -126,6 +143,34 @@ describe.skipIf(!ready)('decks over HTTP and PostgreSQL', () => {expect(await attachCardRender({ deckId: deck.id, cardId: randomUUID(), filename: 'x.png', jobId: 'j' })).toBeNull();});+ it('reads a card whose render job is gone as interrupted without moving the deck clock', async () => {+ const deck = await create({ name: 'Interrupted Deck', kind: 'playing' });+ const card = deck.cards.find((c) => c.key === 'spades-K');+ await patch(`/${deck.id}/cards/${card.id}`, { prompt: 'a king' });+ const before = (await get(`/${deck.id}`)).body;+ const beforeCard = before.cards.find((c) => c.id === card.id);+ await markCardsRenderQueued(deck.id, [{ cardId: card.id, render: { jobId: 'job-dead', status: 'queued', queuedAt: '2026-01-01T00:00:00.000Z' } }]);+ const read = await get(`/${deck.id}`);+ const healed = read.body.cards.find((c) => c.id === card.id);+ expect(healed.render).toMatchObject({ jobId: 'job-dead', status: 'failed', error: 'interrupted by restart' });+ expect(cardStatus(healed)).toBe('failed');+ expect(read.body.updatedAt).toBe(before.updatedAt);+ expect(healed.updatedAt).toBe(beforeCard.updatedAt);+ expect(cardStatus((await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id))).toBe('failed');+ });++ it('fails a queued card that never received a job id', async () => {+ const deck = await create({ name: 'Unstamped Deck', kind: 'playing' });+ const card = deck.cards.find((c) => c.key === 'clubs-A');+ await query(+ `UPDATE deck_cards SET definition = jsonb_set(COALESCE(definition, '{}'::jsonb), '{render}', $2::jsonb) WHERE id = $1`,+ [card.id, JSON.stringify({ status: 'queued', queuedAt: '2026-01-01T00:00:00.000Z' })],+ );+ const healed = (await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id);+ expect(healed.render).toMatchObject({ status: 'failed', error: 'interrupted by restart' });+ expect(healed.render.jobId ?? null).toBeNull();+ });+it('adds a sample with an adopted style guide in one write, removes it, and cascades on delete', async () => {const deck = await create({ name: 'Sample Deck', kind: 'tarot' });const sample = { id: 'deck-sample-1', title: 'Poster', prompt: 'a lighthouse poster', imageRef: 'upload-1.png' };diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.jsindex 060969876b..7f3485633d 100644--- a/server/routes/musicVideo.js+++ b/server/routes/musicVideo.js@@ -253,14 +253,21 @@ const projectUpdateSchema = musicVideoProjectUpdateSchema.extend(recordRenderPin// Approvals decide stage completion and only the server computes them (#10136), so the read// response carries `productionReadiness` — stages derive from one record, never "not done" while a// second request is in flight. GET /:id/production-review still serves the full review payload.-const presentProjectForRead = (project) => {- const presented = presentProjectAutonomousRun(presentProjectCastAndSets(presentMusicVideoDependencies(project)));+// A restart archives a thumbnail paint without clearing `pending`. Heal on read,+// and only when one is present, so a project list does not load the image stack.+async function presentProjectForRead(project) {+ let current = project;+ if (project?.publishKit?.youtubeThumbnail?.pending) {+ const { reconcileInterruptedYoutubeThumbnail } = await import('../services/musicVideo/youtubeThumbnail.js');+ current = await reconcileInterruptedYoutubeThumbnail(project);+ }+ const presented = presentProjectAutonomousRun(presentProjectCastAndSets(presentMusicVideoDependencies(current)));return { ...presented, productionReadiness: productionReadiness(presented) };-};+}router.get('/', asyncHandler(async (req, res) => {const query = validateRequest(musicVideoProjectListQuerySchema, req.query);- const projects = (await listProjects()).map(presentProjectForRead);+ const projects = await Promise.all((await listProjects()).map((project) => presentProjectForRead(project)));if (query.summary === '1' || query.summary === 'true') {const summaries = projects.sort(compareMusicVideoProjectsNewestFirst).map((project) => summarizeMusicVideoProject(project, project.productionReadiness));@@ -301,7 +308,7 @@ router.get('/:id', asyncHandler(async (req, res) => {const p = await requireProject(req.params.id);// Transient (never persisted): lets a reloaded page reattach to a running publishing-kit build.const activePublishKitBuild = getActivePublishKitBuild(p.id);- const presented = presentProjectForRead(p);+ const presented = await presentProjectForRead(p);res.json(activePublishKitBuild ? { ...presented, activePublishKitBuild } : presented);}));@@ -369,7 +376,7 @@ router.put('/:id/finished-outside', asyncHandler(async (req, res) => {}const finishedOutside = finished ? { markedAt: new Date().toISOString(), ...(note ? { note } : {}) } : null;await updateProject(req.params.id, { finishedOutside });- res.json(presentProjectForRead(await getProject(req.params.id)));+ res.json(await presentProjectForRead(await getProject(req.params.id)));}));// Preview is read-only; Apply rechecks audio and the serialized project basis.diff --git a/server/services/decks.js b/server/services/decks.jsindex 65971babed..2e43120431 100644--- a/server/services/decks.js+++ b/server/services/decks.js@@ -122,6 +122,57 @@ const loadCards = async (client, deckId) => {return rows.map(projectCard);};+const INTERRUPTED_RENDER_ERROR = 'interrupted by restart';++// Same read-time projection as a mood-board note: a queued card whose job is+// gone fails. A missing job id is not a live claim (deck stamps always carry+// one). The wire strips `render`, so this must not move either clock.+function projectInterruptedCardRenders(cards, isLive = () => false) {+ const updates = [];+ let changed = false;+ const next = (Array.isArray(cards) ? cards : []).map((card) => {+ const render = card?.render;+ if (!render || render.status !== 'queued') return card;+ if (render.jobId && isLive(render.jobId)) return card;+ changed = true;+ updates.push({ id: card.id, jobId: render.jobId || '' });+ return { ...card, render: { ...render, status: 'failed', error: INTERRUPTED_RENDER_ERROR } };+ });+ return { cards: changed ? next : cards, changed, updates };+}++async function reconcileInterruptedCardRenders(deckId, cards) {+ if (!cards.some((card) => card.render?.status === 'queued')) return null;+ const { getJob } = await import('./mediaJobQueue/index.js');+ const isLive = (jobId) => {+ const status = getJob(jobId)?.status;+ return status === 'queued' || status === 'running';+ };+ const projected = projectInterruptedCardRenders(cards, isLive);+ if (!projected.changed) return null;+ const stampedAt = new Date().toISOString();+ let wrote = 0;+ await queueRecordWrite(deckId, () => withTransaction(async (client) => {+ for (const card of projected.updates) {+ const { rowCount } = await client.query(+ `UPDATE deck_cards+ SET definition = jsonb_set(+ definition,+ '{render}',+ (definition->'render') || jsonb_build_object('status', 'failed', 'error', $4::text, 'updatedAt', $5::text)+ )+ WHERE deck_id = $1 AND id = $2+ AND definition->'render'->>'status' = 'queued'+ AND COALESCE(definition->'render'->>'jobId', '') = $3`,+ [deckId, card.id, card.jobId, INTERRUPTED_RENDER_ERROR, stampedAt],+ );+ wrote += rowCount;+ }+ }));+ if (wrote) console.warn(`⚠️ Deck ${deckId}: a card render was interrupted by restart`);+ return loadCards({ query }, deckId);+}+export async function listDecks({ includeDeleted = false } = {}) {await ensureSchema();const { rows } = await query(@@ -142,11 +193,17 @@ export async function listDecks({ includeDeleted = false } = {}) {}));}-export async function getDeck(id, { includeDeleted = false } = {}) {+export async function getDeck(id, { includeDeleted = false, reconcileRenders = true } = {}) {await ensureSchema();const { rows } = await query('SELECT * FROM decks WHERE id = $1', [id]);if (!rows[0] || (!includeDeleted && rows[0].deleted === true)) throw notFound();- const cards = await loadCards({ query }, id);+ let cards = await loadCards({ query }, id);+ // Callers already inside this deck's write tail must pass reconcileRenders+ // false. The tail is not re-entrant, and a nested heal would deadlock.+ if (reconcileRenders) {+ const reconciled = await reconcileInterruptedCardRenders(id, cards);+ if (reconciled) cards = reconciled;+ }return { ...projectDeck(rows[0]), cards, completion: deckCompletion(cards) };}@@ -474,7 +531,7 @@ export function removeSample(id, sampleId) {* merge would treat an existing deck as new and skip the LWW compare entirely.*/export async function getDeckForSync(id) {- return getDeck(id, { includeDeleted: true }).catch((error) => {+ return getDeck(id, { includeDeleted: true, reconcileRenders: false }).catch((error) => {if (error?.code === 'DECK_NOT_FOUND') return null;throw error;});@@ -699,6 +756,6 @@ export async function restoreDeck(id, patch = {}) {],);emitRecordUpdated(DECK_KIND, id);- return getDeck(id);+ return getDeck(id, { reconcileRenders: false });});}diff --git a/server/services/moodBoard/db.js b/server/services/moodBoard/db.jsindex a3be690f30..98630537d2 100644--- a/server/services/moodBoard/db.js+++ b/server/services/moodBoard/db.js@@ -34,6 +34,7 @@ import {claimItemRender,settleItemRender,applyRenderedItem,+ projectInterruptedNoteRenders,mergeBoardRecord,applyPinterestLink,healPinterestFeedRecord,@@ -264,6 +265,24 @@ export async function updateBoardItem(id, itemId, patch) {// Text note render lifecycle (#10531) — see the logic.js transforms. The two// settle steps skip the write when the note moved on (stale job).+// Re-check liveness under the row lock. A pre-lock snapshot can mark a render+// that was claimed while this read was in flight. Returns null when the board+// was deleted between the read and the lock.+export async function healInterruptedNoteRenders(id, { isLive = () => false, now } = {}) {+ try {+ let changed = false;+ const locked = await withLockedBoard(id, (board) => {+ const projected = projectInterruptedNoteRenders(board, { isLive, ...(now ? { now } : {}) });+ changed = projected.changed;+ return { board: projected.board, skipPersist: !projected.changed };+ });+ return { board: locked.board, changed };+ } catch (err) {+ if (err?.code === 'NOT_FOUND') return null;+ throw err;+ }+}+export async function claimBoardItemRender(id, itemId, opts) {const { result } = await withLockedBoard(id, (b) => {const { board, item } = claimItemRender(b, itemId, opts);diff --git a/server/services/moodBoard/index.js b/server/services/moodBoard/index.jsindex 4df1910a19..0f5f793351 100644--- a/server/services/moodBoard/index.js+++ b/server/services/moodBoard/index.js@@ -20,18 +20,42 @@ import * as store from './db.js';import { localizeImageUrl } from './localize.js';import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';-// Read paths + federation entry points pass straight through to the store. The+// Read paths + federation entry points pass straight through to the store, except+// getBoard, which projects a note render a restart interrupted. The// asset-manifest filename resolver lives in logic.js (pure) and is re-exported// here so peerSync imports a single module (mirrors creativeDirector/local.js// re-exporting startingImageFilename).export {listBoards,- getBoard,listBoardIds,listBoardNames,mergeBoardsFromSync,pruneTombstonedBoards,} from './db.js';++function hasQueuedTextNote(board) {+ return Array.isArray(board?.items) && board.items.some((item) => item?.type === 'text' && item.render?.status === 'queued');+}++// Sync loads with includeDeleted stay a pure read: healing there would emit+// from inside a peer push. Liveness matches claimItemRender (queued or running).+export async function getBoard(id, options = {}) {+ const board = await store.getBoard(id, options);+ if (!board || options.includeDeleted || !hasQueuedTextNote(board)) return board;+ const { getJob } = await import('../mediaJobQueue/index.js');+ const healed = await store.healInterruptedNoteRenders(id, {+ isLive: (jobId) => {+ const status = getJob(jobId)?.status;+ return status === 'queued' || status === 'running';+ },+ });+ if (!healed) return null;+ if (healed.changed) {+ emitRecordUpdated('moodBoard', id);+ console.warn(`⚠️ Mood board ${id}: a note render was interrupted by restart`);+ }+ return healed.board;+}export { imageUrlToAppAsset } from './logic.js';// Pinterest importer (link + manual "Sync now"). Lives in its own modulediff --git a/server/services/moodBoard/logic.js b/server/services/moodBoard/logic.jsindex 835336504a..3a1755a22f 100644--- a/server/services/moodBoard/logic.js+++ b/server/services/moodBoard/logic.js@@ -306,6 +306,29 @@ function replaceItem(board, items, idx, item) {// earlier render's job is still queued/running; a dead one may be replaced. A// claim that never got its job id (the process died mid-enqueue) expires.const RENDER_CLAIM_TTL_MS = 10 * 60 * 1000;+const INTERRUPTED_RENDER_ERROR = 'interrupted by restart';++// A restart archives the media job without telling the note, and replaying that+// failure at boot would run every other terminal listener. Project the same+// outcome when the board is read: a queued note whose job is gone fails, and a+// claim that never received a job id keeps the claim TTL above. `updatedAt`+// is wall-clock so the healed record wins LWW; `now` is only the TTL clock.+export function projectInterruptedNoteRenders(board, { isLive = () => false, now = nowIso() } = {}) {+ if (!board || !Array.isArray(board.items)) return { board, changed: false };+ let changed = false;+ const items = board.items.map((item) => {+ const render = item?.render;+ if (!item || item.type !== 'text' || render?.status !== 'queued') return item;+ const live = render.jobId+ ? Boolean(isLive(render.jobId))+ : Date.parse(now) - Date.parse(render.queuedAt) < RENDER_CLAIM_TTL_MS;+ if (live) return item;+ changed = true;+ return { ...item, render: { ...render, status: 'failed', error: INTERRUPTED_RENDER_ERROR } };+ });+ if (!changed) return { board, changed: false };+ return { board: { ...board, items, updatedAt: nowIso() }, changed: true };+}export function claimItemRender(board, itemId, { isLive = () => false, now = nowIso(), options } = {}) {const { items, idx } = findItemIndex(board, itemId);diff --git a/server/services/moodBoard/logic.test.js b/server/services/moodBoard/logic.test.jsindex 52e0e5739a..3e95003e00 100644--- a/server/services/moodBoard/logic.test.js+++ b/server/services/moodBoard/logic.test.js@@ -26,6 +26,7 @@ import {claimItemRender,settleItemRender,applyRenderedItem,+ projectInterruptedNoteRenders,} from './logic.js';describe('buildBoardRecord', () => {@@ -582,6 +583,35 @@ describe('text note render lifecycle (#10531)', () => {expect(() => claimItemRender(board(), 'i0')).toThrow(expect.objectContaining({ code: 'MOOD_BOARD_ITEM_NOT_TEXT' }));expect(() => claimItemRender(board(), 'missing')).toThrow(expect.objectContaining({ code: 'NOT_FOUND' }));});++ it('projects a queued note whose job is gone as interrupted, and leaves a live one', () => {+ const queued = settleItemRender(claimItemRender(board(), 'n1').board, 'n1', { jobId: 'job-dead', status: 'queued' }).board;+ queued.updatedAt = '2020-01-01T00:00:00.000Z';+ const dead = projectInterruptedNoteRenders(queued, { isLive: () => false });+ expect(dead.changed).toBe(true);+ expect(dead.board.items.find((it) => it.id === 'n1').render).toMatchObject({+ status: 'failed', error: 'interrupted by restart', jobId: 'job-dead',+ });+ expect(dead.board.updatedAt).not.toBe(queued.updatedAt);+ const live = projectInterruptedNoteRenders(queued, { isLive: (jobId) => jobId === 'job-dead' });+ expect(live).toEqual({ board: queued, changed: false });+ });++ it('keeps a claim with no job id until the claim TTL, then fails it', () => {+ const claimed = claimItemRender(board(), 'n1', { now: '2026-01-01T00:00:00.000Z' }).board;+ expect(projectInterruptedNoteRenders(claimed, { now: '2026-01-01T00:09:00.000Z' }).changed).toBe(false);+ const expired = projectInterruptedNoteRenders(claimed, { now: '2026-01-01T00:11:00.000Z' });+ expect(expired.board.items.find((it) => it.id === 'n1').render).toMatchObject({+ status: 'failed', error: 'interrupted by restart', jobId: null,+ });+ });++ it('ignores an image item and a note that is not queued', () => {+ const image = { id: 'i0', type: 'image', mediaKey: 'image:a.png', render: { status: 'queued', jobId: 'job-x' } };+ const failed = { id: 'n1', type: 'text', text: 'note', render: { status: 'failed', jobId: 'job-y', error: 'boom' } };+ const board = { items: [image, failed] };+ expect(projectInterruptedNoteRenders(board, { isLive: () => false })).toEqual({ board, changed: false });+ });});diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.jsindex 033cb5a1b4..c58a675cda 100644--- a/server/services/musicVideo/projects.js+++ b/server/services/musicVideo/projects.js@@ -531,7 +531,8 @@ async function mutateProjectRecordAdmitted(id, transform, { source = null, runIdlet dropped = null;const write = mutate || (await selectBackend()).mutateProjectRecord;const outcome = await write(id, async (current) => {- const outcome = transform(current);+ // A transform may be async: the thumbnail heal re-checks job liveness under the lock.+ const outcome = await transform(current);if (source) {const guarded = preserveAutomatedShotLocks(current, outcome.project, source, { runId });outcome.project = guarded.project;diff --git a/server/services/musicVideo/projectsDB.js b/server/services/musicVideo/projectsDB.jsindex 342bac2899..dc022b4e78 100644--- a/server/services/musicVideo/projectsDB.js+++ b/server/services/musicVideo/projectsDB.js@@ -103,7 +103,7 @@ async function mutateLocked(id, transform) {async function mutateProjectRecordUnqueued(id, transform) {const { project, result } = await withLockedProject(id, async (current) => {const outcome = await transform(current);- return { project: outcome.project, result: outcome };+ return { project: outcome.project, result: outcome, skipPersist: outcome.skipPersist === true };});return { ...result, project };}diff --git a/server/services/musicVideo/projectsFile.js b/server/services/musicVideo/projectsFile.jsindex 25ec9f444d..6887dbadd9 100644--- a/server/services/musicVideo/projectsFile.js+++ b/server/services/musicVideo/projectsFile.js@@ -106,6 +106,7 @@ async function reorderProjectScenesUnqueued(id, orderedIds) {async function mutateProject(id, transform) {const { all, idx } = await loadAllAndIndex(id);const outcome = await transform(all[idx]);+ if (outcome.skipPersist) return outcome;all[idx] = outcome.project;await saveAll(all);return outcome;diff --git a/server/services/musicVideo/youtubeThumbnail.js b/server/services/musicVideo/youtubeThumbnail.jsindex b59052fe27..ec54228250 100644--- a/server/services/musicVideo/youtubeThumbnail.js+++ b/server/services/musicVideo/youtubeThumbnail.js@@ -206,6 +206,43 @@ async function pendingIsLive(pending) {return Date.now() - Date.parse(pending.requestedAt || 0) < RESERVATION_TTL_MS;}+const INTERRUPTED_PAINT_ERROR = 'interrupted by restart';++// A restart archives the paint without clearing `pending`, and replaying the+// job failure at boot would run every other terminal listener. The publish-kit+// read projects that outcome instead. Not safe from getProject: the completion+// hook treats a missing pending as "not this request".+export async function reconcileInterruptedYoutubeThumbnail(project) {+ if (!project?.id) return project;+ const thumb = projectYoutubeThumbnail(project);+ if (!thumb.pending || await pendingIsLive(thumb.pending)) return project;+ const seenRequestId = thumb.pending.requestId || null;+ const seenJobId = thumb.pending.jobId || null;+ const out = await mutateProjectRecord(project.id, async (current) => {+ const currentThumb = projectYoutubeThumbnail(current);+ const pending = currentThumb.pending;+ const same = pending+ && (pending.requestId || null) === seenRequestId+ && (pending.jobId || null) === seenJobId;+ if (!same || await pendingIsLive(pending)) return { project: current, skipPersist: true, healed: false };+ const kit = projectPublishKit(current);+ return {+ project: {+ ...current,+ publishKit: {+ ...kit,+ youtubeThumbnail: { ...currentThumb, pending: null, lastError: INTERRUPTED_PAINT_ERROR },+ },+ },+ healed: true,+ };+ });+ if (!out.healed) return out.project;+ console.warn(`⚠️ Music Video YouTube thumbnail ${String(project.id).slice(3, 11)}: painting interrupted by restart`);+ musicVideoEvents.emit('cover-art', { projectId: project.id, project: out.project });+ return out.project;+}+const writeThumb = (projectId, update) => mutateProjectRecord(projectId, (current) => {const kit = projectPublishKit(current);const thumb = projectYoutubeThumbnail(current);diff --git a/server/services/musicVideo/youtubeThumbnail.test.js b/server/services/musicVideo/youtubeThumbnail.test.jsindex 63a3835808..29377161d2 100644--- a/server/services/musicVideo/youtubeThumbnail.test.js+++ b/server/services/musicVideo/youtubeThumbnail.test.js@@ -144,6 +144,56 @@ describe('painting the title with the image model', () => {expect(Math.max(...stats.channels.map((c) => c.stdev))).toBeLessThan(2);});+ it('clears a paint a restart interrupted and still accepts the next one', async () => {+ const { id, frame } = await projectWithFrame();+ await projects.mutateProjectRecord(id, (current) => ({+ project: {+ ...current,+ publishKit: {+ ...current.publishKit,+ youtubeThumbnail: {+ pending: { requestId: 'req-dead', jobId: 'job-dead', mode: 'codex', requestedAt: '2020-01-01T00:00:00.000Z' },+ lastError: null,+ },+ },+ },+ }));+ thumbs.__setYoutubeThumbnailDepsForTests({ jobStatus: async () => 'failed' });+ const healed = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));+ expect(healed.publishKit.youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });+ expect((await kitOf(id)).youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });++ const enqueue = vi.fn(async () => ({ jobId: 'job-next' }));+ thumbs.__setYoutubeThumbnailDepsForTests({+ getSettings: async () => ({}), defaultRoute: async () => ({ mode: 'codex', model: null }),+ imageParams: async (_s, _r, common) => common, enqueue, jobStatus: async () => 'queued',+ });+ await thumbs.paintYoutubeThumbnail(id, { source: { kind: 'thumbnail', filename: frame }, title: 'Example Song' });+ expect((await kitOf(id)).youtubeThumbnail.pending).toMatchObject({ jobId: 'job-next' });+ thumbs.__setYoutubeThumbnailDepsForTests({});+ });++ it('leaves a live paint and a fresh reservation with no job id', async () => {+ const { id } = await projectWithFrame();+ const stamp = async (pending) => projects.mutateProjectRecord(id, (current) => ({+ project: { ...current, publishKit: { ...current.publishKit, youtubeThumbnail: { pending, lastError: null } } },+ }));+ thumbs.__setYoutubeThumbnailDepsForTests({ jobStatus: async () => 'running' });+ await stamp({ requestId: 'req-live', jobId: 'job-live', mode: 'codex', requestedAt: '2020-01-01T00:00:00.000Z' });+ const live = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));+ expect(live.publishKit.youtubeThumbnail.pending).toMatchObject({ jobId: 'job-live' });+ expect((await kitOf(id)).youtubeThumbnail.pending.jobId).toBe('job-live');++ await stamp({ requestId: 'req-reserved', jobId: null, mode: 'codex', requestedAt: new Date().toISOString() });+ const reserved = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));+ expect(reserved.publishKit.youtubeThumbnail.pending).toMatchObject({ requestId: 'req-reserved', jobId: null });++ await stamp({ requestId: 'req-stale', jobId: null, mode: 'codex', requestedAt: new Date(Date.now() - 3 * 60 * 1000).toISOString() });+ const stale = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));+ expect(stale.publishKit.youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });+ thumbs.__setYoutubeThumbnailDepsForTests({});+ });+it('records why a paint failed', async () => {const { id, frame } = await projectWithFrame();const enqueue = vi.fn(async () => ({ jobId: 'job-failing' }));