Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12619]): fail renders a restart left queued (#12747)

[._.]/cc38e7daeeb2
diff --git a/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx b/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx
index 9fa82cce04..63b6158150 100644
--- a/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx
+++ b/client/src/components/musicVideo/YoutubeThumbnailPanel.jsx
@@ -105,6 +105,9 @@ export default function YoutubeThumbnailPanel({ project, publishKit }) {
const tests = Array.isArray(kit.youtubeThumbnailTests) ? kit.youtubeThumbnailTests.filter((t) => t?.filename) : [];
const inTest = Boolean(thumb.filename) && tests.some((t) => t.filename === thumb.filename);
const crop = wide ? focusX : focusY;
+ // `pending` survives a read only while the paint is live. A restart clears a
+ // dead one and sets lastError, which leaves Paint enabled. The click itself
+ // stays disabled until that request returns.
const painting = Boolean(publishKit.paintingThumbnail || thumb.pending);
const paintTitle = title.trim();
// The exact prompt the paint would send, from the server's own builder, refreshed as the title or the committed notes change.
diff --git a/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx b/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx
index 7c97062296..1dfbe24297 100644
--- a/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx
+++ b/client/src/components/musicVideo/YoutubeThumbnailPanel.test.jsx
@@ -56,4 +56,31 @@ describe('YoutubeThumbnailPanel', () => {
fireEvent.click(screen.getByRole('button', { name: /Paint the title into this image/ }));
expect(k.paintThumbnail).toHaveBeenCalledWith({ source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', notes: 'chrome letters' });
});
+
+ const openPaint = async (kit, publishKit) => {
+ render(<YoutubeThumbnailPanel project={project({ youtubeThumbnail: kit })} publishKit={publishKit} />);
+ fireEvent.click(screen.getByRole('button', { name: /YouTube thumbnail/ }));
+ fireEvent.click(screen.getByText('Paint the title with the image model'));
+ await waitFor(() => expect(screen.getByRole('button', { name: '+ YouTube thumbnail brief' })).toBeTruthy());
+ };
+
+ it('disables Paint while the server still reports a live paint', async () => {
+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true, pending: { requestId: 'r1', jobId: 'job-live', mode: 'codex' } };
+ await openPaint(made, hook());
+ expect(screen.getByRole('button', { name: 'Painting…' })).toBeDisabled();
+ expect(screen.getByRole('status').textContent).toContain('Painting on codex');
+ });
+
+ it('enables Paint once a restart has cleared the pending paint', async () => {
+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true, pending: null, lastError: 'interrupted by restart' };
+ await openPaint(made, hook());
+ expect(screen.getByRole('button', { name: 'Paint the title into this image' })).toBeEnabled();
+ expect(screen.getByRole('alert').textContent).toBe('interrupted by restart');
+ });
+
+ it('disables Paint while the click is still queuing', async () => {
+ const made = { filename: 'yt-1.jpg', source: { kind: 'thumbnail', filename: 'best.jpg' }, title: 'Example Song', lettering: true };
+ await openPaint(made, hook({ paintingThumbnail: true }));
+ expect(screen.getByRole('button', { name: 'Queuing…' })).toBeDisabled();
+ });
});
diff --git a/server/routes/decks.db.test.js b/server/routes/decks.db.test.js
index c095b91c1f..eda097dba4 100644
--- a/server/routes/decks.db.test.js
+++ b/server/routes/decks.db.test.js
@@ -1,5 +1,5 @@
/** Deck designer over HTTP and PostgreSQL. Runs only against portos_test. */
-import { afterAll, describe, expect, it } from 'vitest';
+import { afterAll, describe, expect, it, vi } from 'vitest';
import express from 'express';
import { randomUUID } from 'node:crypto';
import { checkHealth, ensureSchema, query, close } from '../lib/db.js';
@@ -9,6 +9,7 @@ import { errorMiddleware } from '../lib/errorHandler.js';
import deckRoutes from './decks.js';
import { attachCardRender, markCardsRenderQueued, markCardRenderTerminal } from '../services/decks.js';
import { cardStatus } from '../lib/deckTemplates.js';
+import * as mediaJobs from '../services/mediaJobQueue/index.js';
const health = await checkHealth().catch((error) => ({ connected: false, error: error.message }));
const ready = requireDbOrSkip('routes/decks.db.test', health.connected, health.error);
@@ -101,11 +102,27 @@ describe.skipIf(!ready)('decks over HTTP and PostgreSQL', () => {
await patch(`/${deck.id}/cards/${card.id}`, { prompt: 'a queen' });
await markCardsRenderQueued(deck.id, [{ cardId: card.id, render: { jobId: 'job-1', status: 'queued', queuedAt: new Date().toISOString() } }]);
const readCard = async () => (await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id);
- expect(cardStatus(await readCard())).toBe('queued');
+ // job-1 is not in the media queue. A detail read treats that as interrupted
+ // unless the job is still queued or running.
+ const live = vi.spyOn(mediaJobs, 'getJob').mockImplementation((jobId) => (
+ jobId === 'job-1' ? { id: jobId, status: 'queued' } : null
+ ));
+ try {
+ expect(cardStatus(await readCard())).toBe('queued');
+ } finally {
+ live.mockRestore();
+ }
// A failure event for a superseded job must not flip the card.
expect(await markCardRenderTerminal(deck.id, card.id, { jobId: 'job-0', status: 'failed' })).toBeNull();
- expect(cardStatus(await readCard())).toBe('queued');
+ const stillQueued = vi.spyOn(mediaJobs, 'getJob').mockImplementation((jobId) => (
+ jobId === 'job-1' ? { id: jobId, status: 'queued' } : null
+ ));
+ try {
+ expect(cardStatus(await readCard())).toBe('queued');
+ } finally {
+ stillQueued.mockRestore();
+ }
const attached = await attachCardRender({ deckId: deck.id, cardId: card.id, filename: 'job-1.png', jobId: 'job-1' });
expect(attached).toMatchObject({ imageRefs: ['job-1.png'], primaryImageRef: 'job-1.png' });
@@ -126,6 +143,34 @@ describe.skipIf(!ready)('decks over HTTP and PostgreSQL', () => {
expect(await attachCardRender({ deckId: deck.id, cardId: randomUUID(), filename: 'x.png', jobId: 'j' })).toBeNull();
});
+ it('reads a card whose render job is gone as interrupted without moving the deck clock', async () => {
+ const deck = await create({ name: 'Interrupted Deck', kind: 'playing' });
+ const card = deck.cards.find((c) => c.key === 'spades-K');
+ await patch(`/${deck.id}/cards/${card.id}`, { prompt: 'a king' });
+ const before = (await get(`/${deck.id}`)).body;
+ const beforeCard = before.cards.find((c) => c.id === card.id);
+ await markCardsRenderQueued(deck.id, [{ cardId: card.id, render: { jobId: 'job-dead', status: 'queued', queuedAt: '2026-01-01T00:00:00.000Z' } }]);
+ const read = await get(`/${deck.id}`);
+ const healed = read.body.cards.find((c) => c.id === card.id);
+ expect(healed.render).toMatchObject({ jobId: 'job-dead', status: 'failed', error: 'interrupted by restart' });
+ expect(cardStatus(healed)).toBe('failed');
+ expect(read.body.updatedAt).toBe(before.updatedAt);
+ expect(healed.updatedAt).toBe(beforeCard.updatedAt);
+ expect(cardStatus((await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id))).toBe('failed');
+ });
+
+ it('fails a queued card that never received a job id', async () => {
+ const deck = await create({ name: 'Unstamped Deck', kind: 'playing' });
+ const card = deck.cards.find((c) => c.key === 'clubs-A');
+ await query(
+ `UPDATE deck_cards SET definition = jsonb_set(COALESCE(definition, '{}'::jsonb), '{render}', $2::jsonb) WHERE id = $1`,
+ [card.id, JSON.stringify({ status: 'queued', queuedAt: '2026-01-01T00:00:00.000Z' })],
+ );
+ const healed = (await get(`/${deck.id}`)).body.cards.find((c) => c.id === card.id);
+ expect(healed.render).toMatchObject({ status: 'failed', error: 'interrupted by restart' });
+ expect(healed.render.jobId ?? null).toBeNull();
+ });
+
it('adds a sample with an adopted style guide in one write, removes it, and cascades on delete', async () => {
const deck = await create({ name: 'Sample Deck', kind: 'tarot' });
const sample = { id: 'deck-sample-1', title: 'Poster', prompt: 'a lighthouse poster', imageRef: 'upload-1.png' };
diff --git a/server/routes/musicVideo.js b/server/routes/musicVideo.js
index 060969876b..7f3485633d 100644
--- a/server/routes/musicVideo.js
+++ b/server/routes/musicVideo.js
@@ -253,14 +253,21 @@ const projectUpdateSchema = musicVideoProjectUpdateSchema.extend(recordRenderPin
// Approvals decide stage completion and only the server computes them (#10136), so the read
// response carries `productionReadiness` — stages derive from one record, never "not done" while a
// second request is in flight. GET /:id/production-review still serves the full review payload.
-const presentProjectForRead = (project) => {
- const presented = presentProjectAutonomousRun(presentProjectCastAndSets(presentMusicVideoDependencies(project)));
+// A restart archives a thumbnail paint without clearing `pending`. Heal on read,
+// and only when one is present, so a project list does not load the image stack.
+async function presentProjectForRead(project) {
+ let current = project;
+ if (project?.publishKit?.youtubeThumbnail?.pending) {
+ const { reconcileInterruptedYoutubeThumbnail } = await import('../services/musicVideo/youtubeThumbnail.js');
+ current = await reconcileInterruptedYoutubeThumbnail(project);
+ }
+ const presented = presentProjectAutonomousRun(presentProjectCastAndSets(presentMusicVideoDependencies(current)));
return { ...presented, productionReadiness: productionReadiness(presented) };
-};
+}
router.get('/', asyncHandler(async (req, res) => {
const query = validateRequest(musicVideoProjectListQuerySchema, req.query);
- const projects = (await listProjects()).map(presentProjectForRead);
+ const projects = await Promise.all((await listProjects()).map((project) => presentProjectForRead(project)));
if (query.summary === '1' || query.summary === 'true') {
const summaries = projects.sort(compareMusicVideoProjectsNewestFirst)
.map((project) => summarizeMusicVideoProject(project, project.productionReadiness));
@@ -301,7 +308,7 @@ router.get('/:id', asyncHandler(async (req, res) => {
const p = await requireProject(req.params.id);
// Transient (never persisted): lets a reloaded page reattach to a running publishing-kit build.
const activePublishKitBuild = getActivePublishKitBuild(p.id);
- const presented = presentProjectForRead(p);
+ const presented = await presentProjectForRead(p);
res.json(activePublishKitBuild ? { ...presented, activePublishKitBuild } : presented);
}));
@@ -369,7 +376,7 @@ router.put('/:id/finished-outside', asyncHandler(async (req, res) => {
}
const finishedOutside = finished ? { markedAt: new Date().toISOString(), ...(note ? { note } : {}) } : null;
await updateProject(req.params.id, { finishedOutside });
- res.json(presentProjectForRead(await getProject(req.params.id)));
+ res.json(await presentProjectForRead(await getProject(req.params.id)));
}));
// Preview is read-only; Apply rechecks audio and the serialized project basis.
diff --git a/server/services/decks.js b/server/services/decks.js
index 65971babed..2e43120431 100644
--- a/server/services/decks.js
+++ b/server/services/decks.js
@@ -122,6 +122,57 @@ const loadCards = async (client, deckId) => {
return rows.map(projectCard);
};
+const INTERRUPTED_RENDER_ERROR = 'interrupted by restart';
+
+// Same read-time projection as a mood-board note: a queued card whose job is
+// gone fails. A missing job id is not a live claim (deck stamps always carry
+// one). The wire strips `render`, so this must not move either clock.
+function projectInterruptedCardRenders(cards, isLive = () => false) {
+ const updates = [];
+ let changed = false;
+ const next = (Array.isArray(cards) ? cards : []).map((card) => {
+ const render = card?.render;
+ if (!render || render.status !== 'queued') return card;
+ if (render.jobId && isLive(render.jobId)) return card;
+ changed = true;
+ updates.push({ id: card.id, jobId: render.jobId || '' });
+ return { ...card, render: { ...render, status: 'failed', error: INTERRUPTED_RENDER_ERROR } };
+ });
+ return { cards: changed ? next : cards, changed, updates };
+}
+
+async function reconcileInterruptedCardRenders(deckId, cards) {
+ if (!cards.some((card) => card.render?.status === 'queued')) return null;
+ const { getJob } = await import('./mediaJobQueue/index.js');
+ const isLive = (jobId) => {
+ const status = getJob(jobId)?.status;
+ return status === 'queued' || status === 'running';
+ };
+ const projected = projectInterruptedCardRenders(cards, isLive);
+ if (!projected.changed) return null;
+ const stampedAt = new Date().toISOString();
+ let wrote = 0;
+ await queueRecordWrite(deckId, () => withTransaction(async (client) => {
+ for (const card of projected.updates) {
+ const { rowCount } = await client.query(
+ `UPDATE deck_cards
+ SET definition = jsonb_set(
+ definition,
+ '{render}',
+ (definition->'render') || jsonb_build_object('status', 'failed', 'error', $4::text, 'updatedAt', $5::text)
+ )
+ WHERE deck_id = $1 AND id = $2
+ AND definition->'render'->>'status' = 'queued'
+ AND COALESCE(definition->'render'->>'jobId', '') = $3`,
+ [deckId, card.id, card.jobId, INTERRUPTED_RENDER_ERROR, stampedAt],
+ );
+ wrote += rowCount;
+ }
+ }));
+ if (wrote) console.warn(`⚠️ Deck ${deckId}: a card render was interrupted by restart`);
+ return loadCards({ query }, deckId);
+}
+
export async function listDecks({ includeDeleted = false } = {}) {
await ensureSchema();
const { rows } = await query(
@@ -142,11 +193,17 @@ export async function listDecks({ includeDeleted = false } = {}) {
}));
}
-export async function getDeck(id, { includeDeleted = false } = {}) {
+export async function getDeck(id, { includeDeleted = false, reconcileRenders = true } = {}) {
await ensureSchema();
const { rows } = await query('SELECT * FROM decks WHERE id = $1', [id]);
if (!rows[0] || (!includeDeleted && rows[0].deleted === true)) throw notFound();
- const cards = await loadCards({ query }, id);
+ let cards = await loadCards({ query }, id);
+ // Callers already inside this deck's write tail must pass reconcileRenders
+ // false. The tail is not re-entrant, and a nested heal would deadlock.
+ if (reconcileRenders) {
+ const reconciled = await reconcileInterruptedCardRenders(id, cards);
+ if (reconciled) cards = reconciled;
+ }
return { ...projectDeck(rows[0]), cards, completion: deckCompletion(cards) };
}
@@ -474,7 +531,7 @@ export function removeSample(id, sampleId) {
* merge would treat an existing deck as new and skip the LWW compare entirely.
*/
export async function getDeckForSync(id) {
- return getDeck(id, { includeDeleted: true }).catch((error) => {
+ return getDeck(id, { includeDeleted: true, reconcileRenders: false }).catch((error) => {
if (error?.code === 'DECK_NOT_FOUND') return null;
throw error;
});
@@ -699,6 +756,6 @@ export async function restoreDeck(id, patch = {}) {
],
);
emitRecordUpdated(DECK_KIND, id);
- return getDeck(id);
+ return getDeck(id, { reconcileRenders: false });
});
}
diff --git a/server/services/moodBoard/db.js b/server/services/moodBoard/db.js
index a3be690f30..98630537d2 100644
--- a/server/services/moodBoard/db.js
+++ b/server/services/moodBoard/db.js
@@ -34,6 +34,7 @@ import {
claimItemRender,
settleItemRender,
applyRenderedItem,
+ projectInterruptedNoteRenders,
mergeBoardRecord,
applyPinterestLink,
healPinterestFeedRecord,
@@ -264,6 +265,24 @@ export async function updateBoardItem(id, itemId, patch) {
// Text note render lifecycle (#10531) — see the logic.js transforms. The two
// settle steps skip the write when the note moved on (stale job).
+// Re-check liveness under the row lock. A pre-lock snapshot can mark a render
+// that was claimed while this read was in flight. Returns null when the board
+// was deleted between the read and the lock.
+export async function healInterruptedNoteRenders(id, { isLive = () => false, now } = {}) {
+ try {
+ let changed = false;
+ const locked = await withLockedBoard(id, (board) => {
+ const projected = projectInterruptedNoteRenders(board, { isLive, ...(now ? { now } : {}) });
+ changed = projected.changed;
+ return { board: projected.board, skipPersist: !projected.changed };
+ });
+ return { board: locked.board, changed };
+ } catch (err) {
+ if (err?.code === 'NOT_FOUND') return null;
+ throw err;
+ }
+}
+
export async function claimBoardItemRender(id, itemId, opts) {
const { result } = await withLockedBoard(id, (b) => {
const { board, item } = claimItemRender(b, itemId, opts);
diff --git a/server/services/moodBoard/index.js b/server/services/moodBoard/index.js
index 4df1910a19..0f5f793351 100644
--- a/server/services/moodBoard/index.js
+++ b/server/services/moodBoard/index.js
@@ -20,18 +20,42 @@ import * as store from './db.js';
import { localizeImageUrl } from './localize.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
-// Read paths + federation entry points pass straight through to the store. The
+// Read paths + federation entry points pass straight through to the store, except
+// getBoard, which projects a note render a restart interrupted. The
// asset-manifest filename resolver lives in logic.js (pure) and is re-exported
// here so peerSync imports a single module (mirrors creativeDirector/local.js
// re-exporting startingImageFilename).
export {
listBoards,
- getBoard,
listBoardIds,
listBoardNames,
mergeBoardsFromSync,
pruneTombstonedBoards,
} from './db.js';
+
+function hasQueuedTextNote(board) {
+ return Array.isArray(board?.items) && board.items.some((item) => item?.type === 'text' && item.render?.status === 'queued');
+}
+
+// Sync loads with includeDeleted stay a pure read: healing there would emit
+// from inside a peer push. Liveness matches claimItemRender (queued or running).
+export async function getBoard(id, options = {}) {
+ const board = await store.getBoard(id, options);
+ if (!board || options.includeDeleted || !hasQueuedTextNote(board)) return board;
+ const { getJob } = await import('../mediaJobQueue/index.js');
+ const healed = await store.healInterruptedNoteRenders(id, {
+ isLive: (jobId) => {
+ const status = getJob(jobId)?.status;
+ return status === 'queued' || status === 'running';
+ },
+ });
+ if (!healed) return null;
+ if (healed.changed) {
+ emitRecordUpdated('moodBoard', id);
+ console.warn(`⚠️ Mood board ${id}: a note render was interrupted by restart`);
+ }
+ return healed.board;
+}
export { imageUrlToAppAsset } from './logic.js';
// Pinterest importer (link + manual "Sync now"). Lives in its own module
diff --git a/server/services/moodBoard/logic.js b/server/services/moodBoard/logic.js
index 835336504a..3a1755a22f 100644
--- a/server/services/moodBoard/logic.js
+++ b/server/services/moodBoard/logic.js
@@ -306,6 +306,29 @@ function replaceItem(board, items, idx, item) {
// earlier render's job is still queued/running; a dead one may be replaced. A
// claim that never got its job id (the process died mid-enqueue) expires.
const RENDER_CLAIM_TTL_MS = 10 * 60 * 1000;
+const INTERRUPTED_RENDER_ERROR = 'interrupted by restart';
+
+// A restart archives the media job without telling the note, and replaying that
+// failure at boot would run every other terminal listener. Project the same
+// outcome when the board is read: a queued note whose job is gone fails, and a
+// claim that never received a job id keeps the claim TTL above. `updatedAt`
+// is wall-clock so the healed record wins LWW; `now` is only the TTL clock.
+export function projectInterruptedNoteRenders(board, { isLive = () => false, now = nowIso() } = {}) {
+ if (!board || !Array.isArray(board.items)) return { board, changed: false };
+ let changed = false;
+ const items = board.items.map((item) => {
+ const render = item?.render;
+ if (!item || item.type !== 'text' || render?.status !== 'queued') return item;
+ const live = render.jobId
+ ? Boolean(isLive(render.jobId))
+ : Date.parse(now) - Date.parse(render.queuedAt) < RENDER_CLAIM_TTL_MS;
+ if (live) return item;
+ changed = true;
+ return { ...item, render: { ...render, status: 'failed', error: INTERRUPTED_RENDER_ERROR } };
+ });
+ if (!changed) return { board, changed: false };
+ return { board: { ...board, items, updatedAt: nowIso() }, changed: true };
+}
export function claimItemRender(board, itemId, { isLive = () => false, now = nowIso(), options } = {}) {
const { items, idx } = findItemIndex(board, itemId);
diff --git a/server/services/moodBoard/logic.test.js b/server/services/moodBoard/logic.test.js
index 52e0e5739a..3e95003e00 100644
--- a/server/services/moodBoard/logic.test.js
+++ b/server/services/moodBoard/logic.test.js
@@ -26,6 +26,7 @@ import {
claimItemRender,
settleItemRender,
applyRenderedItem,
+ projectInterruptedNoteRenders,
} from './logic.js';
describe('buildBoardRecord', () => {
@@ -582,6 +583,35 @@ describe('text note render lifecycle (#10531)', () => {
expect(() => claimItemRender(board(), 'i0')).toThrow(expect.objectContaining({ code: 'MOOD_BOARD_ITEM_NOT_TEXT' }));
expect(() => claimItemRender(board(), 'missing')).toThrow(expect.objectContaining({ code: 'NOT_FOUND' }));
});
+
+ it('projects a queued note whose job is gone as interrupted, and leaves a live one', () => {
+ const queued = settleItemRender(claimItemRender(board(), 'n1').board, 'n1', { jobId: 'job-dead', status: 'queued' }).board;
+ queued.updatedAt = '2020-01-01T00:00:00.000Z';
+ const dead = projectInterruptedNoteRenders(queued, { isLive: () => false });
+ expect(dead.changed).toBe(true);
+ expect(dead.board.items.find((it) => it.id === 'n1').render).toMatchObject({
+ status: 'failed', error: 'interrupted by restart', jobId: 'job-dead',
+ });
+ expect(dead.board.updatedAt).not.toBe(queued.updatedAt);
+ const live = projectInterruptedNoteRenders(queued, { isLive: (jobId) => jobId === 'job-dead' });
+ expect(live).toEqual({ board: queued, changed: false });
+ });
+
+ it('keeps a claim with no job id until the claim TTL, then fails it', () => {
+ const claimed = claimItemRender(board(), 'n1', { now: '2026-01-01T00:00:00.000Z' }).board;
+ expect(projectInterruptedNoteRenders(claimed, { now: '2026-01-01T00:09:00.000Z' }).changed).toBe(false);
+ const expired = projectInterruptedNoteRenders(claimed, { now: '2026-01-01T00:11:00.000Z' });
+ expect(expired.board.items.find((it) => it.id === 'n1').render).toMatchObject({
+ status: 'failed', error: 'interrupted by restart', jobId: null,
+ });
+ });
+
+ it('ignores an image item and a note that is not queued', () => {
+ const image = { id: 'i0', type: 'image', mediaKey: 'image:a.png', render: { status: 'queued', jobId: 'job-x' } };
+ const failed = { id: 'n1', type: 'text', text: 'note', render: { status: 'failed', jobId: 'job-y', error: 'boom' } };
+ const board = { items: [image, failed] };
+ expect(projectInterruptedNoteRenders(board, { isLive: () => false })).toEqual({ board, changed: false });
+ });
});
diff --git a/server/services/musicVideo/projects.js b/server/services/musicVideo/projects.js
index 033cb5a1b4..c58a675cda 100644
--- a/server/services/musicVideo/projects.js
+++ b/server/services/musicVideo/projects.js
@@ -531,7 +531,8 @@ async function mutateProjectRecordAdmitted(id, transform, { source = null, runId
let dropped = null;
const write = mutate || (await selectBackend()).mutateProjectRecord;
const outcome = await write(id, async (current) => {
- const outcome = transform(current);
+ // A transform may be async: the thumbnail heal re-checks job liveness under the lock.
+ const outcome = await transform(current);
if (source) {
const guarded = preserveAutomatedShotLocks(current, outcome.project, source, { runId });
outcome.project = guarded.project;
diff --git a/server/services/musicVideo/projectsDB.js b/server/services/musicVideo/projectsDB.js
index 342bac2899..dc022b4e78 100644
--- a/server/services/musicVideo/projectsDB.js
+++ b/server/services/musicVideo/projectsDB.js
@@ -103,7 +103,7 @@ async function mutateLocked(id, transform) {
async function mutateProjectRecordUnqueued(id, transform) {
const { project, result } = await withLockedProject(id, async (current) => {
const outcome = await transform(current);
- return { project: outcome.project, result: outcome };
+ return { project: outcome.project, result: outcome, skipPersist: outcome.skipPersist === true };
});
return { ...result, project };
}
diff --git a/server/services/musicVideo/projectsFile.js b/server/services/musicVideo/projectsFile.js
index 25ec9f444d..6887dbadd9 100644
--- a/server/services/musicVideo/projectsFile.js
+++ b/server/services/musicVideo/projectsFile.js
@@ -106,6 +106,7 @@ async function reorderProjectScenesUnqueued(id, orderedIds) {
async function mutateProject(id, transform) {
const { all, idx } = await loadAllAndIndex(id);
const outcome = await transform(all[idx]);
+ if (outcome.skipPersist) return outcome;
all[idx] = outcome.project;
await saveAll(all);
return outcome;
diff --git a/server/services/musicVideo/youtubeThumbnail.js b/server/services/musicVideo/youtubeThumbnail.js
index b59052fe27..ec54228250 100644
--- a/server/services/musicVideo/youtubeThumbnail.js
+++ b/server/services/musicVideo/youtubeThumbnail.js
@@ -206,6 +206,43 @@ async function pendingIsLive(pending) {
return Date.now() - Date.parse(pending.requestedAt || 0) < RESERVATION_TTL_MS;
}
+const INTERRUPTED_PAINT_ERROR = 'interrupted by restart';
+
+// A restart archives the paint without clearing `pending`, and replaying the
+// job failure at boot would run every other terminal listener. The publish-kit
+// read projects that outcome instead. Not safe from getProject: the completion
+// hook treats a missing pending as "not this request".
+export async function reconcileInterruptedYoutubeThumbnail(project) {
+ if (!project?.id) return project;
+ const thumb = projectYoutubeThumbnail(project);
+ if (!thumb.pending || await pendingIsLive(thumb.pending)) return project;
+ const seenRequestId = thumb.pending.requestId || null;
+ const seenJobId = thumb.pending.jobId || null;
+ const out = await mutateProjectRecord(project.id, async (current) => {
+ const currentThumb = projectYoutubeThumbnail(current);
+ const pending = currentThumb.pending;
+ const same = pending
+ && (pending.requestId || null) === seenRequestId
+ && (pending.jobId || null) === seenJobId;
+ if (!same || await pendingIsLive(pending)) return { project: current, skipPersist: true, healed: false };
+ const kit = projectPublishKit(current);
+ return {
+ project: {
+ ...current,
+ publishKit: {
+ ...kit,
+ youtubeThumbnail: { ...currentThumb, pending: null, lastError: INTERRUPTED_PAINT_ERROR },
+ },
+ },
+ healed: true,
+ };
+ });
+ if (!out.healed) return out.project;
+ console.warn(`⚠️ Music Video YouTube thumbnail ${String(project.id).slice(3, 11)}: painting interrupted by restart`);
+ musicVideoEvents.emit('cover-art', { projectId: project.id, project: out.project });
+ return out.project;
+}
+
const writeThumb = (projectId, update) => mutateProjectRecord(projectId, (current) => {
const kit = projectPublishKit(current);
const thumb = projectYoutubeThumbnail(current);
diff --git a/server/services/musicVideo/youtubeThumbnail.test.js b/server/services/musicVideo/youtubeThumbnail.test.js
index 63a3835808..29377161d2 100644
--- a/server/services/musicVideo/youtubeThumbnail.test.js
+++ b/server/services/musicVideo/youtubeThumbnail.test.js
@@ -144,6 +144,56 @@ describe('painting the title with the image model', () => {
expect(Math.max(...stats.channels.map((c) => c.stdev))).toBeLessThan(2);
});
+ it('clears a paint a restart interrupted and still accepts the next one', async () => {
+ const { id, frame } = await projectWithFrame();
+ await projects.mutateProjectRecord(id, (current) => ({
+ project: {
+ ...current,
+ publishKit: {
+ ...current.publishKit,
+ youtubeThumbnail: {
+ pending: { requestId: 'req-dead', jobId: 'job-dead', mode: 'codex', requestedAt: '2020-01-01T00:00:00.000Z' },
+ lastError: null,
+ },
+ },
+ },
+ }));
+ thumbs.__setYoutubeThumbnailDepsForTests({ jobStatus: async () => 'failed' });
+ const healed = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));
+ expect(healed.publishKit.youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });
+ expect((await kitOf(id)).youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });
+
+ const enqueue = vi.fn(async () => ({ jobId: 'job-next' }));
+ thumbs.__setYoutubeThumbnailDepsForTests({
+ getSettings: async () => ({}), defaultRoute: async () => ({ mode: 'codex', model: null }),
+ imageParams: async (_s, _r, common) => common, enqueue, jobStatus: async () => 'queued',
+ });
+ await thumbs.paintYoutubeThumbnail(id, { source: { kind: 'thumbnail', filename: frame }, title: 'Example Song' });
+ expect((await kitOf(id)).youtubeThumbnail.pending).toMatchObject({ jobId: 'job-next' });
+ thumbs.__setYoutubeThumbnailDepsForTests({});
+ });
+
+ it('leaves a live paint and a fresh reservation with no job id', async () => {
+ const { id } = await projectWithFrame();
+ const stamp = async (pending) => projects.mutateProjectRecord(id, (current) => ({
+ project: { ...current, publishKit: { ...current.publishKit, youtubeThumbnail: { pending, lastError: null } } },
+ }));
+ thumbs.__setYoutubeThumbnailDepsForTests({ jobStatus: async () => 'running' });
+ await stamp({ requestId: 'req-live', jobId: 'job-live', mode: 'codex', requestedAt: '2020-01-01T00:00:00.000Z' });
+ const live = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));
+ expect(live.publishKit.youtubeThumbnail.pending).toMatchObject({ jobId: 'job-live' });
+ expect((await kitOf(id)).youtubeThumbnail.pending.jobId).toBe('job-live');
+
+ await stamp({ requestId: 'req-reserved', jobId: null, mode: 'codex', requestedAt: new Date().toISOString() });
+ const reserved = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));
+ expect(reserved.publishKit.youtubeThumbnail.pending).toMatchObject({ requestId: 'req-reserved', jobId: null });
+
+ await stamp({ requestId: 'req-stale', jobId: null, mode: 'codex', requestedAt: new Date(Date.now() - 3 * 60 * 1000).toISOString() });
+ const stale = await thumbs.reconcileInterruptedYoutubeThumbnail(await projects.getProject(id));
+ expect(stale.publishKit.youtubeThumbnail).toMatchObject({ pending: null, lastError: 'interrupted by restart' });
+ thumbs.__setYoutubeThumbnailDepsForTests({});
+ });
+
it('records why a paint failed', async () => {
const { id, frame } = await projectWithFrame();
const enqueue = vi.fn(async () => ({ jobId: 'job-failing' }));