Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: cancel a local image or video render that is still preparing (#11773)
Adam Eivyb2d2b7306b53
diff --git a/server/services/imageGen/index.js b/server/services/imageGen/index.jsindex bb7aaa451a..c877f8eb3c 100644--- a/server/services/imageGen/index.js+++ b/server/services/imageGen/index.js@@ -282,7 +282,7 @@ export const cancel = () => {const grokCancelled = grok.cancelAll();const agyCancelled = agy.cancelAll();const falCancelled = fal.cancelAll();- return localCancelled || codexCancelled || grokCancelled || agyCancelled || falCancelled;+ return Boolean(localCancelled || codexCancelled || grokCancelled || agyCancelled || falCancelled);};// Exact-ID cancel (#9932). The explicit-target counterpart to cancel(): stopsdiff --git a/server/services/imageGen/local.js b/server/services/imageGen/local.jsindex 093a7e4c64..b630dfa1f3 100644--- a/server/services/imageGen/local.js+++ b/server/services/imageGen/local.js@@ -69,8 +69,17 @@ export const getActiveJob = () => activeJob;export const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);+// The job between stepwise-dir creation and a wired child. cancel() has no+// process to signal in that window (heavy-claim wait, model unload), so it flags+// the job and the pre-spawn gate aborts it.+let preparing = null;+export const cancel = () => {- if (!activeProcess) return false;+ if (!activeProcess) {+ if (!preparing) return false;+ preparing.cancelRequested = true;+ return 'pending';+ }const proc = activeProcess;// KEEP activeProcess + activeJob set until proc.on('close') clears them.// Otherwise BUSY immediately allows a new generation while the SIGTERM'd@@ -711,6 +720,8 @@ export async function generateImage({ pythonPath, prompt = '', negativePrompt =// Per-job stepwise output dir under the OS temp dir. mflux writes one PNG// per inference step here; we watch and stream the latest as `currentImage`.const stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-stepwise-'));+ const preparation = { cancelRequested: false };+ preparing = preparation;let heavyClaim = null;const releaseHeavyClaim = () => {@@ -748,15 +759,23 @@ export async function generateImage({ pythonPath, prompt = '', negativePrompt =}if (memoryReport.unloaded.length) console.log(`🧹 Image generation [${jobId.slice(0, 8)}] freed ${memoryReport.unloaded.length} resident model(s)`);+ const childEnv = await hfChildEnv();+ // Last spawn gate: a cancel() during the awaits above found no child to signal.+ if (preparation.cancelRequested) {+ throw new ServerError('Canceled before the render started', { status: 409, code: 'IMAGE_GEN_CANCELED' });+ }+console.log(`🎨 Generating image [${jobId.slice(0, 8)}] local: ${modelId} ${width}x${height} steps=${actualSteps} references=${validReferenceImagePaths.length}`);imageGenEvents.emit('started', { generationId: jobId, totalSteps: actualSteps });activeJob = { ...meta, generationId: jobId, totalSteps: actualSteps, step: 0, progress: 0, currentImage: null, mode: IMAGE_GEN_MODE.LOCAL };- proc = spawn(bin, args, safeChildProcessOptions({ env: await hfChildEnv(), stdio: ['ignore', 'pipe', 'pipe'] }));+ proc = spawn(bin, args, safeChildProcessOptions({ env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] }));activeProcess = proc;+ preparing = null;await heavyClaim.handoffTo?.(proc.pid);claimHandedOff = true;} catch (err) {+ if (preparing === preparation) preparing = null;// Nothing is wired to this job yet, so it can never report its own terminal// event — unwind it exactly the way the busy branch above does, or a refusal// strands a `running` entry and its stepwise dir every time.diff --git a/server/services/imageGen/local.test.js b/server/services/imageGen/local.test.jsindex d4a77ae6b1..c2f55aa4af 100644--- a/server/services/imageGen/local.test.js+++ b/server/services/imageGen/local.test.js@@ -20,7 +20,16 @@ const mockSpawn = vi.fn();vi.mock('../../lib/childProcess.js', async (original) => ({ ...await original(), spawn: (...args) => mockSpawn(...args) }));const releaseClaim = vi.fn(async () => {});vi.mock('../../lib/heavyJobClaim.js', () => ({ claimHeavyLocalJob: async () => ({ ok: true, release: releaseClaim }) }));-vi.mock('../localMemory.js', () => ({ gpuBlockersMessage: vi.fn(), prepareLocalMemory: async () => ({ blockers: [], unloaded: [] }) }));+// A test sets `hold` to a pending promise to park generateImage inside memory preparation.+const memoryGate = vi.hoisted(() => ({ hold: undefined, entered: undefined }));+vi.mock('../localMemory.js', () => ({+ gpuBlockersMessage: vi.fn(),+ prepareLocalMemory: async () => {+ memoryGate.entered?.();+ await memoryGate.hold;+ return { blockers: [], unloaded: [] };+ },+}));vi.mock('../hfToken.js', () => ({ hfChildEnv: async () => ({}) }));vi.mock('../../lib/fileUtils.js', async (original) => ({ ...await original(), ensureDir: vi.fn() }));@@ -1089,6 +1098,39 @@ describe('imageGen local.generateImage runtime pre-flight', () => {claimSpy.mockRestore();rmSpy.mockRestore();});++ it('cancel() during memory preparation answers pending and aborts before spawn', async () => {+ mockResolveFlux2Python.mockReturnValue('/fake/venv-flux2/bin/python3');+ mockIsFlux2VenvHealthy.mockResolvedValue(true);+ mockSpawn.mockClear();+ releaseClaim.mockClear();+ const { cancel } = await import('./local.js');+ const { imageGenEvents } = await import('../imageGenEvents.js');+ const terminal = vi.fn();+ for (const event of ['started', 'completed', 'failed']) imageGenEvents.on(event, terminal);+ let release;+ memoryGate.hold = new Promise((resolve) => { release = resolve; });+ const entered = new Promise((resolve) => { memoryGate.entered = resolve; });+ try {+ const rendering = generateImage({ modelId: 'qwen-image-2.1', prompt: 'a fox', jobId: 'test-cancel-while-preparing' });+ const refused = expect(rendering).rejects.toMatchObject({ code: 'IMAGE_GEN_CANCELED' });+ await entered;+ expect(cancel()).toBe('pending');+ release();+ await refused;+ } finally {+ memoryGate.hold = undefined;+ memoryGate.entered = undefined;+ for (const event of ['started', 'completed', 'failed']) imageGenEvents.off(event, terminal);+ }+ expect(mockSpawn).not.toHaveBeenCalled();+ expect(releaseClaim).toHaveBeenCalledTimes(1);+ // The rejection is the one terminal signal (the queue maps it to canceled);+ // no render ever started, so nothing else may be emitted.+ expect(terminal).not.toHaveBeenCalled();+ // Once the job unwound there is nothing left to cancel.+ expect(cancel()).toBe(false);+ });});// Regression: SSE had the Python diagnosis, but the queue and status snapshotdiff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.jsindex cbebb7b086..ffc48e06c2 100644--- a/server/services/mediaJobQueue/index.js+++ b/server/services/mediaJobQueue/index.js@@ -1818,6 +1818,9 @@ export async function cancelJob(jobId) {// cancellation rather than silently resurrecting the remote render.await persist();}+ // Provider contract: `true` = child signalled, `'pending'` = the provider owns+ // the job but is still preparing (it aborts at its spawn gate and emits+ // `failed`), `false` = nothing cancelable, i.e. finalization already began.if (mod?.cancel && mod.cancel(jobId) === false&& (runningJob.providerInvoked || isRemoteMediaJob(runningJob))) {// A provider may have crossed its durable finalization boundary beforediff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.jsindex 80508fa510..3fa271df06 100644--- a/server/services/mediaJobQueue/index.test.js+++ b/server/services/mediaJobQueue/index.test.js@@ -1080,7 +1080,7 @@ describe('mediaJobQueue', () => {expect(mediaJobQueue.getJob(job.jobId).error).toBe('History write failed');});- it.each([true, undefined])('preserves accepted or legacy cancellation results (%s)', async (accepted) => {+ it.each([true, undefined, 'pending'])('preserves accepted, pending or legacy cancellation results (%s)', async (accepted) => {const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { mode: 'grok', prompt: 'cancel render', videoProduction: { projectId: 'example-project', attemptId: 'example-attempt' } } });await waitFor(() => stubs.generateVideoGrok.mock.calls.length === 1);stubs.cancelVideo.mockImplementationOnce(() => {diff --git a/server/services/videoGen/generateVideo.js b/server/services/videoGen/generateVideo.jsindex 74d95f87c5..a6163cb82c 100644--- a/server/services/videoGen/generateVideo.js+++ b/server/services/videoGen/generateVideo.js@@ -55,7 +55,7 @@ import {import { getSettings } from '../settings.js';import { loadHistory, saveHistory, mutateVideoHistory, getHistoryItem } from './history.js';import { VIDEO_MODE_GATED_RUNTIMES } from './modeContract.js';-import { videoJobState } from './jobState.js';+import { videoJobState, beginPreparation } from './jobState.js';import {assertRenderModeContract,buildArgs,@@ -315,7 +315,7 @@ async function resolveRequestedDraftDecoder({ model, draftDecode, jobId }) {return draftDecoder;}-export async function generateVideo({ pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {+async function renderLocalVideo(preparation, { pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {const sourceAudioProvenance = audioConditioning == null ? null : federatedSourceAudioSchema.parse(audioConditioning);uploadedTempPaths = Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [];if (!prompt?.trim()) throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });@@ -722,6 +722,8 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', mconst { ensureMiniMaxH3Weights } = await import('./ensureWeights.js');await ensureMiniMaxH3Weights(model);}+ // Weights can take a long first-run download; honor a cancel that landed during it.+ preparation.throwIfCanceled();} catch (err) {job.status = 'error';const reason = err.message || 'Failed to prepare video generation';@@ -742,6 +744,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', mawait spawnAndWatchVideo({jobId,+ preparation,batch,// Keeps this render's ETA samples in its own cost bucket — a speed profile// changes the slope, not just the step count.@@ -782,3 +785,14 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', mreturn { jobId, generationId: jobId, filename, mode: 'local', model: modelId };}++// Pre-spawn work (weight download, memory preparation) runs with no child to+// signal, so cancel() answers 'pending' and the spawn gates abort the render.+export async function generateVideo(params) {+ const preparation = beginPreparation();+ try {+ return await renderLocalVideo(preparation, params);+ } finally {+ preparation.end();+ }+}diff --git a/server/services/videoGen/jobState.js b/server/services/videoGen/jobState.jsindex 660aca4ef7..30691109fa 100644--- a/server/services/videoGen/jobState.js+++ b/server/services/videoGen/jobState.js@@ -6,6 +6,7 @@*/import { killWithEscalation } from '../../lib/killWithEscalation.js';+import { ServerError } from '../../lib/errorHandler.js';import { attachSseClient as attachSse } from '../../lib/sseUtils.js';export const videoJobState = {@@ -13,6 +14,29 @@ export const videoJobState = {activeProcess: null,cancelEpoch: 0,activeChain: null,+ // Renders between generateVideo() entry and a wired child. A cancel() in that+ // window has no process to kill but must still stop the render from spawning.+ preparing: 0,+};++// Brackets one render's pre-spawn phase. `throwIfCanceled()` is the spawn gate:+// any cancel() since this call (epoch bump) aborts the render before it starts.+export const beginPreparation = () => {+ const epoch = videoJobState.cancelEpoch;+ videoJobState.preparing += 1;+ let ended = false;+ const wasCanceled = () => videoJobState.cancelEpoch !== epoch;+ return {+ wasCanceled,+ throwIfCanceled: () => {+ if (wasCanceled()) throw new ServerError('Canceled before the render started', { status: 409, code: 'VIDEO_GEN_CANCELED' });+ },+ end: () => {+ if (ended) return;+ ended = true;+ videoJobState.preparing -= 1;+ },+ };};export const attachSseClient = (jobId, res) => attachSse(videoJobState.jobs, jobId, res);@@ -22,7 +46,12 @@ export const cancel = () => {// Flag a chain so its orchestrator stops between chunks, then terminate the// child currently holding the local GPU lane when one exists.if (videoJobState.activeChain) videoJobState.activeChain.stopped = true;- if (!videoJobState.activeProcess) return !!videoJobState.activeChain;+ if (!videoJobState.activeProcess) {+ if (videoJobState.activeChain) return true;+ // 'pending': the job is ours and still preparing; the render will abort+ // itself at its next spawn gate. `false` stays "nothing cancelable here".+ return videoJobState.preparing > 0 ? 'pending' : false;+ }const proc = videoJobState.activeProcess;// Keep the handle until close so a new render cannot replace it before the// escalation timer has had a chance to SIGKILL a stubborn child.diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex 665c54ffa4..476312646d 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -3191,6 +3191,42 @@ describe('generateVideo — MiniMax H3 MLX contract', () => {))).toEqual(expectedAnchors);});+ it('cancel() while first-run weights are still preparing answers pending and never spawns', async () => {+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ const { cancel } = await import('./local.js');+ vi.mocked(spawnDetached).mockClear();+ let release;+ const gate = new Promise((resolve) => { release = resolve; });+ let enteredWeights;+ const entered = new Promise((resolve) => { enteredWeights = resolve; });+ // quant_config.json is probed only by ensureMiniMaxH3Weights.+ mockFindCachedRepoFile.mockImplementation(async (_repo, filename) => {+ if (filename === 'quant_config.json') {+ enteredWeights();+ await gate;+ }+ return join('/mock/hf/snap', filename);+ });+ try {+ const rendering = generateVideo({+ jobId: 'h3-cancel-while-preparing',+ modelId: 'minimax_h3_8bit',+ prompt: 'a fox watches the rain',+ width: 512, height: 320, numFrames: 141, fps: 24, mode: 'text',+ });+ const refused = expect(rendering).rejects.toMatchObject({ code: 'VIDEO_GEN_CANCELED' });+ await entered;+ expect(cancel()).toBe('pending');+ release();+ await refused;+ } finally {+ mockFindCachedRepoFile.mockImplementation(async (_repo, filename) => join('/mock/hf/snap', filename));+ }+ expect(spawnDetached).not.toHaveBeenCalled();+ // Preparation ended with the render, so a later cancel has nothing to claim.+ expect(cancel()).toBe(false);+ });+it('uses the pinned cache-only helper, locked sampler and credential-free environment', async () => {const { spawnDetached } = await import('../../lib/detachedSpawn.js');const { hfChildEnv } = await import('../hfToken.js');@@ -5468,6 +5504,33 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {expect(failures).toHaveLength(1);});+ // Before activeProcess is set, cancel() has no child to signal; the epoch+ // taken at render entry is what delivers it once the spawn resolves.+ it('stops the first child when a cancel lands during its spawn await', async () => {+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ const { cancel } = await import('./local.js');+ const spawnMock = vi.mocked(spawnDetached);+ const first = makeDrivenProc(113);+ spawnMock.mockClear();+ spawnMock.mockImplementationOnce(async () => {+ expect(cancel()).toBe('pending');+ return first.proc;+ });+ await generateVideo({+ jobId: 'cancel-during-first-spawn',+ pythonPath: '/usr/bin/python3',+ modelId: 'ltx2_unified',+ prompt: 'a lighthouse in fog',+ width: 512,+ height: 512,+ numFrames: 25,+ fps: 24,+ seed: 987654,+ });++ expect(first.proc.kill).toHaveBeenCalledWith('SIGTERM');+ });+// A replacement child that never gets its close listener can never report a// terminal event, so it must not be left running.it('stops the replacement child when wiring it throws', async () => {diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.jsindex 16bbff3405..e60a611336 100644--- a/server/services/videoGen/spawnWatch.js+++ b/server/services/videoGen/spawnWatch.js@@ -42,7 +42,7 @@ import { isDisplaySleepEnabled, sleepDisplayForVideo, wakeDisplayForVideo } fromimport { loadHistory, mutateVideoHistory } from './history.js';import { estimateRenderMs } from './eta.js';import { spliceExtendSource } from './extendSplice.js';-import { videoJobState } from './jobState.js';+import { videoJobState, cancel as cancelLocalVideo } from './jobState.js';const MODULE_NOT_FOUND_RE = /ModuleNotFoundError: No module named ['"]([^'"]+)['"]/;const STDERR_TAIL_LINES = 40;@@ -58,6 +58,7 @@ export async function spawnAndWatchVideo({jobId,batch = null,speedProfileId = null,+ preparation = null,cleanupTempFiles,stepwiseDir,job,@@ -837,6 +838,8 @@ export async function spawnAndWatchVideo({// still `proc.kill()` it directly by PID on cancel / watchdog. `cleanup: true`// lets the helper drop that scratch dir on every terminal path (close/error)// so it can't accumulate under data/videos.+ // Last spawn gate; the spawn call follows with no intervening await.+ preparation?.throwIfCanceled();firstProc = await spawnDetached(bin, args, {env: childEnv,controlDir: join(PATHS.videos, '.detached', jobId),@@ -854,6 +857,8 @@ export async function spawnAndWatchVideo({// with it.takeEarlyExit = bufferChildExit(firstProc, { bufferOutput: true });videoJobState.activeProcess = firstProc;+ // A cancel() during the spawn await saw no process; deliver it now.+ if (preparation?.wasCanceled()) cancelLocalVideo();await heavyClaim.handoffTo?.(firstProc.pid);} catch (err) {await abandonBeforeWiring(err);