Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: cancel a local image or video render that is still preparing (#11773)

Adam Eivyb2d2b7306b53
diff --git a/server/services/imageGen/index.js b/server/services/imageGen/index.js
index bb7aaa451a..c877f8eb3c 100644
--- a/server/services/imageGen/index.js
+++ b/server/services/imageGen/index.js
@@ -282,7 +282,7 @@ export const cancel = () => {
const grokCancelled = grok.cancelAll();
const agyCancelled = agy.cancelAll();
const falCancelled = fal.cancelAll();
- return localCancelled || codexCancelled || grokCancelled || agyCancelled || falCancelled;
+ return Boolean(localCancelled || codexCancelled || grokCancelled || agyCancelled || falCancelled);
};
// Exact-ID cancel (#9932). The explicit-target counterpart to cancel(): stops
diff --git a/server/services/imageGen/local.js b/server/services/imageGen/local.js
index 093a7e4c64..b630dfa1f3 100644
--- a/server/services/imageGen/local.js
+++ b/server/services/imageGen/local.js
@@ -69,8 +69,17 @@ export const getActiveJob = () => activeJob;
export const attachSseClient = (jobId, res) => attachSse(jobs, jobId, res);
+// The job between stepwise-dir creation and a wired child. cancel() has no
+// process to signal in that window (heavy-claim wait, model unload), so it flags
+// the job and the pre-spawn gate aborts it.
+let preparing = null;
+
export const cancel = () => {
- if (!activeProcess) return false;
+ if (!activeProcess) {
+ if (!preparing) return false;
+ preparing.cancelRequested = true;
+ return 'pending';
+ }
const proc = activeProcess;
// KEEP activeProcess + activeJob set until proc.on('close') clears them.
// Otherwise BUSY immediately allows a new generation while the SIGTERM'd
@@ -711,6 +720,8 @@ export async function generateImage({ pythonPath, prompt = '', negativePrompt =
// Per-job stepwise output dir under the OS temp dir. mflux writes one PNG
// per inference step here; we watch and stream the latest as `currentImage`.
const stepwiseDir = await mkdtemp(join(tmpdir(), 'portos-stepwise-'));
+ const preparation = { cancelRequested: false };
+ preparing = preparation;
let heavyClaim = null;
const releaseHeavyClaim = () => {
@@ -748,15 +759,23 @@ export async function generateImage({ pythonPath, prompt = '', negativePrompt =
}
if (memoryReport.unloaded.length) console.log(`🧹 Image generation [${jobId.slice(0, 8)}] freed ${memoryReport.unloaded.length} resident model(s)`);
+ const childEnv = await hfChildEnv();
+ // Last spawn gate: a cancel() during the awaits above found no child to signal.
+ if (preparation.cancelRequested) {
+ throw new ServerError('Canceled before the render started', { status: 409, code: 'IMAGE_GEN_CANCELED' });
+ }
+
console.log(`🎨 Generating image [${jobId.slice(0, 8)}] local: ${modelId} ${width}x${height} steps=${actualSteps} references=${validReferenceImagePaths.length}`);
imageGenEvents.emit('started', { generationId: jobId, totalSteps: actualSteps });
activeJob = { ...meta, generationId: jobId, totalSteps: actualSteps, step: 0, progress: 0, currentImage: null, mode: IMAGE_GEN_MODE.LOCAL };
- proc = spawn(bin, args, safeChildProcessOptions({ env: await hfChildEnv(), stdio: ['ignore', 'pipe', 'pipe'] }));
+ proc = spawn(bin, args, safeChildProcessOptions({ env: childEnv, stdio: ['ignore', 'pipe', 'pipe'] }));
activeProcess = proc;
+ preparing = null;
await heavyClaim.handoffTo?.(proc.pid);
claimHandedOff = true;
} catch (err) {
+ if (preparing === preparation) preparing = null;
// Nothing is wired to this job yet, so it can never report its own terminal
// event — unwind it exactly the way the busy branch above does, or a refusal
// strands a `running` entry and its stepwise dir every time.
diff --git a/server/services/imageGen/local.test.js b/server/services/imageGen/local.test.js
index d4a77ae6b1..c2f55aa4af 100644
--- a/server/services/imageGen/local.test.js
+++ b/server/services/imageGen/local.test.js
@@ -20,7 +20,16 @@ const mockSpawn = vi.fn();
vi.mock('../../lib/childProcess.js', async (original) => ({ ...await original(), spawn: (...args) => mockSpawn(...args) }));
const releaseClaim = vi.fn(async () => {});
vi.mock('../../lib/heavyJobClaim.js', () => ({ claimHeavyLocalJob: async () => ({ ok: true, release: releaseClaim }) }));
-vi.mock('../localMemory.js', () => ({ gpuBlockersMessage: vi.fn(), prepareLocalMemory: async () => ({ blockers: [], unloaded: [] }) }));
+// A test sets `hold` to a pending promise to park generateImage inside memory preparation.
+const memoryGate = vi.hoisted(() => ({ hold: undefined, entered: undefined }));
+vi.mock('../localMemory.js', () => ({
+ gpuBlockersMessage: vi.fn(),
+ prepareLocalMemory: async () => {
+ memoryGate.entered?.();
+ await memoryGate.hold;
+ return { blockers: [], unloaded: [] };
+ },
+}));
vi.mock('../hfToken.js', () => ({ hfChildEnv: async () => ({}) }));
vi.mock('../../lib/fileUtils.js', async (original) => ({ ...await original(), ensureDir: vi.fn() }));
@@ -1089,6 +1098,39 @@ describe('imageGen local.generateImage runtime pre-flight', () => {
claimSpy.mockRestore();
rmSpy.mockRestore();
});
+
+ it('cancel() during memory preparation answers pending and aborts before spawn', async () => {
+ mockResolveFlux2Python.mockReturnValue('/fake/venv-flux2/bin/python3');
+ mockIsFlux2VenvHealthy.mockResolvedValue(true);
+ mockSpawn.mockClear();
+ releaseClaim.mockClear();
+ const { cancel } = await import('./local.js');
+ const { imageGenEvents } = await import('../imageGenEvents.js');
+ const terminal = vi.fn();
+ for (const event of ['started', 'completed', 'failed']) imageGenEvents.on(event, terminal);
+ let release;
+ memoryGate.hold = new Promise((resolve) => { release = resolve; });
+ const entered = new Promise((resolve) => { memoryGate.entered = resolve; });
+ try {
+ const rendering = generateImage({ modelId: 'qwen-image-2.1', prompt: 'a fox', jobId: 'test-cancel-while-preparing' });
+ const refused = expect(rendering).rejects.toMatchObject({ code: 'IMAGE_GEN_CANCELED' });
+ await entered;
+ expect(cancel()).toBe('pending');
+ release();
+ await refused;
+ } finally {
+ memoryGate.hold = undefined;
+ memoryGate.entered = undefined;
+ for (const event of ['started', 'completed', 'failed']) imageGenEvents.off(event, terminal);
+ }
+ expect(mockSpawn).not.toHaveBeenCalled();
+ expect(releaseClaim).toHaveBeenCalledTimes(1);
+ // The rejection is the one terminal signal (the queue maps it to canceled);
+ // no render ever started, so nothing else may be emitted.
+ expect(terminal).not.toHaveBeenCalled();
+ // Once the job unwound there is nothing left to cancel.
+ expect(cancel()).toBe(false);
+ });
});
// Regression: SSE had the Python diagnosis, but the queue and status snapshot
diff --git a/server/services/mediaJobQueue/index.js b/server/services/mediaJobQueue/index.js
index cbebb7b086..ffc48e06c2 100644
--- a/server/services/mediaJobQueue/index.js
+++ b/server/services/mediaJobQueue/index.js
@@ -1818,6 +1818,9 @@ export async function cancelJob(jobId) {
// cancellation rather than silently resurrecting the remote render.
await persist();
}
+ // Provider contract: `true` = child signalled, `'pending'` = the provider owns
+ // the job but is still preparing (it aborts at its spawn gate and emits
+ // `failed`), `false` = nothing cancelable, i.e. finalization already began.
if (mod?.cancel && mod.cancel(jobId) === false
&& (runningJob.providerInvoked || isRemoteMediaJob(runningJob))) {
// A provider may have crossed its durable finalization boundary before
diff --git a/server/services/mediaJobQueue/index.test.js b/server/services/mediaJobQueue/index.test.js
index 80508fa510..3fa271df06 100644
--- a/server/services/mediaJobQueue/index.test.js
+++ b/server/services/mediaJobQueue/index.test.js
@@ -1080,7 +1080,7 @@ describe('mediaJobQueue', () => {
expect(mediaJobQueue.getJob(job.jobId).error).toBe('History write failed');
});
- it.each([true, undefined])('preserves accepted or legacy cancellation results (%s)', async (accepted) => {
+ it.each([true, undefined, 'pending'])('preserves accepted, pending or legacy cancellation results (%s)', async (accepted) => {
const job = await mediaJobQueue.enqueueJob({ kind: 'video', params: { mode: 'grok', prompt: 'cancel render', videoProduction: { projectId: 'example-project', attemptId: 'example-attempt' } } });
await waitFor(() => stubs.generateVideoGrok.mock.calls.length === 1);
stubs.cancelVideo.mockImplementationOnce(() => {
diff --git a/server/services/videoGen/generateVideo.js b/server/services/videoGen/generateVideo.js
index 74d95f87c5..a6163cb82c 100644
--- a/server/services/videoGen/generateVideo.js
+++ b/server/services/videoGen/generateVideo.js
@@ -55,7 +55,7 @@ import {
import { getSettings } from '../settings.js';
import { loadHistory, saveHistory, mutateVideoHistory, getHistoryItem } from './history.js';
import { VIDEO_MODE_GATED_RUNTIMES } from './modeContract.js';
-import { videoJobState } from './jobState.js';
+import { videoJobState, beginPreparation } from './jobState.js';
import {
assertRenderModeContract,
buildArgs,
@@ -315,7 +315,7 @@ async function resolveRequestedDraftDecoder({ model, draftDecode, jobId }) {
return draftDecoder;
}
-export async function generateVideo({ pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {
+async function renderLocalVideo(preparation, { pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {
const sourceAudioProvenance = audioConditioning == null ? null : federatedSourceAudioSchema.parse(audioConditioning);
uploadedTempPaths = Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [];
if (!prompt?.trim()) throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });
@@ -722,6 +722,8 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
const { ensureMiniMaxH3Weights } = await import('./ensureWeights.js');
await ensureMiniMaxH3Weights(model);
}
+ // Weights can take a long first-run download; honor a cancel that landed during it.
+ preparation.throwIfCanceled();
} catch (err) {
job.status = 'error';
const reason = err.message || 'Failed to prepare video generation';
@@ -742,6 +744,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
await spawnAndWatchVideo({
jobId,
+ preparation,
batch,
// Keeps this render's ETA samples in its own cost bucket — a speed profile
// changes the slope, not just the step count.
@@ -782,3 +785,14 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', m
return { jobId, generationId: jobId, filename, mode: 'local', model: modelId };
}
+
+// Pre-spawn work (weight download, memory preparation) runs with no child to
+// signal, so cancel() answers 'pending' and the spawn gates abort the render.
+export async function generateVideo(params) {
+ const preparation = beginPreparation();
+ try {
+ return await renderLocalVideo(preparation, params);
+ } finally {
+ preparation.end();
+ }
+}
diff --git a/server/services/videoGen/jobState.js b/server/services/videoGen/jobState.js
index 660aca4ef7..30691109fa 100644
--- a/server/services/videoGen/jobState.js
+++ b/server/services/videoGen/jobState.js
@@ -6,6 +6,7 @@
*/
import { killWithEscalation } from '../../lib/killWithEscalation.js';
+import { ServerError } from '../../lib/errorHandler.js';
import { attachSseClient as attachSse } from '../../lib/sseUtils.js';
export const videoJobState = {
@@ -13,6 +14,29 @@ export const videoJobState = {
activeProcess: null,
cancelEpoch: 0,
activeChain: null,
+ // Renders between generateVideo() entry and a wired child. A cancel() in that
+ // window has no process to kill but must still stop the render from spawning.
+ preparing: 0,
+};
+
+// Brackets one render's pre-spawn phase. `throwIfCanceled()` is the spawn gate:
+// any cancel() since this call (epoch bump) aborts the render before it starts.
+export const beginPreparation = () => {
+ const epoch = videoJobState.cancelEpoch;
+ videoJobState.preparing += 1;
+ let ended = false;
+ const wasCanceled = () => videoJobState.cancelEpoch !== epoch;
+ return {
+ wasCanceled,
+ throwIfCanceled: () => {
+ if (wasCanceled()) throw new ServerError('Canceled before the render started', { status: 409, code: 'VIDEO_GEN_CANCELED' });
+ },
+ end: () => {
+ if (ended) return;
+ ended = true;
+ videoJobState.preparing -= 1;
+ },
+ };
};
export const attachSseClient = (jobId, res) => attachSse(videoJobState.jobs, jobId, res);
@@ -22,7 +46,12 @@ export const cancel = () => {
// Flag a chain so its orchestrator stops between chunks, then terminate the
// child currently holding the local GPU lane when one exists.
if (videoJobState.activeChain) videoJobState.activeChain.stopped = true;
- if (!videoJobState.activeProcess) return !!videoJobState.activeChain;
+ if (!videoJobState.activeProcess) {
+ if (videoJobState.activeChain) return true;
+ // 'pending': the job is ours and still preparing; the render will abort
+ // itself at its next spawn gate. `false` stays "nothing cancelable here".
+ return videoJobState.preparing > 0 ? 'pending' : false;
+ }
const proc = videoJobState.activeProcess;
// Keep the handle until close so a new render cannot replace it before the
// escalation timer has had a chance to SIGKILL a stubborn child.
diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.js
index 665c54ffa4..476312646d 100644
--- a/server/services/videoGen/local.test.js
+++ b/server/services/videoGen/local.test.js
@@ -3191,6 +3191,42 @@ describe('generateVideo — MiniMax H3 MLX contract', () => {
))).toEqual(expectedAnchors);
});
+ it('cancel() while first-run weights are still preparing answers pending and never spawns', async () => {
+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');
+ const { cancel } = await import('./local.js');
+ vi.mocked(spawnDetached).mockClear();
+ let release;
+ const gate = new Promise((resolve) => { release = resolve; });
+ let enteredWeights;
+ const entered = new Promise((resolve) => { enteredWeights = resolve; });
+ // quant_config.json is probed only by ensureMiniMaxH3Weights.
+ mockFindCachedRepoFile.mockImplementation(async (_repo, filename) => {
+ if (filename === 'quant_config.json') {
+ enteredWeights();
+ await gate;
+ }
+ return join('/mock/hf/snap', filename);
+ });
+ try {
+ const rendering = generateVideo({
+ jobId: 'h3-cancel-while-preparing',
+ modelId: 'minimax_h3_8bit',
+ prompt: 'a fox watches the rain',
+ width: 512, height: 320, numFrames: 141, fps: 24, mode: 'text',
+ });
+ const refused = expect(rendering).rejects.toMatchObject({ code: 'VIDEO_GEN_CANCELED' });
+ await entered;
+ expect(cancel()).toBe('pending');
+ release();
+ await refused;
+ } finally {
+ mockFindCachedRepoFile.mockImplementation(async (_repo, filename) => join('/mock/hf/snap', filename));
+ }
+ expect(spawnDetached).not.toHaveBeenCalled();
+ // Preparation ended with the render, so a later cancel has nothing to claim.
+ expect(cancel()).toBe(false);
+ });
+
it('uses the pinned cache-only helper, locked sampler and credential-free environment', async () => {
const { spawnDetached } = await import('../../lib/detachedSpawn.js');
const { hfChildEnv } = await import('../hfToken.js');
@@ -5468,6 +5504,33 @@ describe('generateVideo — Gemma prompt-encode watchdog relaunch', () => {
expect(failures).toHaveLength(1);
});
+ // Before activeProcess is set, cancel() has no child to signal; the epoch
+ // taken at render entry is what delivers it once the spawn resolves.
+ it('stops the first child when a cancel lands during its spawn await', async () => {
+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');
+ const { cancel } = await import('./local.js');
+ const spawnMock = vi.mocked(spawnDetached);
+ const first = makeDrivenProc(113);
+ spawnMock.mockClear();
+ spawnMock.mockImplementationOnce(async () => {
+ expect(cancel()).toBe('pending');
+ return first.proc;
+ });
+ await generateVideo({
+ jobId: 'cancel-during-first-spawn',
+ pythonPath: '/usr/bin/python3',
+ modelId: 'ltx2_unified',
+ prompt: 'a lighthouse in fog',
+ width: 512,
+ height: 512,
+ numFrames: 25,
+ fps: 24,
+ seed: 987654,
+ });
+
+ expect(first.proc.kill).toHaveBeenCalledWith('SIGTERM');
+ });
+
// A replacement child that never gets its close listener can never report a
// terminal event, so it must not be left running.
it('stops the replacement child when wiring it throws', async () => {
diff --git a/server/services/videoGen/spawnWatch.js b/server/services/videoGen/spawnWatch.js
index 16bbff3405..e60a611336 100644
--- a/server/services/videoGen/spawnWatch.js
+++ b/server/services/videoGen/spawnWatch.js
@@ -42,7 +42,7 @@ import { isDisplaySleepEnabled, sleepDisplayForVideo, wakeDisplayForVideo } from
import { loadHistory, mutateVideoHistory } from './history.js';
import { estimateRenderMs } from './eta.js';
import { spliceExtendSource } from './extendSplice.js';
-import { videoJobState } from './jobState.js';
+import { videoJobState, cancel as cancelLocalVideo } from './jobState.js';
const MODULE_NOT_FOUND_RE = /ModuleNotFoundError: No module named ['"]([^'"]+)['"]/;
const STDERR_TAIL_LINES = 40;
@@ -58,6 +58,7 @@ export async function spawnAndWatchVideo({
jobId,
batch = null,
speedProfileId = null,
+ preparation = null,
cleanupTempFiles,
stepwiseDir,
job,
@@ -837,6 +838,8 @@ export async function spawnAndWatchVideo({
// still `proc.kill()` it directly by PID on cancel / watchdog. `cleanup: true`
// lets the helper drop that scratch dir on every terminal path (close/error)
// so it can't accumulate under data/videos.
+ // Last spawn gate; the spawn call follows with no intervening await.
+ preparation?.throwIfCanceled();
firstProc = await spawnDetached(bin, args, {
env: childEnv,
controlDir: join(PATHS.videos, '.detached', jobId),
@@ -854,6 +857,8 @@ export async function spawnAndWatchVideo({
// with it.
takeEarlyExit = bufferChildExit(firstProc, { bufferOutput: true });
videoJobState.activeProcess = firstProc;
+ // A cancel() during the spawn await saw no process; deliver it now.
+ if (preparation?.wasCanceled()) cancelLocalVideo();
await heavyClaim.handoffTo?.(firstProc.pid);
} catch (err) {
await abandonBeforeWiring(err);