Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: separate local video asset and decoder preparation
Adam Eivy820847fc68e9
diff --git a/server/services/videoGen/generateVideo.js b/server/services/videoGen/generateVideo.jsindex 4472891307..74d95f87c5 100644--- a/server/services/videoGen/generateVideo.js+++ b/server/services/videoGen/generateVideo.js@@ -172,53 +172,9 @@ export async function localVideoSentPrompt({ prompt = '', negativePrompt, loraFi};}-export async function generateVideo({ pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {- const sourceAudioProvenance = audioConditioning == null ? null : federatedSourceAudioSchema.parse(audioConditioning);- uploadedTempPaths = Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [];- if (!prompt?.trim()) throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });- // Single-flight is now enforced by the mediaJobQueue worker upstream — only- // one job is dequeued at a time, so we don't need a BUSY guard here. Direct- // callers (legacy / tests) bypass the queue and would clobber the shared active process- // on concurrent calls; that's an explicit "don't do that" contract.-- const { modelId: selectedModelId, model } = await resolveVideoModelSelection(modelId, { resolveModel: resolveVideoModel });- modelId = selectedModelId;- if (!model) throw new ServerError(`Unknown video model: ${modelId}`, { status: 400, code: 'VALIDATION_ERROR' });- validateVideoBatch({ batchSize, seed }, model);- if (!isHardwareCompatible(model.hardwareCompatibility)) {- throw new ServerError(- hardwareUnavailableReason(`Video model "${modelId}"`, model.hardwareCompatibility),- { status: 400, code: 'MODEL_HARDWARE_UNAVAILABLE' },- );- }- // Validate the mode contract before cache lookups, image resize, or staging- // work. Internal producers and persisted/retried jobs bypass route- // preparation, so silently dropping one of these inputs here would render a- // materially different video than the caller requested. Ungated runtimes- // (ltx2 / mlx_video) fall through untouched.- //- // Promote before checking: the route sets both fields, but a direct caller- // that only staged `uploadedTempPath` would otherwise pass the mode guard and- // then render text-only with its image dropped.- if (VIDEO_MODE_GATED_RUNTIMES.has(model.runtime)) sourceImagePath ||= uploadedTempPath;- assertRenderModeContract({- model,- mode,- sourceImagePath,- lastImagePath,- keyframes,- extendFromVideoPath,- audioFilePath,- audioStartSec,- icReferencePaths,- });- // Registry defaults are user-editable. Validate them at this internal-call- // boundary so an empty or malformed value cannot turn into a 0/NaN runner- // argument when the caller deliberately leaves the resolution unset.- ({ width, height } = resolveVideoDimensions(model, width, height));- numFrames = numFrames ?? model.defaultFrames ?? DEFAULT_NUM_FRAMES;- const frameCountError = wan22FrameCountError(model, numFrames);- if (frameCountError) throw frameCountError;+// Resolve required render assets before staging media or acquiring GPU work.+// Runtime-specific pin/fallback policies and dependency ordering stay explicit.+async function resolveVideoModelAssets({ model, modelId, textEncoderId }) {let wanModelPath = null;const wanRequiredWeights = [];if (model.runtime === 'wan22' || model.runtime === 'wan22_cuda') {@@ -311,6 +267,104 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', mconfigOverrides: textEncoderOption.configOverrides,};}+ return { wanModelPath, wanRequiredWeights, fastvideoModelPath, ltxModelPath, ref2vaModelPath, resolvedTextEncoder };+}++// Preview-fidelity video decode (#5423). Every gate lives in+// lib/videoDraftDecoders.js and RETURNS its reason; the render always+// proceeds, on the full decoder, because a decode substitution only ever+// makes a render cheaper. Resolved here rather than in the arg builder for+// the same reason the substituted conditioner is: a missing download has to+// be discovered before any GPU work, and the decision has to be recorded on+// the history row.+//+// Two gates need I/O and so cannot live in the pure module: whether the+// installed runner checkout is the revision the asset was verified against+// (an older one ignores the flags, which would let a full decode report+// itself as a draft), and whether the pinned files are actually in the HF+// cache. Both are only probed when a draft decode was requested AND the model+// declares one, so a full-decode render costs nothing extra.+async function resolveRequestedDraftDecoder({ model, draftDecode, jobId }) {+ let draftDecoder = null;+ if (!isFullDecode(draftDecode)) {+ const declared = model.draftDecoder;+ const runtimeRevision = (declared && await isByovRuntimeCurrent(model.runtime))+ ? byovRuntimeExpectedRevision(model.runtime)+ : null;+ // Only probed once the cheaper gates have passed — a model with no+ // declaration, or a checkout at the wrong revision, never touches the cache.+ const decoderPaths = (declared && runtimeRevision === declared.runtimeRevision)+ ? await findCachedRepoFiles(declared.repo, declared.files, { revision: declared.revision })+ : null;+ const gateArgs = {+ model,+ models: getVideoModels(),+ decodeId: draftDecode,+ runtimeRevision,+ assetCached: Array.isArray(decoderPaths) && decoderPaths.length > 0,+ };+ const resolved = resolveVideoDraftDecoder(gateArgs);+ if (resolved) {+ draftDecoder = { ...resolved, paths: decoderPaths };+ console.log(`🩻 Draft decode "${resolved.id}" [${jobId.slice(0, 8)}] via ${resolved.label}`);+ } else {+ const declined = draftDecodeDeclineReason(gateArgs);+ console.log(`⚠️ Draft decode declined [${jobId.slice(0, 8)}] ${declined?.code || 'UNKNOWN'}: ${declined?.message || 'rendering on the full decoder'}`);+ }+ }+ return draftDecoder;+}++export async function generateVideo({ pythonPath, prompt, negativePrompt = '', modelId, width = null, height = null, numFrames = null, fps = 24, steps, guidanceScale, seed, batchSize = 1, tiling = 'auto', disableAudio = false, sourceImagePath = null, uploadedTempPath = null, uploadedTempPaths = [], lastImagePath = null, keyframes = null, extendFromVideoPath = null, audioFilePath = null, audioStartSec = null, audioConditioning = null, mode = null, imageStrength = null, i2vReferenceMode = null, loras = null, icReferencePaths = null, icStrength = null, icAttentionStrength = null, icSkipStage2 = false, textEncoderId = null, speedProfileId = null, draftDecode = null, streamingMode = null, visualConditioning = null, hidden = false, keepExtendSource = false, displaySleep = null, jobId: providedJobId = null, promptAdditions = [] }) {+ const sourceAudioProvenance = audioConditioning == null ? null : federatedSourceAudioSchema.parse(audioConditioning);+ uploadedTempPaths = Array.isArray(uploadedTempPaths) ? uploadedTempPaths : [];+ if (!prompt?.trim()) throw new ServerError('Prompt is required', { status: 400, code: 'VALIDATION_ERROR' });+ // Single-flight is now enforced by the mediaJobQueue worker upstream — only+ // one job is dequeued at a time, so we don't need a BUSY guard here. Direct+ // callers (legacy / tests) bypass the queue and would clobber the shared active process+ // on concurrent calls; that's an explicit "don't do that" contract.++ const { modelId: selectedModelId, model } = await resolveVideoModelSelection(modelId, { resolveModel: resolveVideoModel });+ modelId = selectedModelId;+ if (!model) throw new ServerError(`Unknown video model: ${modelId}`, { status: 400, code: 'VALIDATION_ERROR' });+ validateVideoBatch({ batchSize, seed }, model);+ if (!isHardwareCompatible(model.hardwareCompatibility)) {+ throw new ServerError(+ hardwareUnavailableReason(`Video model "${modelId}"`, model.hardwareCompatibility),+ { status: 400, code: 'MODEL_HARDWARE_UNAVAILABLE' },+ );+ }+ // Validate the mode contract before cache lookups, image resize, or staging+ // work. Internal producers and persisted/retried jobs bypass route+ // preparation, so silently dropping one of these inputs here would render a+ // materially different video than the caller requested. Ungated runtimes+ // (ltx2 / mlx_video) fall through untouched.+ //+ // Promote before checking: the route sets both fields, but a direct caller+ // that only staged `uploadedTempPath` would otherwise pass the mode guard and+ // then render text-only with its image dropped.+ if (VIDEO_MODE_GATED_RUNTIMES.has(model.runtime)) sourceImagePath ||= uploadedTempPath;+ assertRenderModeContract({+ model,+ mode,+ sourceImagePath,+ lastImagePath,+ keyframes,+ extendFromVideoPath,+ audioFilePath,+ audioStartSec,+ icReferencePaths,+ });+ // Registry defaults are user-editable. Validate them at this internal-call+ // boundary so an empty or malformed value cannot turn into a 0/NaN runner+ // argument when the caller deliberately leaves the resolution unset.+ ({ width, height } = resolveVideoDimensions(model, width, height));+ numFrames = numFrames ?? model.defaultFrames ?? DEFAULT_NUM_FRAMES;+ const frameCountError = wan22FrameCountError(model, numFrames);+ if (frameCountError) throw frameCountError;+ const {+ wanModelPath, wanRequiredWeights, fastvideoModelPath, ltxModelPath, ref2vaModelPath, resolvedTextEncoder,+ } = await resolveVideoModelAssets({ model, modelId, textEncoderId });// Only require the legacy mlx_video pythonPath when the chosen runtime// actually uses it. BYOV runtimes resolve their own venv path inside@@ -421,47 +475,7 @@ export async function generateVideo({ pythonPath, prompt, negativePrompt = '', mif (speedProfileDeclined) {console.log(`⚠️ Speed profile declined [${jobId.slice(0, 8)}] ${speedProfileDeclined.code}: ${speedProfileDeclined.message}`);}- // Preview-fidelity video decode (#5423). Every gate lives in- // lib/videoDraftDecoders.js and RETURNS its reason; the render always- // proceeds, on the full decoder, because a decode substitution only ever- // makes a render cheaper. Resolved here rather than in the arg builder for- // the same reason the substituted conditioner is: a missing download has to- // be discovered before any GPU work, and the decision has to be recorded on- // the history row.- //- // Two gates need I/O and so cannot live in the pure module: whether the- // installed runner checkout is the revision the asset was verified against- // (an older one ignores the flags, which would let a full decode report- // itself as a draft), and whether the pinned files are actually in the HF- // cache. Both are only probed when a draft decode was requested AND the model- // declares one, so a full-decode render costs nothing extra.- let draftDecoder = null;- if (!isFullDecode(draftDecode)) {- const declared = model.draftDecoder;- const runtimeRevision = (declared && await isByovRuntimeCurrent(model.runtime))- ? byovRuntimeExpectedRevision(model.runtime)- : null;- // Only probed once the cheaper gates have passed — a model with no- // declaration, or a checkout at the wrong revision, never touches the cache.- const decoderPaths = (declared && runtimeRevision === declared.runtimeRevision)- ? await findCachedRepoFiles(declared.repo, declared.files, { revision: declared.revision })- : null;- const gateArgs = {- model,- models: getVideoModels(),- decodeId: draftDecode,- runtimeRevision,- assetCached: Array.isArray(decoderPaths) && decoderPaths.length > 0,- };- const resolved = resolveVideoDraftDecoder(gateArgs);- if (resolved) {- draftDecoder = { ...resolved, paths: decoderPaths };- console.log(`🩻 Draft decode "${resolved.id}" [${jobId.slice(0, 8)}] via ${resolved.label}`);- } else {- const declined = draftDecodeDeclineReason(gateArgs);- console.log(`⚠️ Draft decode declined [${jobId.slice(0, 8)}] ${declined?.code || 'UNKNOWN'}: ${declined?.message || 'rendering on the full decoder'}`);- }- }+ const draftDecoder = await resolveRequestedDraftDecoder({ model, draftDecode, jobId });const sampler = resolveVideoSampler({ model, steps, guidanceScale, speedProfile });let actualSteps = sampler.steps;diff --git a/server/services/videoGen/local.test.js b/server/services/videoGen/local.test.jsindex 24b273ab55..d0abd76847 100644--- a/server/services/videoGen/local.test.js+++ b/server/services/videoGen/local.test.js@@ -3062,6 +3062,45 @@ describe('generateVideo — Wan MLX-Gen contract', () => {expect(spawnDetached).not.toHaveBeenCalled();});+ it('rejects mismatched Wan dependency roles before staging or spawning', async () => {+ const { getVideoModels } = await import('../../lib/mediaModels.js');+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ const catalog = getVideoModels();+ vi.mocked(getVideoModels).mockReturnValueOnce(catalog.map((model) => (+ model.id === 'wan22_t2v_a14b_lightning'+ ? { ...model, requiredWeights: [{ ...model.requiredWeights[0], targetRoles: ['high_noise_transformer'] }] }+ : model+ )));++ await expect(generateVideo({+ modelId: 'wan22_t2v_a14b_lightning', prompt: 'a quiet lake',+ width: 480, height: 256, numFrames: 81, fps: 20, mode: 'text',+ })).rejects.toMatchObject({+ status: 500, code: 'VIDEO_MODEL_MISCONFIGURED',+ message: 'Wan model "wan22_t2v_a14b_lightning" has an invalid requiredWeights entry.',+ });+ expect(mockFindCachedRepoFile).not.toHaveBeenCalled();+ expect(fsPromisesMock.mkdtemp).not.toHaveBeenCalled();+ expect(spawnDetached).not.toHaveBeenCalled();+ });++ it('rejects a partially cached Wan adapter pair before staging or spawning', async () => {+ const { spawnDetached } = await import('../../lib/detachedSpawn.js');+ mockFindCachedRepoFile+ .mockResolvedValueOnce('/mock/hf/high_noise_model.safetensors')+ .mockResolvedValueOnce(null);++ await expect(generateVideo({+ modelId: 'wan22_t2v_a14b_lightning', prompt: 'a quiet lake',+ width: 480, height: 256, numFrames: 81, fps: 20, mode: 'text',+ })).rejects.toMatchObject({+ status: 400, code: 'WAN22_REQUIRED_WEIGHT_NOT_CACHED',+ message: 'Wan T2V Lightning is missing required weight Wan2.2-T2V-A14B-4steps-lora-rank64-Seko-V1.1/low_noise_model.safetensors. Download or repair its dependencies in Video Gen.',+ });+ expect(fsPromisesMock.mkdtemp).not.toHaveBeenCalled();+ expect(spawnDetached).not.toHaveBeenCalled();+ });+it('rejects a T2V-only Wan chain before rendering chunk one', async () => {const { spawnDetached } = await import('../../lib/detachedSpawn.js');vi.mocked(spawnDetached).mockClear();