Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

38.9 KB
Raw Blame
import { describe, it, expect, vi, beforeEach, afterAll } from 'vitest';
import { writeFileSync, rmSync, mkdirSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { EventEmitter } from 'events';
import { cleanupTempDataRoots, lazyTempDataRoot, makePathsProxy } from '../../lib/mockPathsDataRoot.js';
import { makeVideoGenLineHandler, isWatchdogSuccess, finalizeGeneratedVideo, parseByteProgress, formatBytes, formatDownloadMessage, describeSignalDeath, formatRuntimeFingerprint, describeRenderConditioning, isPromptEncodingMetalWatchdog, bufferChildExit, isNativeTeardownAbort, verifyPostCompletionOutputs, POST_COMPLETION_ABORT_RUNTIME, RENDER_INPUTS_VERSION, emitCloudRenderStatus, CLOUD_RENDER_PHASE } from './generateVideoHelpers.js';

describe('parseByteProgress', () => {
  it('parses single byte value (e.g., "2.5G")', () => {
    const result = parseByteProgress('model is 2.5G');
    expect(result.downloaded).toBeNull();
    expect(result.total).toBeCloseTo(2.5 * 1024 ** 3, -5);
  });

  it('parses downloaded/total format (e.g., "1.5G/2.0G")', () => {
    const result = parseByteProgress('1.5G/2.0G downloaded');
    expect(result.downloaded).toBeCloseTo(1.5 * 1024 ** 3, -5);
    expect(result.total).toBeCloseTo(2.0 * 1024 ** 3, -5);
  });

  it('parses MB values', () => {
    const result = parseByteProgress('500MB/1024MB');
    expect(result.downloaded).toBeCloseTo(500 * 1024 ** 2, -5);
    expect(result.total).toBeCloseTo(1024 * 1024 ** 2, -5);
  });

  it('parses M suffix (common in tqdm)', () => {
    const result = parseByteProgress('512M/1.0G');
    expect(result.downloaded).toBeCloseTo(512 * 1024 ** 2, -5);
    expect(result.total).toBeCloseTo(1.0 * 1024 ** 3, -5);
  });

  it('returns nulls when no byte values found', () => {
    const result = parseByteProgress('model.safetensors 40%');
    expect(result.downloaded).toBeNull();
    expect(result.total).toBeNull();
  });

  it('parses tqdm-style progress bars with bytes', () => {
    const result = parseByteProgress('50%|█████     | 1.00G/2.00G [00:22<00:22, 45.6MB/s]');
    expect(result.downloaded).toBeCloseTo(1.0 * 1024 ** 3, -5);
    expect(result.total).toBeCloseTo(2.0 * 1024 ** 3, -5);
  });

  it('parses GiB suffix', () => {
    const result = parseByteProgress('1.5GiB/3.0GiB');
    expect(result.downloaded).toBeCloseTo(1.5 * 1024 ** 3, -5);
    expect(result.total).toBeCloseTo(3.0 * 1024 ** 3, -5);
  });
});

describe('formatBytes (re-exported from fileUtils)', () => {
  it('formats bytes as B', () => {
    expect(formatBytes(512)).toBe('512 B');
  });

  it('formats kilobytes', () => {
    expect(formatBytes(2048)).toBe('2 KB');
  });

  it('formats megabytes', () => {
    expect(formatBytes(1024 * 1024 * 1.5)).toBe('1.5 MB');
  });

  it('formats gigabytes', () => {
    expect(formatBytes(1024 ** 3 * 2.5)).toBe('2.5 GB');
  });

  it('formats terabytes', () => {
    expect(formatBytes(1024 ** 4 * 1.2)).toBe('1.2 TB');
  });

  it('handles null/undefined as 0 B', () => {
    expect(formatBytes(null)).toBe('0 B');
    expect(formatBytes(undefined)).toBe('0 B');
  });
});

describe('formatDownloadMessage', () => {
  it('formats with both downloaded and total', () => {
    const byteInfo = { downloaded: 1.5 * 1024 ** 3, total: 2.5 * 1024 ** 3 };
    expect(formatDownloadMessage('raw text', byteInfo)).toBe('Downloading model · first run · 1.5 GB / 2.5 GB');
  });

  it('formats with only total', () => {
    const byteInfo = { downloaded: null, total: 2.5 * 1024 ** 3 };
    expect(formatDownloadMessage('raw text', byteInfo)).toBe('Downloading model · first run · 2.5 GB');
  });

  it('falls back to raw text when no byte info', () => {
    const byteInfo = { downloaded: null, total: null };
    expect(formatDownloadMessage('model.safetensors 40%', byteInfo)).toBe('Downloading model... model.safetensors 40%');
  });
});

// broadcastSse + videoGenEvents are the two output sinks the line handler
// writes to; capture both so we can assert the parse → frame mapping.
const sse = vi.hoisted(() => vi.fn());
const emitted = vi.hoisted(() => []);
vi.mock('../../lib/sseUtils.js', () => ({ broadcastSse: sse }));
vi.mock('./events.js', () => ({
  videoGenEvents: { emit: (type, payload) => { emitted.push({ type, payload }); } },
}));
// generateVideoHelpers also imports ffmpeg + fs at module top; stub ffmpeg so
// the import graph stays light (finalize isn't exercised in this file).
const probeMock = vi.hoisted(() => ({ frames: vi.fn(async () => null), duration: vi.fn(async () => null) }));
vi.mock('../../lib/ffmpeg.js', () => ({
  generateThumbnail: vi.fn(),
  optimizeForStreaming: vi.fn(),
  probeFrameCount: probeMock.frames,
  probeVideoDuration: probeMock.duration,
}));
vi.mock('../../lib/fileUtils.js', async importOriginal => makePathsProxy(await importOriginal(), {
  dataRoot: () => lazyTempDataRoot('portos-video-finalizer-metadata-'),
}));
afterAll(() => cleanupTempDataRoots());

const PYTHON_NOISE_RE = /^(Loading|Fetching|tokenizer|Some weights)/;

describe('makeVideoGenLineHandler', () => {
  let job;
  let handle;

  beforeEach(() => {
    sse.mockClear();
    emitted.length = 0;
    job = { id: 'j1', clients: [] };
    handle = makeVideoGenLineHandler({ job, jobId: 'job-12345678', pythonNoiseRe: PYTHON_NOISE_RE });
  });

  const sseFrames = () => sse.mock.calls.map((c) => c[1]);
  const eventsOfType = (t) => emitted.filter((e) => e.type === t).map((e) => e.payload);

  // Prompt-encode phase (#4589) — the signal that decides whether a later
  // SIGABRT is worth one relaunch with smaller command buffers.
  it('tracks the Gemma prompt-encode phase as three distinct states', () => {
    // Never reported: every runtime other than the LTX-2 MLX helper stays here,
    // and an unstamped field must not read as "encoding right now".
    expect(job.promptEncodePhase).toBeUndefined();
    handle('STAGE:load-pipeline');
    expect(job.promptEncodePhase).toBeUndefined();

    handle('STAGE:encode-prompt');
    expect(job.promptEncodePhase).toBe('active');

    // The end marker is a PREFIX EXTENSION of the begin marker, so a startsWith
    // comparison would read the end of the encode as another beginning and leave
    // the phase open forever.
    handle('STAGE:encode-prompt-done');
    expect(job.promptEncodePhase).toBe('done');
  });

  it('still surfaces the prompt-encode markers to the client as status frames', () => {
    handle('STAGE:encode-prompt');
    expect(eventsOfType('status')).toContainEqual({ generationId: 'job-12345678', message: 'encode-prompt', phase: 'encode-prompt' });
  });

  it('repeats the job ETA on every progress frame, and omits it when absent (#3801)', () => {
    // No estimate on the job → the key must be absent, never etaMs: 0.
    handle('STAGE:inference:step:5:30:Rendering');
    handle('60%|██████    | 6/10');
    expect(eventsOfType('progress').every((p) => !('etaMs' in p))).toBe(true);

    emitted.length = 0;
    job.etaMs = 1_800_000;
    handle('STAGE:inference:step:6:30:Rendering');
    handle('70%|███████   | 7/10');
    const progress = eventsOfType('progress');
    expect(progress).toHaveLength(2);
    expect(progress.every((p) => p.etaMs === 1_800_000)).toBe(true);
  });

  it('suppresses blank + python-noise lines without emitting', () => {
    expect(handle('')).toBe(true);
    expect(handle('   ')).toBe(true);
    expect(handle('Loading pipeline components...')).toBe(true);
    expect(sse).not.toHaveBeenCalled();
    expect(emitted).toHaveLength(0);
  });

  it('STATUS: → status SSE frame + status event, and an activity heartbeat', () => {
    expect(handle('STATUS:Generating I2V…')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'status', message: 'Generating I2V…', phase: 'starting' });
    expect(eventsOfType('status')).toContainEqual({ generationId: 'job-12345678', message: 'Generating I2V…', phase: 'starting' });
    expect(eventsOfType('activity')).toContainEqual({ generationId: 'job-12345678' });
  });

  it('STAGE:<s>:step:<cur>:<total>:<label> → fractional progress with label and phase', () => {
    expect(handle('STAGE:render:step:6:10:Sampling latents')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'progress', progress: 0.6, message: 'Sampling latents', phase: 'render' });
    expect(eventsOfType('progress')).toContainEqual({
      generationId: 'job-12345678', progress: 0.6, step: 6, totalSteps: 10, message: 'Sampling latents', phase: 'render',
    });
  });

  it('STAGE: heartbeat does NOT become bogus progress (regression: 20s → 2000%)', () => {
    expect(handle('STAGE:download-clip:heartbeat:20s')).toBe(true);
    // Heartbeat is a status line, never a progress frame.
    expect(sseFrames()).toContainEqual({ type: 'status', message: 'download-clip: heartbeat 20s', phase: 'download-clip' });
    expect(sseFrames().some((f) => f.type === 'progress')).toBe(false);
  });

  it('normalizes uppercase STEP tag (generate_ltx2.py emits STEP:)', () => {
    expect(handle('STAGE:render:STEP:1:4:warmup')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'progress', progress: 0.25, message: 'warmup', phase: 'render' });
  });

  it('bare STAGE: phase marker → status (no division-by-undefined progress)', () => {
    expect(handle('STAGE:load-pipeline')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'status', message: 'load-pipeline', phase: 'load-pipeline' });
    expect(sseFrames().some((f) => f.type === 'progress')).toBe(false);
  });

  it('DOWNLOAD: → prefixed status frame with phase', () => {
    expect(handle('DOWNLOAD:model.safetensors 40%')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'status', message: 'Downloading model... model.safetensors 40%', phase: 'download' });
  });

  it('DOWNLOAD: with byte values → formatted GB message', () => {
    expect(handle('DOWNLOAD:1.5G/2.0G model.safetensors')).toBe(true);
    const frame = sseFrames().find(f => f.type === 'status');
    expect(frame.message).toBe('Downloading model · first run · 1.5 GB / 2.0 GB');
    expect(frame.downloadedBytes).toBeCloseTo(1.5 * 1024 ** 3, -5);
    expect(frame.totalBytes).toBeCloseTo(2.0 * 1024 ** 3, -5);
  });

  it('tqdm bar → progress frame with phase; queue event omits the noisy message', () => {
    expect(handle('60%|██████    | 6/10 [00:30<00:20, 1.2s/it]')).toBe(true);
    expect(sseFrames()).toContainEqual({ type: 'progress', progress: 0.6, message: '60%|██████    | 6/10 [00:30<00:20, 1.2s/it]', phase: 'starting' });
    // The mediaJobQueue dispatcher emit must NOT carry the raw bar as message.
    expect(eventsOfType('progress')).toContainEqual({ generationId: 'job-12345678', progress: 0.6, phase: 'starting' });
  });

  it('tqdm bar with byte sizes during download → formatted GB message', () => {
    // Enter download phase first
    handle('DOWNLOAD:1/5:model.safetensors');
    sse.mockClear();
    // Now a tqdm bar with byte counts
    expect(handle('50%|█████     | 1.00G/2.00G [00:22<00:22, 45.6MB/s]')).toBe(true);
    const frame = sseFrames().find(f => f.type === 'progress');
    expect(frame.message).toBe('Downloading model · first run · 1.0 GB / 2.0 GB');
    expect(frame.phase).toBe('download');
    expect(frame.downloadedBytes).toBeCloseTo(1.0 * 1024 ** 3, -5);
    expect(frame.totalBytes).toBeCloseTo(2.0 * 1024 ** 3, -5);
  });

  it('returns false for an unrecognized line (caller raw-logs it)', () => {
    expect(handle('🐍 some unexpected diagnostic')).toBe(false);
  });

  it('RUNTIME:<json> → stamps job.runtime and suppresses raw logging', () => {
    const fp = { runtime: 'ltx2', versions: { mlx: '0.22.0' }, chip: 'Apple M5 Max', os: 'macOS-15.4-arm64' };
    expect(handle(`RUNTIME:${JSON.stringify(fp)}`)).toBe(true);
    expect(job.runtime).toEqual(fp);
    // It's a one-shot metadata line, not progress/status — no SSE frame.
    expect(sse).not.toHaveBeenCalled();
  });

  it('malformed RUNTIME: line falls through to raw-logging and leaves job.runtime unset', () => {
    expect(handle('RUNTIME:{not json')).toBe(false);
    expect(job.runtime).toBeUndefined();
  });

  // #4875 — what the runner ACTUALLY applied of a requested speed profile.
  it('SPEEDPROFILE:<json> → stamps job.speedProfile and suppresses raw logging', () => {
    const applied = { id: 'fast', teacache: true, adapter: 'ltx-2.5-22b-distilled-lora-450.safetensors', degraded: [] };
    expect(handle('SPEEDPROFILE:' + JSON.stringify(applied))).toBe(true);
    expect(job.speedProfile).toEqual(applied);
    // A one-shot metadata line, not progress/status — no SSE frame.
    expect(sse).not.toHaveBeenCalled();
  });

  it('keeps the degraded lever list a render could not apply', () => {
    handle('SPEEDPROFILE:{"id":"fast","teacache":false,"degraded":["teacache"]}');
    expect(job.speedProfile.degraded).toEqual(['teacache']);
    expect(job.speedProfile.teacache).toBe(false);
  });

  it('malformed SPEEDPROFILE: line falls through to raw-logging and leaves job.speedProfile unset', () => {
    expect(handle('SPEEDPROFILE:{not json')).toBe(false);
    expect(job.speedProfile).toBeUndefined();
  });

  // #12262 — how each LoRA adapter was actually applied on this render.
  it('LORASTRATEGY:<json> → stamps job.loraStrategy; a malformed payload falls through', () => {
    const report = { adapters: [{ role: 'ic', name: 'ingredients.safetensors', strategy: 'runtime-delta', stages: ['stage-1'], strength: 1, quantizedTargets: 480, fusedTargets: 0 }] };
    expect(handle('LORASTRATEGY:{not json')).toBe(false);
    expect(handle('LORASTRATEGY:{"adapters":"nope"}')).toBe(false);
    expect(job.loraStrategy).toBeUndefined();
    expect(handle(`LORASTRATEGY:${JSON.stringify(report)}`)).toBe(true);
    expect(job.loraStrategy).toEqual(report);
    expect(sse).not.toHaveBeenCalled();
  });
});

describe('finalizeGeneratedVideo runtime persistence', () => {
  const baseCtx = (job) => ({
    job,
    jobId: 'job-abcdef12',
    outputPath: '/tmp/out.mp4',
    filename: 'out.mp4',
    meta: { id: 'job-abcdef12', prompt: 'hi', modelId: 'ltx2_unified' },
    actualSeed: 7,
  });

  it('persists job.runtime onto the saved history record', async () => {
    const fp = { runtime: 'ltx2', versions: { mlx: '0.22.0' }, chip: 'Apple M5 Max' };
    const job = { id: 'job-abcdef12', clients: [], runtime: fp };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    expect(saved).toHaveLength(1);
    expect(saved[0].runtime).toEqual(fp);
  });

  it('omits runtime when the child never emitted a fingerprint (absent sentinel)', async () => {
    const job = { id: 'job-abcdef12', clients: [] };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    expect(saved).toHaveLength(1);
    expect('runtime' in saved[0]).toBe(false);
  });

  it('carries the durable re-render inputs through to the saved record (#3696)', async () => {
    const job = { id: 'job-abcdef12', clients: [] };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      meta: {
        ...baseCtx(job).meta,
        seed: 424242,
        mode: 'text',
        renderInputsVersion: RENDER_INPUTS_VERSION,
        conditioning: [],
      },
      actualSeed: 424242,
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    expect(saved[0].seed).toBe(424242);
    expect(saved[0].renderInputsVersion).toBe(RENDER_INPUTS_VERSION);
    expect(saved[0].conditioning).toEqual([]);
    // No staging/temp path may ride along on a user-facing history record.
    expect(JSON.stringify(saved[0])).not.toMatch(/\/tmp\/|uploads/);
  });

  it('stamps wall-clock render timing so future renders are estimable (#3801)', async () => {
    const job = { id: 'job-abcdef12', clients: [], renderStartedAtMs: Date.now() - 5000 };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    expect(saved[0].renderMs).toBeGreaterThanOrEqual(5000);
    expect(Date.parse(saved[0].renderStartedAt)).toBe(job.renderStartedAtMs);
    expect(Date.parse(saved[0].renderCompletedAt)).toBeGreaterThanOrEqual(job.renderStartedAtMs);
  });

  it('persists what the runner actually applied of a speed profile (#4875)', async () => {
    const applied = { id: 'fast', teacache: false, adapter: 'ltx-2.3-22b-distilled-lora-384.safetensors', degraded: ['teacache', 'adapter'] };
    const job = { id: 'job-abcdef12', clients: [], speedProfile: applied };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      meta: { ...baseCtx(job).meta, speedProfileId: 'fast' },
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    // The REQUEST and the OUTCOME are separate fields on purpose: a degraded
    // render must not read back as a full speed claim.
    expect(saved[0].speedProfileId).toBe('fast');
    expect(saved[0].speedProfileApplied).toEqual(applied);
  });

  it('persists the LoRA strategy the runner reported, and omits it when none was (#12262)', async () => {
    const report = { adapters: [{ role: 'user', name: 'character.safetensors', strategy: 'runtime-delta', stages: ['all'], strength: 0.8, quantizedTargets: 480, fusedTargets: 0 }] };
    let saved = null;
    const mutateHistory = async (fn) => { saved = await fn([]); return saved; };
    await finalizeGeneratedVideo({ ...baseCtx({ id: 'job-abcdef12', clients: [], loraStrategy: report }), mutateHistory });
    expect(saved[0].loraStrategyApplied).toEqual(report);
    await finalizeGeneratedVideo({ ...baseCtx({ id: 'job-abcdef12', clients: [] }), mutateHistory });
    expect('loraStrategyApplied' in saved[0]).toBe(false);
  });

  it('omits speedProfileApplied when the runner reported none (absent sentinel)', async () => {
    const job = { id: 'job-abcdef12', clients: [] };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    expect('speedProfileApplied' in saved[0]).toBe(false);
  });

  it('omits render timing entirely when the spawn instant was never observed (#3801)', async () => {
    const job = { id: 'job-abcdef12', clients: [] };
    let saved = null;
    await finalizeGeneratedVideo({
      ...baseCtx(job),
      mutateHistory: async (fn) => { saved = await fn([]); return saved; },
    });
    // Absent, NOT renderMs: 0 — a zero-duration sample would poison the
    // estimator's cost model.
    expect('renderMs' in saved[0]).toBe(false);
    expect('renderStartedAt' in saved[0]).toBe(false);
  });
});

describe('describeRenderConditioning (#3696)', () => {
  it('reports an empty inventory for a plain text-to-video render', () => {
    expect(describeRenderConditioning()).toEqual([]);
    expect(describeRenderConditioning({})).toEqual([]);
    expect(describeRenderConditioning({ sourceImagePath: null, keyframes: [], icReferencePaths: [] })).toEqual([]);
  });

  it('names each conditioning input that steered the render', () => {
    expect(describeRenderConditioning({ sourceImagePath: '/tmp/a.png' })).toEqual(['image']);
    expect(describeRenderConditioning({ lastImagePath: '/tmp/z.png' })).toEqual(['lastImage']);
    expect(describeRenderConditioning({ keyframes: [{ path: '/tmp/1.png' }] })).toEqual(['keyframes']);
    expect(describeRenderConditioning({ extendFromVideoPath: '/data/videos/x.mp4' })).toEqual(['extend']);
    expect(describeRenderConditioning({ audioFilePath: '/tmp/song.wav' })).toEqual(['audio']);
    expect(describeRenderConditioning({ icReferencePaths: ['/tmp/ref.mp4'] })).toEqual(['icReference']);
  });

  it('returns a stable sorted list for a multi-input render', () => {
    expect(describeRenderConditioning({
      sourceImagePath: '/tmp/a.png',
      lastImagePath: '/tmp/z.png',
      audioFilePath: '/tmp/song.wav',
    })).toEqual(['audio', 'image', 'lastImage']);
  });

  it('records kinds only — never the staging paths it was given', () => {
    const kinds = describeRenderConditioning({ sourceImagePath: '/tmp/upload-9f3.png' });
    expect(kinds.join(',')).not.toMatch(/tmp|9f3/);
  });
});

describe('formatRuntimeFingerprint', () => {
  it('renders runtime | versions | chip | os', () => {
    expect(formatRuntimeFingerprint({
      runtime: 'ltx2',
      versions: { mlx: '0.22.0', mlx_metal: '0.22.0' },
      chip: 'Apple M4 Max',
      os: 'macOS-15.4-arm64',
    })).toBe('ltx2 | mlx 0.22.0, mlx_metal 0.22.0 | Apple M4 Max | macOS-15.4-arm64');
  });

  it('omits missing segments rather than leaving empty separators', () => {
    expect(formatRuntimeFingerprint({ chip: 'Apple M4 Max', os: 'macOS-15.4-arm64' }))
      .toBe('Apple M4 Max | macOS-15.4-arm64');
  });

  it('returns empty string for a missing/non-object payload', () => {
    expect(formatRuntimeFingerprint(null)).toBe('');
    expect(formatRuntimeFingerprint(undefined)).toBe('');
    expect(formatRuntimeFingerprint('ltx2')).toBe('');
    expect(formatRuntimeFingerprint({})).toBe('');
  });
});

describe('describeSignalDeath (#3101 signal → actionable cause)', () => {
  it('returns null when the child did not die on a signal', () => {
    expect(describeSignalDeath(null)).toBeNull();
    expect(describeSignalDeath(undefined)).toBeNull();
    expect(describeSignalDeath('')).toBeNull();
  });

  const MLX = { mlxRuntime: true, platform: 'darwin', arch: 'arm64' };

  it('names the Metal watchdog only for a captured timeout signature on an Apple-silicon MLX runtime', () => {
    for (const stderr of ['libc++abi: kIOGPUCommandBufferCallbackErrorImpactingInteractivity', 'Caused GPU Timeout Error', 'kIOGPUCommandBufferCallbackErrorTimeout']) {
      const reason = describeSignalDeath('SIGABRT', { stderr, ...MLX });
      expect(reason).toMatch(/Metal command-buffer watchdog/i);
      expect(reason).toMatch(/frame count/i);
    }
    // Same evidence on a CUDA/Windows host or a non-MLX runtime is not a Metal watchdog.
    expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, platform: 'win32' })).toMatch(/unknown/);
    expect(describeSignalDeath('SIGABRT', { stderr: 'Caused GPU Timeout Error', ...MLX, mlxRuntime: false })).toMatch(/unknown/);
  });

  it('reports an unexplained native abort when no diagnostic was captured, never the watchdog', () => {
    for (const opts of [undefined, { stderr: '', ...MLX }, { stderr: 'Loading weights...', ...MLX }]) {
      const reason = describeSignalDeath('SIGABRT', opts);
      expect(reason).toMatch(/native abort with no diagnostic/);
      expect(reason).not.toMatch(/watchdog/i);
    }
  });

  it('keeps OOM, innocent-victim and assertion aborts distinct from the watchdog, even beside a timeout line', () => {
    expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorOutOfMemory', ...MLX })).toMatch(/ran out of memory/);
    expect(describeSignalDeath('SIGABRT', { stderr: 'kIOGPUCommandBufferCallbackErrorInnocentVictim\nCaused GPU Timeout Error', ...MLX })).toMatch(/innocent victim/);
    expect(describeSignalDeath('SIGABRT', { stderr: 'Assertion failed: (x > 0), function f, file a.cpp', ...MLX })).toMatch(/failed assertion/);
  });

  it('names a native MLX/Metal crash for SIGBUS and SIGSEGV', () => {
    for (const sig of ['SIGBUS', 'SIGSEGV']) {
      const reason = describeSignalDeath(sig);
      expect(reason).toMatch(new RegExp(sig));
      expect(reason).toMatch(/MLX\/Metal/);
      expect(reason).toMatch(/native fault/i);
    }
  });

  it('keeps the existing OOM wording for SIGKILL', () => {
    expect(describeSignalDeath('SIGKILL')).toBe('Process killed (likely out of memory — try a smaller model or resolution)');
  });

  it('falls back to the verbatim signal name for an unmapped signal', () => {
    expect(describeSignalDeath('SIGTERM')).toBe('Killed by signal SIGTERM');
    expect(describeSignalDeath('SIGPIPE')).toBe('Killed by signal SIGPIPE');
  });

  it('appends the runtime fingerprint when one is known', () => {
    const reason = describeSignalDeath('SIGABRT', {
      fingerprint: { runtime: 'ltx2', versions: { mlx: '0.22.0', mlx_metal: '0.22.0' }, chip: 'Apple M4 Max', os: 'macOS-15.4-arm64' },
    });
    expect(reason).toMatch(/\[runtime: ltx2 \| mlx 0\.22\.0, mlx_metal 0\.22\.0 \| Apple M4 Max \| macOS-15\.4-arm64\]$/);
  });

  it('omits the fingerprint suffix entirely when nothing is known (no empty brackets)', () => {
    expect(describeSignalDeath('SIGSEGV', { fingerprint: null })).not.toMatch(/runtime:/);
    expect(describeSignalDeath('SIGSEGV', { fingerprint: {} })).not.toMatch(/runtime:/);
  });
});

describe('isWatchdogSuccess', () => {
  // The non-fs short-circuits are pure; the on-disk branch is gated on a real
  // existsSync + non-empty statSync, exercised against actual temp files.
  it('false unless the watchdog actually fired', () => {
    expect(isWatchdogSuccess({ completionWatchdogFired: false, signal: 'SIGKILL', outputPath: '/tmp/x.mp4' })).toBe(false);
  });

  it('false unless the kill signal was SIGKILL', () => {
    expect(isWatchdogSuccess({ completionWatchdogFired: true, signal: 'SIGTERM', outputPath: '/tmp/x.mp4' })).toBe(false);
  });

  it('false when the output file is absent (no real render landed)', () => {
    expect(isWatchdogSuccess({ completionWatchdogFired: true, signal: 'SIGKILL', outputPath: `/tmp/definitely-missing-${process.pid}.mp4` })).toBe(false);
  });

  it('true when the watchdog fired on SIGKILL and a non-empty output exists', () => {
    const p = join(tmpdir(), `wd-success-${process.pid}.mp4`);
    writeFileSync(p, 'x');
    try {
      expect(isWatchdogSuccess({ completionWatchdogFired: true, signal: 'SIGKILL', outputPath: p })).toBe(true);
    } finally {
      rmSync(p, { force: true });
    }
  });

  it('false when the output file exists but is empty (marker without real render)', () => {
    const p = join(tmpdir(), `wd-empty-${process.pid}.mp4`);
    writeFileSync(p, '');
    try {
      expect(isWatchdogSuccess({ completionWatchdogFired: true, signal: 'SIGKILL', outputPath: p })).toBe(false);
    } finally {
      rmSync(p, { force: true });
    }
  });
});

// ── Gemma prompt-encode watchdog retry (#4589) ──────────────────────────────
// A real Metal abort cannot be reproduced in a unit test, so the coverage here
// is over the two decisions PortOS actually makes from the wreckage: which
// abort text counts as a prompt-encode watchdog kill, and whether the render
// may be relaunched.
describe('isPromptEncodingMetalWatchdog', () => {
  // The abort banner MLX prints on the way down, in the two shapes the watchdog
  // uses. The impacting-interactivity one is what newer Apple silicon / macOS
  // report where older combinations reported a timeout — matching only the
  // timeout wording is exactly the portability bug this classifier fixes.
  const TIMEOUT_ABORT = 'libc++abi: terminating due to uncaught exception of type std::runtime_error: [METAL] Command buffer execution failed: Caused GPU Timeout Error (00000002:kIOGPUCommandBufferCallbackErrorTimeout)';
  const INTERACTIVITY_ABORT = 'libc++abi: terminating due to uncaught exception of type std::runtime_error: [METAL] Command buffer execution failed: (00000004:kIOGPUCommandBufferCallbackErrorImpactingInteractivity)';
  const OOM_ABORT = '[METAL] Command buffer execution failed: (00000008:kIOGPUCommandBufferCallbackErrorOutOfMemory)';
  const VICTIM_ABORT = '[METAL] Command buffer execution failed: (00000006:kIOGPUCommandBufferCallbackErrorInnocentVictim)';

  it.each([
    ['the classic timeout signature', TIMEOUT_ABORT],
    ['the impacting-interactivity signature newer macOS reports instead', INTERACTIVITY_ABORT],
  ])('recognizes %s', (_label, stderr) => {
    expect(isPromptEncodingMetalWatchdog({ signal: 'SIGABRT', stderr, promptEncodePhase: 'active' })).toBe(true);
  });

  // Neither is a "this buffer ran too long" kill, so neither is fixed by a
  // shorter prompt: OOM means the machine is out of headroom, and an innocent
  // victim was killed for some OTHER process wedging the GPU.
  it.each([
    ['an out-of-memory abort', OOM_ABORT],
    ['an innocent-victim abort', VICTIM_ABORT],
  ])('excludes %s', (_label, stderr) => {
    expect(isPromptEncodingMetalWatchdog({ signal: 'SIGABRT', stderr, promptEncodePhase: 'active' })).toBe(false);
  });

  // A mixed abort is the case where the timeout is a downstream symptom of the
  // real cause, so the veto wins rather than the match.
  it('excludes an abort that carries both a timeout and an out-of-memory cause', () => {
    expect(isPromptEncodingMetalWatchdog({
      signal: 'SIGABRT',
      stderr: `${TIMEOUT_ABORT}\n${OOM_ABORT}`,
      promptEncodePhase: 'active',
    })).toBe(false);
  });

  // The three phase states must stay distinct: "never reported" is every
  // non-LTX-2 runtime and must never read as "encoding right now".
  it.each([
    ['done — the abort landed in the denoise loop, which a shorter prompt cannot fix', 'done'],
    ['null — this runner never reported an encode boundary at all', null],
    ['undefined — same, expressed as an unstamped job field', undefined],
  ])('refuses to fire when the prompt-encode phase is %s', (_label, promptEncodePhase) => {
    expect(isPromptEncodingMetalWatchdog({ signal: 'SIGABRT', stderr: TIMEOUT_ABORT, promptEncodePhase })).toBe(false);
  });

  // A watchdog kill always arrives as SIGABRT (the Metal layer raises a C++
  // exception that terminate() converts), never as a clean non-zero exit or an
  // OOM SIGKILL.
  it.each([[null], ['SIGKILL'], ['SIGTERM'], ['SIGSEGV']])('refuses to fire on signal %s', (signal) => {
    expect(isPromptEncodingMetalWatchdog({ signal, stderr: TIMEOUT_ABORT, promptEncodePhase: 'active' })).toBe(false);
  });

  it('tolerates a missing/blank stderr tail and an empty call', () => {
    expect(isPromptEncodingMetalWatchdog()).toBe(false);
    expect(isPromptEncodingMetalWatchdog({ signal: 'SIGABRT', promptEncodePhase: 'active' })).toBe(false);
    expect(isPromptEncodingMetalWatchdog({ signal: 'SIGABRT', stderr: null, promptEncodePhase: 'active' })).toBe(false);
  });
});

// The window between "spawn resolved" and "real listeners attached" is an await
// on real file I/O (the accelerator claim handoff). A child that dies in it
// emits unsubscribed — and for 'error' that is not merely lost but fatal, since
// an EventEmitter with no 'error' listener throws.
describe('bufferChildExit', () => {
  it('reports nothing for a child that is still alive', () => {
    expect(bufferChildExit(new EventEmitter())()).toBeNull();
  });

  it('holds a close emitted before the real listeners were attached', () => {
    const proc = new EventEmitter();
    const take = bufferChildExit(proc, { bufferOutput: true });
    proc.emit('close', 3, null);
    expect(take()).toEqual({ type: 'close', code: 3, signal: null });
  });

  it('retains output emitted before the real readers were attached', () => {
    const proc = new EventEmitter();
    proc.stdout = new EventEmitter();
    proc.stderr = new EventEmitter();
    const take = bufferChildExit(proc, { bufferOutput: true });
    proc.stderr.emit('data', Buffer.from('RuntimeError: shader compilation failed\\n'));
    proc.emit('close', 1, null);
    const early = take();
    expect(early).toMatchObject({ type: 'close', code: 1, signal: null });
    expect(early.output).toEqual([{ stream: 'stderr', chunk: expect.any(Buffer) }]);
  });

  it('absorbs an error that would otherwise throw out of the emitter', () => {
    const proc = new EventEmitter();
    const take = bufferChildExit(proc);
    const err = new Error('detached spawn produced no PID');
    expect(() => proc.emit('error', err)).not.toThrow();
    expect(take()).toEqual({ type: 'error', error: err });
  });

  it('keeps the FIRST terminal event — a later one must not rewrite the verdict', () => {
    const proc = new EventEmitter();
    const take = bufferChildExit(proc);
    proc.emit('error', new Error('spawn failed'));
    proc.emit('close', 0, null);
    expect(take().type).toBe('error');
  });

  it('detaches on take, so the real handlers own everything that follows', () => {
    const proc = new EventEmitter();
    const take = bufferChildExit(proc);
    expect(take()).toBeNull();
    expect(proc.listenerCount('close')).toBe(0);
    expect(proc.listenerCount('error')).toBe(0);
  });

  it('stays a sink for a child that is abandoned rather than wired', () => {
    const proc = new EventEmitter();
    bufferChildExit(proc);
    expect(() => proc.emit('error', new Error('abandoned'))).not.toThrow();
  });
});


// The cloud lanes used to broadcast their status lines on the provider job's
// own SSE stream ONLY. The Video Gen page listens to the media-job queue, which
// relays videoGenEvents — so it saw no status frame at all between "Starting
// render…" and completion, and its step list had no phase to advance on.
describe('emitCloudRenderStatus', () => {
  it('publishes the phase on the provider SSE stream and on videoGenEvents', () => {
    emitted.length = 0;
    sse.mockClear();
    const job = { clients: [], status: 'running' };

    emitCloudRenderStatus(job, 'job-1', CLOUD_RENDER_PHASE.RENDER, 'Reactor session rendering…');

    expect(sse).toHaveBeenCalledWith(job, { type: 'status', message: 'Reactor session rendering…', phase: 'render' });
    expect(emitted).toEqual([
      { type: 'activity', payload: { generationId: 'job-1' } },
      { type: 'status', payload: { generationId: 'job-1', message: 'Reactor session rendering…', phase: 'render' } },
    ]);
  });

  // grok narrates on every stdout chunk and fal re-reports the same poll status
  // every couple of seconds. The heartbeat has to land each time (it is what
  // the idle watchdog reads); the unchanged line does not.
  it('repeats the heartbeat but not an unchanged line', () => {
    emitted.length = 0;
    sse.mockClear();
    const job = { clients: [], status: 'running' };

    emitCloudRenderStatus(job, 'job-2', CLOUD_RENDER_PHASE.RENDER, 'Running…');
    emitCloudRenderStatus(job, 'job-2', CLOUD_RENDER_PHASE.RENDER, 'Running…');
    emitCloudRenderStatus(job, 'job-2', CLOUD_RENDER_PHASE.FETCH, 'Downloading video…');

    expect(emitted.filter((e) => e.type === 'activity')).toHaveLength(3);
    expect(emitted.filter((e) => e.type === 'status').map((e) => e.payload.phase)).toEqual(['render', 'fetch']);
    expect(sse).toHaveBeenCalledTimes(2);
  });
});

// The gates on the post-completion teardown recovery (#6501). The recovery
// itself is asserted end to end in local.test.js; these pin the two things that
// suite cannot reach cheaply — the host/runtime gate (every other runtime and
// every non-macOS host must keep its current outcome, and CI runs on Windows)
// and the frame/duration tolerance that separates "the muxer rounded" from
// "the file is truncated".
describe('isNativeTeardownAbort', () => {
  const ABORT = { runtime: POST_COMPLETION_ABORT_RUNTIME, signal: 'SIGABRT', childKilled: false, platform: 'darwin' };

  it('recognizes the H3 MLX teardown abort', () => {
    expect(isNativeTeardownAbort(ABORT)).toBe(true);
  });

  it.each([
    ['a non-macOS host, where MLX does not run', { platform: 'win32' }],
    ['the CUDA sibling runtime', { runtime: 'minimax_h3_cuda' }],
    ['the native Ref2VA runtime', { runtime: 'minimax_h3_ref2va' }],
    ['another local runtime entirely', { runtime: 'ltx2' }],
    ['a child PortOS killed on purpose', { childKilled: true }],
    ['an OOM/watchdog SIGKILL', { signal: 'SIGKILL' }],
    ['a native crash that can equally land mid-render', { signal: 'SIGSEGV' }],
    ['an ordinary exit with no signal at all', { signal: null }],
  ])('refuses %s', (_label, override) => {
    expect(isNativeTeardownAbort({ ...ABORT, ...override })).toBe(false);
  });

  it('refuses an empty call rather than defaulting open', () => {
    expect(isNativeTeardownAbort()).toBe(false);
  });
});

describe('verifyPostCompletionOutputs', () => {
  const dir = join(tmpdir(), `portos-teardown-${Date.now()}`);
  const outputPath = join(dir, 'clip.mp4');
  const EXPECTED_FRAMES = 124;
  const FPS = 24;
  let startedAtMs;

  const verify = () => verifyPostCompletionOutputs({
    outputPaths: [outputPath],
    completedOutputs: new Set([outputPath]),
    startedAtMs,
    expectedFrames: EXPECTED_FRAMES,
    fps: FPS,
  });

  beforeEach(() => {
    rmSync(dir, { recursive: true, force: true });
    mkdirSync(dir, { recursive: true });
    startedAtMs = Date.now();
    writeFileSync(outputPath, 'mp4-bytes', { flag: 'w' });
    probeMock.frames.mockResolvedValue(EXPECTED_FRAMES);
    probeMock.duration.mockResolvedValue(EXPECTED_FRAMES / FPS);
  });

  // beforeEach removes-then-recreates `dir` before each test, so the LAST
  // test's directory is never removed — clean it once after the suite (#9032).
  afterAll(() => {
    rmSync(dir, { recursive: true, force: true });
  });

  it('accepts an exact match', async () => {
    await expect(verify()).resolves.toEqual({ ok: true });
  });

  // The muxer can round a stream's reported length, and a clip carrying audio
  // is not exactly frames/fps long. Neither is a truncated file.
  it.each([
    ['a frame count two off the request', { frames: EXPECTED_FRAMES - 2 }, true],
    ['a duration a hair under the request', { duration: (EXPECTED_FRAMES / FPS) - 0.2 }, true],
    ['a clip missing a third of its frames', { frames: 80 }, false],
    ['a clip cut to a couple of seconds', { duration: 2 }, false],
  ])('%s', async (_label, override, ok) => {
    if (override.frames != null) probeMock.frames.mockResolvedValue(override.frames);
    if (override.duration != null) probeMock.duration.mockResolvedValue(override.duration);
    expect((await verify()).ok).toBe(ok);
  });

  // Sentinel discipline: a probe that could not answer is "unknown", never
  // "consistent" — a host with no ffprobe must not auto-recover every abort.
  it.each([
    ['the frame probe', () => probeMock.frames.mockResolvedValue(null)],
    ['the duration probe', () => probeMock.duration.mockResolvedValue(null)],
  ])('refuses when %s could not answer', async (_label, breakProbe) => {
    breakProbe();
    expect((await verify()).ok).toBe(false);
  });

  it('refuses when the render start was never stamped', async () => {
    startedAtMs = null;
    const verdict = await verify();
    expect(verdict.ok).toBe(false);
    expect(verdict.reason).toMatch(/start time/i);
  });

  it('refuses an output the child never reported by name', async () => {
    const verdict = await verifyPostCompletionOutputs({
      outputPaths: [outputPath],
      completedOutputs: new Set([join(dir, 'someone-elses.mp4')]),
      startedAtMs,
      expectedFrames: EXPECTED_FRAMES,
      fps: FPS,
    });
    expect(verdict.ok).toBe(false);
    expect(verdict.reason).toMatch(/never reported complete/i);
  });

  it('refuses a batch where only some outputs are present', async () => {
    const second = join(dir, 'clip-2.mp4');
    const verdict = await verifyPostCompletionOutputs({
      outputPaths: [outputPath, second],
      completedOutputs: new Set([outputPath, second]),
      startedAtMs,
      expectedFrames: EXPECTED_FRAMES,
      fps: FPS,
    });
    expect(verdict.ok).toBe(false);
    expect(verdict.reason).toMatch(/output 2\/2 is not on disk/);
  });
});