Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
[issue-11672] Safely publish ingested audio across filesystems (#11710)
[._.]/e5416931a7a9
diff --git a/server/services/youtubeIngest.js b/server/services/youtubeIngest.jsindex 1e52a9d00c..0ab4de7f22 100644--- a/server/services/youtubeIngest.js+++ b/server/services/youtubeIngest.js@@ -37,12 +37,22 @@*/import { spawn } from '../lib/childProcess.js';-import { readdir, readFile, rename, rm, stat, unlink } from 'fs/promises';+import { readdir, readFile, rm, stat, unlink } from 'fs/promises';import { tmpdir } from 'os';import { join } from 'path';import { randomUUID } from 'crypto';import { ServerError } from '../lib/errorHandler.js';-import { atomicWrite, ensureDir, PATHS, readJSONFile, shortId, tryReadFile } from '../lib/fileUtils.js';+import {+ atomicWrite,+ copyFileGuarded,+ ensureDir,+ PATHS,+ readJSONFile,+ renameGuarded,+ shortId,+ tryReadFile,+ unlinkGuarded,+} from '../lib/fileUtils.js';import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';import { findFfmpeg } from '../lib/ffmpeg.js';import { findYtDlp } from '../lib/ytdlp.js';@@ -52,7 +62,7 @@ import { attachSseClient as attachSse, broadcastSse, closeJobAfterDelay } from 'import { vttToPlainText } from '../lib/vttTranscript.js';import { createMutex } from '../lib/asyncMutex.js';import { createSettingsStore } from '../lib/settingsStore.js';-import { downloadAudioToTempMp3 } from './ytdlpAudioImport.js';+import { cleanupYtDlpTemp, downloadAudioToTempMp3 } from './ytdlpAudioImport.js';import { downloadVideoIntoLibrary } from './videoDownload.js';import { YOUTUBE_VIDEO_URL_RE } from '../lib/youtubeUrl.js';import { assertYoutubeVideoUrl } from '../lib/youtubeUrlAssert.js';@@ -462,6 +472,12 @@ export async function startYoutubeIngest({(async () => {const registerProcess = (proc) => { job.process = proc; };const stage = (name, percent) => broadcastSse(job, { type: 'progress', stage: name, ...(percent != null ? { percent } : {}) });+ const tempPrefix = ingestAudio ? `portos-yt-ingest-${jobId}` : null;+ // Copy the downloaded bytes to a unique sibling before atomically replacing+ // the durable name. A failed/partial copy must not truncate audio from a+ // prior ingest of this video.+ const audioStagingPath = ingestAudio ? join(INGEST_DIR, `.${jobId}.mp3.part`) : null;+ let terminalEvent = null;// Non-fatal outcomes the user still needs told about (no captions, duplicate// task). Broadcast live AND replayed on the terminal frame — a client using// the generic single-slot SSE hook only reads the terminal payload, so a@@ -589,7 +605,6 @@ export async function startYoutubeIngest({if (job.canceled) throw new Error('canceled');if (ingestAudio) {stage('audio', 0);- const tempPrefix = `portos-yt-ingest-${jobId}`;const result = await downloadAudioToTempMp3({url,ytDlp,@@ -605,9 +620,16 @@ export async function startYoutubeIngest({// Kept next to the transcript rather than pushed into the music library:// a two-hour talk is reference material, not a track.const audioPath = join(INGEST_DIR, `${meta.videoId}.mp3`);- await rename(result.outPath, audioPath);+ // Copy to same-filesystem staging before publishing, so cross-device+ // yt-dlp output and failed copies cannot damage prior audio.+ await withBackupAssetPublication(async () => {+ await copyFileGuarded(result.outPath, audioStagingPath);+ if (job.canceled) throw new Error('canceled');+ await renameGuarded(audioStagingPath, audioPath);+ });const bytes = await stat(audioPath).then((s) => s.size).catch(() => null);landed.audio = { path: audioPath, bytes };+ await unlinkGuarded(result.outPath).catch(() => {});}// ── Persist the "I consumed and kept this" artifacts ──@@ -714,7 +736,7 @@ export async function startYoutubeIngest({});console.log(`📺 YouTube ingest ${shortId(jobId)} complete — "${meta.title}"${taskId ? ` → task ${taskId}` : ''}`);- broadcastSse(job, { type: 'complete', ingest: record, warnings });+ terminalEvent = { type: 'complete', ingest: record, warnings };} catch (err) {const message = err?.message || String(err);// Record whatever DID land before the failure. A transcript written (and@@ -735,15 +757,31 @@ export async function startYoutubeIngest({}if (message === 'canceled' || job.canceled) {console.log(`🛑 YouTube ingest ${shortId(jobId)} cancelled`);- broadcastSse(job, { type: 'canceled' });+ terminalEvent = { type: 'canceled' };} else {console.error(`❌ YouTube ingest ${shortId(jobId)} failed: ${message}`);- broadcastSse(job, { type: 'error', error: message });+ terminalEvent = { type: 'error', error: message };}} finally {// One rm regardless of outcome — the caption files are scratch, and on// cancel/failure yt-dlp may have written some of them.if (subsDir) await rm(subsDir, { recursive: true, force: true }).catch(() => {});+ // These paths are scoped to this job id. Remove only this job's exact+ // staging file and yt-dlp prefix; durable transcript/audio files are+ // intentionally left for the index/delete workflow to own.+ if (audioStagingPath) {+ await unlinkGuarded(audioStagingPath).catch((error) => {+ if (error.code !== 'ENOENT') {+ console.error(`🧹 YouTube ingest staging cleanup failed for ${shortId(jobId)}: ${error.message}`);+ }+ });+ }+ if (tempPrefix) {+ await cleanupYtDlpTemp(tempPrefix).catch((error) => {+ console.error(`🧹 YouTube ingest temp cleanup failed for ${shortId(jobId)}: ${error.message}`);+ });+ }+ if (terminalEvent) broadcastSse(job, terminalEvent);closeJobAfterDelay(ingestJobs, jobId);}})();diff --git a/server/services/youtubeIngest.test.js b/server/services/youtubeIngest.test.jsindex ef77fe1b0e..586855b925 100644--- a/server/services/youtubeIngest.test.js+++ b/server/services/youtubeIngest.test.js@@ -1,4 +1,17 @@-import { describe, it, expect, vi, beforeEach } from 'vitest';+import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest';+import { EventEmitter } from 'node:events';+import { PassThrough } from 'node:stream';+import { readdir, readFile, rm, writeFile } from 'fs/promises';+import { tmpdir } from 'os';+import { basename, join } from 'path';++const testDoubles = vi.hoisted(() => ({+ spawn: vi.fn(),+ downloadAudioToTempMp3: vi.fn(),+ copyFileGuarded: vi.fn(),+ unlinkGuarded: vi.fn(),+ rename: vi.fn(),+}));// The ingest module pulls in the brain/cos/media graphs at import time for its// spawn path. Stub the heavy edges so this suite exercises the service-level@@ -7,13 +20,46 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';// none of this.vi.mock('./apps.js', () => ({ getAppById: vi.fn(), PORTOS_APP_ID: 'portos-default' }));vi.mock('./brain.js', () => ({ createLinkFromUrl: vi.fn() }));-vi.mock('./brainStorage.js', () => ({ getLinkByUrl: vi.fn() }));+vi.mock('./brainStorage.js', () => ({ getLinkById: vi.fn(), getLinkByUrl: vi.fn() }));vi.mock('./brainJournal.js', () => ({ getSettings: vi.fn(async () => ({ obsidianVaultId: null })) }));vi.mock('./cosTaskStore.js', () => ({ addTask: vi.fn() }));vi.mock('./humanActivity.js', () => ({ recordEvents: vi.fn() }));vi.mock('./videoGen/history.js', () => ({ mutateVideoHistory: vi.fn() }));vi.mock('./videoGen/events.js', () => ({ videoGenEvents: { emit: vi.fn() } }));vi.mock('./videoDownload.js', () => ({ buildDownloadHistoryEntry: vi.fn() }));+vi.mock('../lib/childProcess.js', async (importOriginal) => ({+ ...(await importOriginal()),+ spawn: (...args) => testDoubles.spawn(...args),+}));+vi.mock('../lib/ffmpeg.js', async (importOriginal) => ({+ ...(await importOriginal()),+ findFfmpeg: vi.fn(async () => '/test/ffmpeg'),+}));+vi.mock('../lib/ytdlp.js', async (importOriginal) => ({+ ...(await importOriginal()),+ findYtDlp: vi.fn(async () => '/test/yt-dlp'),+}));+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: vi.fn((work) => work()) }));+vi.mock('fs/promises', async (importOriginal) => ({+ ...(await importOriginal()),+ rename: (...args) => testDoubles.rename(...args),+}));+vi.mock('../lib/fileUtils.js', async (importOriginal) => {+ const actual = await importOriginal();+ const { lazyTempDataRoot, makePathsProxy } = await import('../lib/mockPathsDataRoot.js');+ const proxied = makePathsProxy(actual, { dataRoot: () => lazyTempDataRoot('youtube-ingest-workflow-') });+ return new Proxy(proxied, {+ get(target, prop, receiver) {+ if (prop === 'copyFileGuarded') return testDoubles.copyFileGuarded;+ if (prop === 'unlinkGuarded') return testDoubles.unlinkGuarded;+ return Reflect.get(target, prop, receiver);+ },+ });+});+vi.mock('./ytdlpAudioImport.js', async (importOriginal) => ({+ ...(await importOriginal()),+ downloadAudioToTempMp3: (...args) => testDoubles.downloadAudioToTempMp3(...args),+}));import {startYoutubeIngest,@@ -23,6 +69,11 @@ import {__testing,} from './youtubeIngest.js';+afterAll(async () => {+ const { cleanupTempDataRoots } = await import('../lib/mockPathsDataRoot.js');+ cleanupTempDataRoots();+});+describe('YouTube ingest URL allowlist', () => {it('accepts every single-video URL shape', () => {for (const url of [@@ -99,3 +150,177 @@ describe('analysis app admission', () => {}});});++describe('YouTube ingest audio handoff', () => {+ let actualFileUtils;+ let actualFs;+ let downloadedPaths;+ let unrelatedTempPath;+ let audioBytes;++ beforeEach(async () => {+ __testing.ingestJobs.clear();+ actualFileUtils = await vi.importActual('../lib/fileUtils.js');+ actualFs = await vi.importActual('fs/promises');+ downloadedPaths = [];+ unrelatedTempPath = null;+ audioBytes = Buffer.from('complete mp3 bytes');++ testDoubles.spawn.mockReset().mockImplementation(() => {+ const proc = new EventEmitter();+ proc.stdout = new PassThrough();+ proc.stderr = new PassThrough();+ proc.kill = vi.fn();+ queueMicrotask(() => {+ proc.stdout.end(JSON.stringify({ id: 'oCnxnaVg0bY', title: 'Ingested talk', channel: 'Test channel', duration: 120 }));+ proc.stderr.end();+ proc.emit('close', 0, null);+ });+ return proc;+ });+ testDoubles.downloadAudioToTempMp3.mockReset().mockImplementation(async ({ tempPrefix }) => {+ const outPath = join(tmpdir(), `${tempPrefix}.mp3`);+ const intermediatePath = join(tmpdir(), `${tempPrefix}.webm`);+ await writeFile(outPath, audioBytes);+ await writeFile(intermediatePath, 'yt-dlp intermediate');+ downloadedPaths.push(outPath, intermediatePath);+ return { outcome: 'complete', outPath, title: 'Ingested talk' };+ });+ testDoubles.copyFileGuarded.mockReset().mockImplementation((...args) => actualFileUtils.copyFileGuarded(...args));+ testDoubles.unlinkGuarded.mockReset().mockImplementation((...args) => actualFileUtils.unlinkGuarded(...args));+ testDoubles.rename.mockReset().mockImplementation((from, to) => {+ if (basename(from).startsWith('portos-yt-ingest-')) {+ throw Object.assign(new Error('cross-device rename'), { code: 'EXDEV' });+ }+ return actualFs.rename(from, to);+ });+ const { getLinkById, getLinkByUrl } = await import('./brainStorage.js');+ getLinkById.mockResolvedValue(null);+ getLinkByUrl.mockResolvedValue(null);+ const { createLinkFromUrl } = await import('./brain.js');+ createLinkFromUrl.mockResolvedValue({ id: 'test-link' });+ const { recordEvents } = await import('./humanActivity.js');+ recordEvents.mockResolvedValue(undefined);+ });++ afterEach(async () => {+ await Promise.all([...downloadedPaths, unrelatedTempPath].filter(Boolean).map((path) => rm(path, { force: true })));+ });++ async function runAudioIngest() {+ const { attachIngestSseClient } = await import('./youtubeIngest.js');+ let resolveTerminal;+ const terminal = new Promise((resolve) => { resolveTerminal = resolve; });+ const response = {+ req: new EventEmitter(),+ writeHead: vi.fn(),+ end: vi.fn(),+ write: (frame) => {+ const payload = JSON.parse(frame.replace(/^data: /, '').trim());+ if (['complete', 'error', 'canceled'].includes(payload.type)) {+ // Terminal SSE must not race owned-temp cleanup. Observe files at+ // the public boundary instead of polling after the job completes.+ Promise.all(downloadedPaths.map(async (path) => {+ try { await readFile(path); return true; }+ catch (error) { if (error.code === 'ENOENT') return false; throw error; }+ })).then((tempPresence) => resolveTerminal({ payload, tempPresence }));+ }+ },+ };+ const { jobId } = await startYoutubeIngest({+ url: 'https://youtu.be/oCnxnaVg0bY',+ captureTranscript: false,+ ingestAudio: true,+ });+ expect(attachIngestSseClient(jobId, response)).toBe(true);+ return { terminal, response, jobId };+ }++ it('copies completed audio into the ingest directory without renaming across filesystems', async () => {+ const { terminal } = await runAudioIngest();+ const { payload, tempPresence } = await terminal;+ expect(payload.type).toBe('complete');+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));++ const { PATHS } = await import('../lib/fileUtils.js');+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');+ expect(await readFile(audioPath)).toEqual(audioBytes);+ expect(testDoubles.copyFileGuarded).toHaveBeenCalledWith(downloadedPaths[0], expect.stringContaining('.mp3.part'));+ expect(testDoubles.rename).not.toHaveBeenCalledWith(downloadedPaths[0], audioPath);+ for (const path of downloadedPaths) await expect(readFile(path)).rejects.toMatchObject({ code: 'ENOENT' });+ });++ it('cleans this job after a post-download copy failure and preserves prior audio', async () => {+ const { PATHS } = await import('../lib/fileUtils.js');+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');+ await actualFs.mkdir(join(PATHS.brain, 'youtube'), { recursive: true });+ await writeFile(audioPath, 'previous durable audio');+ unrelatedTempPath = join(tmpdir(), `portos-yt-ingest-other-${Date.now()}.mp3`);+ await writeFile(unrelatedTempPath, 'other job');+ testDoubles.copyFileGuarded.mockImplementationOnce(async (source, staging) => {+ await actualFileUtils.copyFileGuarded(source, staging);+ await writeFile(staging, 'partial');+ throw new Error('copy interrupted');+ });++ const { terminal } = await runAudioIngest();+ const { payload, tempPresence } = await terminal;+ expect(payload).toMatchObject({ type: 'error', error: 'copy interrupted' });+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));+ expect(await readFile(audioPath, 'utf-8')).toBe('previous durable audio');+ for (const path of downloadedPaths) await expect(readFile(path)).rejects.toMatchObject({ code: 'ENOENT' });+ expect(await readFile(unrelatedTempPath, 'utf-8')).toBe('other job');+ });++ it('honors cancellation during staging before replacing prior audio', async () => {+ const { PATHS } = await import('../lib/fileUtils.js');+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');+ await actualFs.mkdir(join(PATHS.brain, 'youtube'), { recursive: true });+ await writeFile(audioPath, 'previous durable audio');++ let copyStarted;+ const started = new Promise((resolve) => { copyStarted = resolve; });+ let releaseCopy;+ const copyGate = new Promise((resolve) => { releaseCopy = resolve; });+ testDoubles.copyFileGuarded.mockImplementationOnce(async (source, staging) => {+ await actualFileUtils.copyFileGuarded(source, staging);+ copyStarted();+ await copyGate;+ });++ const { terminal, jobId } = await runAudioIngest();+ await started;+ expect(cancelYoutubeIngest(jobId)).toBe(true);+ releaseCopy();++ const { payload, tempPresence } = await terminal;+ expect(payload.type).toBe('canceled');+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));+ expect(await readFile(audioPath, 'utf-8')).toBe('previous durable audio');+ await expect(readFile(join(PATHS.brain, 'youtube', `.${jobId}.mp3.part`))).rejects.toMatchObject({ code: 'ENOENT' });+ });++ it('logs a staging cleanup failure without failing published audio', async () => {+ const cleanupError = Object.assign(new Error('permission denied'), { code: 'EACCES' });+ testDoubles.unlinkGuarded.mockImplementation(async (path) => {+ if (path.endsWith('.mp3.part')) throw cleanupError;+ return actualFileUtils.unlinkGuarded(path);+ });+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});++ try {+ const { terminal } = await runAudioIngest();+ const { payload } = await terminal;+ expect(payload.type).toBe('complete');+ expect(errorLog).toHaveBeenCalledWith(expect.stringMatching(/YouTube ingest staging cleanup failed.*permission denied/));++ const { PATHS } = await import('../lib/fileUtils.js');+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');+ expect(await readFile(audioPath)).toEqual(audioBytes);+ } finally {+ errorLog.mockRestore();+ const stagingPath = testDoubles.copyFileGuarded.mock.calls.at(-1)?.[1];+ if (stagingPath) await actualFileUtils.unlinkGuarded(stagingPath).catch(() => {});+ }+ });+});