Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

[issue-11672] Safely publish ingested audio across filesystems (#11710)

[._.]/e5416931a7a9
diff --git a/server/services/youtubeIngest.js b/server/services/youtubeIngest.js
index 1e52a9d00c..0ab4de7f22 100644
--- a/server/services/youtubeIngest.js
+++ b/server/services/youtubeIngest.js
@@ -37,12 +37,22 @@
*/
import { spawn } from '../lib/childProcess.js';
-import { readdir, readFile, rename, rm, stat, unlink } from 'fs/promises';
+import { readdir, readFile, rm, stat, unlink } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { randomUUID } from 'crypto';
import { ServerError } from '../lib/errorHandler.js';
-import { atomicWrite, ensureDir, PATHS, readJSONFile, shortId, tryReadFile } from '../lib/fileUtils.js';
+import {
+ atomicWrite,
+ copyFileGuarded,
+ ensureDir,
+ PATHS,
+ readJSONFile,
+ renameGuarded,
+ shortId,
+ tryReadFile,
+ unlinkGuarded,
+} from '../lib/fileUtils.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { findFfmpeg } from '../lib/ffmpeg.js';
import { findYtDlp } from '../lib/ytdlp.js';
@@ -52,7 +62,7 @@ import { attachSseClient as attachSse, broadcastSse, closeJobAfterDelay } from '
import { vttToPlainText } from '../lib/vttTranscript.js';
import { createMutex } from '../lib/asyncMutex.js';
import { createSettingsStore } from '../lib/settingsStore.js';
-import { downloadAudioToTempMp3 } from './ytdlpAudioImport.js';
+import { cleanupYtDlpTemp, downloadAudioToTempMp3 } from './ytdlpAudioImport.js';
import { downloadVideoIntoLibrary } from './videoDownload.js';
import { YOUTUBE_VIDEO_URL_RE } from '../lib/youtubeUrl.js';
import { assertYoutubeVideoUrl } from '../lib/youtubeUrlAssert.js';
@@ -462,6 +472,12 @@ export async function startYoutubeIngest({
(async () => {
const registerProcess = (proc) => { job.process = proc; };
const stage = (name, percent) => broadcastSse(job, { type: 'progress', stage: name, ...(percent != null ? { percent } : {}) });
+ const tempPrefix = ingestAudio ? `portos-yt-ingest-${jobId}` : null;
+ // Copy the downloaded bytes to a unique sibling before atomically replacing
+ // the durable name. A failed/partial copy must not truncate audio from a
+ // prior ingest of this video.
+ const audioStagingPath = ingestAudio ? join(INGEST_DIR, `.${jobId}.mp3.part`) : null;
+ let terminalEvent = null;
// Non-fatal outcomes the user still needs told about (no captions, duplicate
// task). Broadcast live AND replayed on the terminal frame — a client using
// the generic single-slot SSE hook only reads the terminal payload, so a
@@ -589,7 +605,6 @@ export async function startYoutubeIngest({
if (job.canceled) throw new Error('canceled');
if (ingestAudio) {
stage('audio', 0);
- const tempPrefix = `portos-yt-ingest-${jobId}`;
const result = await downloadAudioToTempMp3({
url,
ytDlp,
@@ -605,9 +620,16 @@ export async function startYoutubeIngest({
// Kept next to the transcript rather than pushed into the music library:
// a two-hour talk is reference material, not a track.
const audioPath = join(INGEST_DIR, `${meta.videoId}.mp3`);
- await rename(result.outPath, audioPath);
+ // Copy to same-filesystem staging before publishing, so cross-device
+ // yt-dlp output and failed copies cannot damage prior audio.
+ await withBackupAssetPublication(async () => {
+ await copyFileGuarded(result.outPath, audioStagingPath);
+ if (job.canceled) throw new Error('canceled');
+ await renameGuarded(audioStagingPath, audioPath);
+ });
const bytes = await stat(audioPath).then((s) => s.size).catch(() => null);
landed.audio = { path: audioPath, bytes };
+ await unlinkGuarded(result.outPath).catch(() => {});
}
// ── Persist the "I consumed and kept this" artifacts ──
@@ -714,7 +736,7 @@ export async function startYoutubeIngest({
});
console.log(`📺 YouTube ingest ${shortId(jobId)} complete — "${meta.title}"${taskId ? ` → task ${taskId}` : ''}`);
- broadcastSse(job, { type: 'complete', ingest: record, warnings });
+ terminalEvent = { type: 'complete', ingest: record, warnings };
} catch (err) {
const message = err?.message || String(err);
// Record whatever DID land before the failure. A transcript written (and
@@ -735,15 +757,31 @@ export async function startYoutubeIngest({
}
if (message === 'canceled' || job.canceled) {
console.log(`🛑 YouTube ingest ${shortId(jobId)} cancelled`);
- broadcastSse(job, { type: 'canceled' });
+ terminalEvent = { type: 'canceled' };
} else {
console.error(`❌ YouTube ingest ${shortId(jobId)} failed: ${message}`);
- broadcastSse(job, { type: 'error', error: message });
+ terminalEvent = { type: 'error', error: message };
}
} finally {
// One rm regardless of outcome — the caption files are scratch, and on
// cancel/failure yt-dlp may have written some of them.
if (subsDir) await rm(subsDir, { recursive: true, force: true }).catch(() => {});
+ // These paths are scoped to this job id. Remove only this job's exact
+ // staging file and yt-dlp prefix; durable transcript/audio files are
+ // intentionally left for the index/delete workflow to own.
+ if (audioStagingPath) {
+ await unlinkGuarded(audioStagingPath).catch((error) => {
+ if (error.code !== 'ENOENT') {
+ console.error(`🧹 YouTube ingest staging cleanup failed for ${shortId(jobId)}: ${error.message}`);
+ }
+ });
+ }
+ if (tempPrefix) {
+ await cleanupYtDlpTemp(tempPrefix).catch((error) => {
+ console.error(`🧹 YouTube ingest temp cleanup failed for ${shortId(jobId)}: ${error.message}`);
+ });
+ }
+ if (terminalEvent) broadcastSse(job, terminalEvent);
closeJobAfterDelay(ingestJobs, jobId);
}
})();
diff --git a/server/services/youtubeIngest.test.js b/server/services/youtubeIngest.test.js
index ef77fe1b0e..586855b925 100644
--- a/server/services/youtubeIngest.test.js
+++ b/server/services/youtubeIngest.test.js
@@ -1,4 +1,17 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest';
+import { EventEmitter } from 'node:events';
+import { PassThrough } from 'node:stream';
+import { readdir, readFile, rm, writeFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { basename, join } from 'path';
+
+const testDoubles = vi.hoisted(() => ({
+ spawn: vi.fn(),
+ downloadAudioToTempMp3: vi.fn(),
+ copyFileGuarded: vi.fn(),
+ unlinkGuarded: vi.fn(),
+ rename: vi.fn(),
+}));
// The ingest module pulls in the brain/cos/media graphs at import time for its
// spawn path. Stub the heavy edges so this suite exercises the service-level
@@ -7,13 +20,46 @@ import { describe, it, expect, vi, beforeEach } from 'vitest';
// none of this.
vi.mock('./apps.js', () => ({ getAppById: vi.fn(), PORTOS_APP_ID: 'portos-default' }));
vi.mock('./brain.js', () => ({ createLinkFromUrl: vi.fn() }));
-vi.mock('./brainStorage.js', () => ({ getLinkByUrl: vi.fn() }));
+vi.mock('./brainStorage.js', () => ({ getLinkById: vi.fn(), getLinkByUrl: vi.fn() }));
vi.mock('./brainJournal.js', () => ({ getSettings: vi.fn(async () => ({ obsidianVaultId: null })) }));
vi.mock('./cosTaskStore.js', () => ({ addTask: vi.fn() }));
vi.mock('./humanActivity.js', () => ({ recordEvents: vi.fn() }));
vi.mock('./videoGen/history.js', () => ({ mutateVideoHistory: vi.fn() }));
vi.mock('./videoGen/events.js', () => ({ videoGenEvents: { emit: vi.fn() } }));
vi.mock('./videoDownload.js', () => ({ buildDownloadHistoryEntry: vi.fn() }));
+vi.mock('../lib/childProcess.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ spawn: (...args) => testDoubles.spawn(...args),
+}));
+vi.mock('../lib/ffmpeg.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ findFfmpeg: vi.fn(async () => '/test/ffmpeg'),
+}));
+vi.mock('../lib/ytdlp.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ findYtDlp: vi.fn(async () => '/test/yt-dlp'),
+}));
+vi.mock('../lib/backupSnapshotBoundary.js', () => ({ withBackupAssetPublication: vi.fn((work) => work()) }));
+vi.mock('fs/promises', async (importOriginal) => ({
+ ...(await importOriginal()),
+ rename: (...args) => testDoubles.rename(...args),
+}));
+vi.mock('../lib/fileUtils.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ const { lazyTempDataRoot, makePathsProxy } = await import('../lib/mockPathsDataRoot.js');
+ const proxied = makePathsProxy(actual, { dataRoot: () => lazyTempDataRoot('youtube-ingest-workflow-') });
+ return new Proxy(proxied, {
+ get(target, prop, receiver) {
+ if (prop === 'copyFileGuarded') return testDoubles.copyFileGuarded;
+ if (prop === 'unlinkGuarded') return testDoubles.unlinkGuarded;
+ return Reflect.get(target, prop, receiver);
+ },
+ });
+});
+vi.mock('./ytdlpAudioImport.js', async (importOriginal) => ({
+ ...(await importOriginal()),
+ downloadAudioToTempMp3: (...args) => testDoubles.downloadAudioToTempMp3(...args),
+}));
import {
startYoutubeIngest,
@@ -23,6 +69,11 @@ import {
__testing,
} from './youtubeIngest.js';
+afterAll(async () => {
+ const { cleanupTempDataRoots } = await import('../lib/mockPathsDataRoot.js');
+ cleanupTempDataRoots();
+});
+
describe('YouTube ingest URL allowlist', () => {
it('accepts every single-video URL shape', () => {
for (const url of [
@@ -99,3 +150,177 @@ describe('analysis app admission', () => {
}
});
});
+
+describe('YouTube ingest audio handoff', () => {
+ let actualFileUtils;
+ let actualFs;
+ let downloadedPaths;
+ let unrelatedTempPath;
+ let audioBytes;
+
+ beforeEach(async () => {
+ __testing.ingestJobs.clear();
+ actualFileUtils = await vi.importActual('../lib/fileUtils.js');
+ actualFs = await vi.importActual('fs/promises');
+ downloadedPaths = [];
+ unrelatedTempPath = null;
+ audioBytes = Buffer.from('complete mp3 bytes');
+
+ testDoubles.spawn.mockReset().mockImplementation(() => {
+ const proc = new EventEmitter();
+ proc.stdout = new PassThrough();
+ proc.stderr = new PassThrough();
+ proc.kill = vi.fn();
+ queueMicrotask(() => {
+ proc.stdout.end(JSON.stringify({ id: 'oCnxnaVg0bY', title: 'Ingested talk', channel: 'Test channel', duration: 120 }));
+ proc.stderr.end();
+ proc.emit('close', 0, null);
+ });
+ return proc;
+ });
+ testDoubles.downloadAudioToTempMp3.mockReset().mockImplementation(async ({ tempPrefix }) => {
+ const outPath = join(tmpdir(), `${tempPrefix}.mp3`);
+ const intermediatePath = join(tmpdir(), `${tempPrefix}.webm`);
+ await writeFile(outPath, audioBytes);
+ await writeFile(intermediatePath, 'yt-dlp intermediate');
+ downloadedPaths.push(outPath, intermediatePath);
+ return { outcome: 'complete', outPath, title: 'Ingested talk' };
+ });
+ testDoubles.copyFileGuarded.mockReset().mockImplementation((...args) => actualFileUtils.copyFileGuarded(...args));
+ testDoubles.unlinkGuarded.mockReset().mockImplementation((...args) => actualFileUtils.unlinkGuarded(...args));
+ testDoubles.rename.mockReset().mockImplementation((from, to) => {
+ if (basename(from).startsWith('portos-yt-ingest-')) {
+ throw Object.assign(new Error('cross-device rename'), { code: 'EXDEV' });
+ }
+ return actualFs.rename(from, to);
+ });
+ const { getLinkById, getLinkByUrl } = await import('./brainStorage.js');
+ getLinkById.mockResolvedValue(null);
+ getLinkByUrl.mockResolvedValue(null);
+ const { createLinkFromUrl } = await import('./brain.js');
+ createLinkFromUrl.mockResolvedValue({ id: 'test-link' });
+ const { recordEvents } = await import('./humanActivity.js');
+ recordEvents.mockResolvedValue(undefined);
+ });
+
+ afterEach(async () => {
+ await Promise.all([...downloadedPaths, unrelatedTempPath].filter(Boolean).map((path) => rm(path, { force: true })));
+ });
+
+ async function runAudioIngest() {
+ const { attachIngestSseClient } = await import('./youtubeIngest.js');
+ let resolveTerminal;
+ const terminal = new Promise((resolve) => { resolveTerminal = resolve; });
+ const response = {
+ req: new EventEmitter(),
+ writeHead: vi.fn(),
+ end: vi.fn(),
+ write: (frame) => {
+ const payload = JSON.parse(frame.replace(/^data: /, '').trim());
+ if (['complete', 'error', 'canceled'].includes(payload.type)) {
+ // Terminal SSE must not race owned-temp cleanup. Observe files at
+ // the public boundary instead of polling after the job completes.
+ Promise.all(downloadedPaths.map(async (path) => {
+ try { await readFile(path); return true; }
+ catch (error) { if (error.code === 'ENOENT') return false; throw error; }
+ })).then((tempPresence) => resolveTerminal({ payload, tempPresence }));
+ }
+ },
+ };
+ const { jobId } = await startYoutubeIngest({
+ url: 'https://youtu.be/oCnxnaVg0bY',
+ captureTranscript: false,
+ ingestAudio: true,
+ });
+ expect(attachIngestSseClient(jobId, response)).toBe(true);
+ return { terminal, response, jobId };
+ }
+
+ it('copies completed audio into the ingest directory without renaming across filesystems', async () => {
+ const { terminal } = await runAudioIngest();
+ const { payload, tempPresence } = await terminal;
+ expect(payload.type).toBe('complete');
+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));
+
+ const { PATHS } = await import('../lib/fileUtils.js');
+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');
+ expect(await readFile(audioPath)).toEqual(audioBytes);
+ expect(testDoubles.copyFileGuarded).toHaveBeenCalledWith(downloadedPaths[0], expect.stringContaining('.mp3.part'));
+ expect(testDoubles.rename).not.toHaveBeenCalledWith(downloadedPaths[0], audioPath);
+ for (const path of downloadedPaths) await expect(readFile(path)).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+
+ it('cleans this job after a post-download copy failure and preserves prior audio', async () => {
+ const { PATHS } = await import('../lib/fileUtils.js');
+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');
+ await actualFs.mkdir(join(PATHS.brain, 'youtube'), { recursive: true });
+ await writeFile(audioPath, 'previous durable audio');
+ unrelatedTempPath = join(tmpdir(), `portos-yt-ingest-other-${Date.now()}.mp3`);
+ await writeFile(unrelatedTempPath, 'other job');
+ testDoubles.copyFileGuarded.mockImplementationOnce(async (source, staging) => {
+ await actualFileUtils.copyFileGuarded(source, staging);
+ await writeFile(staging, 'partial');
+ throw new Error('copy interrupted');
+ });
+
+ const { terminal } = await runAudioIngest();
+ const { payload, tempPresence } = await terminal;
+ expect(payload).toMatchObject({ type: 'error', error: 'copy interrupted' });
+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));
+ expect(await readFile(audioPath, 'utf-8')).toBe('previous durable audio');
+ for (const path of downloadedPaths) await expect(readFile(path)).rejects.toMatchObject({ code: 'ENOENT' });
+ expect(await readFile(unrelatedTempPath, 'utf-8')).toBe('other job');
+ });
+
+ it('honors cancellation during staging before replacing prior audio', async () => {
+ const { PATHS } = await import('../lib/fileUtils.js');
+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');
+ await actualFs.mkdir(join(PATHS.brain, 'youtube'), { recursive: true });
+ await writeFile(audioPath, 'previous durable audio');
+
+ let copyStarted;
+ const started = new Promise((resolve) => { copyStarted = resolve; });
+ let releaseCopy;
+ const copyGate = new Promise((resolve) => { releaseCopy = resolve; });
+ testDoubles.copyFileGuarded.mockImplementationOnce(async (source, staging) => {
+ await actualFileUtils.copyFileGuarded(source, staging);
+ copyStarted();
+ await copyGate;
+ });
+
+ const { terminal, jobId } = await runAudioIngest();
+ await started;
+ expect(cancelYoutubeIngest(jobId)).toBe(true);
+ releaseCopy();
+
+ const { payload, tempPresence } = await terminal;
+ expect(payload.type).toBe('canceled');
+ expect(tempPresence).toEqual(downloadedPaths.map(() => false));
+ expect(await readFile(audioPath, 'utf-8')).toBe('previous durable audio');
+ await expect(readFile(join(PATHS.brain, 'youtube', `.${jobId}.mp3.part`))).rejects.toMatchObject({ code: 'ENOENT' });
+ });
+
+ it('logs a staging cleanup failure without failing published audio', async () => {
+ const cleanupError = Object.assign(new Error('permission denied'), { code: 'EACCES' });
+ testDoubles.unlinkGuarded.mockImplementation(async (path) => {
+ if (path.endsWith('.mp3.part')) throw cleanupError;
+ return actualFileUtils.unlinkGuarded(path);
+ });
+ const errorLog = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+ try {
+ const { terminal } = await runAudioIngest();
+ const { payload } = await terminal;
+ expect(payload.type).toBe('complete');
+ expect(errorLog).toHaveBeenCalledWith(expect.stringMatching(/YouTube ingest staging cleanup failed.*permission denied/));
+
+ const { PATHS } = await import('../lib/fileUtils.js');
+ const audioPath = join(PATHS.brain, 'youtube', 'oCnxnaVg0bY.mp3');
+ expect(await readFile(audioPath)).toEqual(audioBytes);
+ } finally {
+ errorLog.mockRestore();
+ const stagingPath = testDoubles.copyFileGuarded.mock.calls.at(-1)?.[1];
+ if (stagingPath) await actualFileUtils.unlinkGuarded(stagingPath).catch(() => {});
+ }
+ });
+});