Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

37.5 KB
Raw Blame
/**
 * YouTube → Brain ingest.
 *
 * "I watched (or want to work from) this video; pull it into PortOS as content
 * I've consumed and deliberately kept." One job, three independent switches:
 *
 *   - `captureTranscript` — pull the captions and turn them into readable prose,
 *     stored locally AND mirrored into the Obsidian vault as a note (so the
 *     transcript is greppable/linkable next to everything else the user keeps).
 *   - `downloadVideo`     — the full video, into the shared media library via
 *     the same `source: 'download'` history entry the Dev Tools downloader writes.
 *   - `ingestAudio`       — an mp3 kept alongside the transcript.
 *
 * Regardless of which are on, an ingest ALWAYS records the consumption itself:
 *   - a brain `links` record (federated — this is the "saved for future
 *     reference" artifact that peers should see), and
 *   - a `media.watch` human-activity event (machine-local, so the video shows up
 *     on the Timeline alongside passively-synced watch history).
 *
 * Finally, when the user supplies an `agentPrompt` at ingest time, a CoS task is
 * queued that points an agent at the freshly-stored transcript with that prompt
 * — "here is what I want done with this content." No prompt, no task: PortOS
 * must never queue AI work the user didn't ask for (root AGENTS.md, AI Provider
 * Usage Policy).
 *
 * Storage:
 *   data/brain/youtube/index.json          — machine-local ingest index (paths!)
 *   data/brain/youtube/<videoId>.md        — canonical transcript copy
 *   data/brain/youtube/<videoId>.mp3       — audio, when ingestAudio was on
 *   data/brain/youtube-ingest-settings.json — vault/folder + option defaults
 *
 * The index is deliberately NOT a synced brain store: every field in it is a
 * local filesystem path, an Obsidian vault id, or a local video-history id —
 * all machine-local, and federating them would poison peer reconcile the same
 * way the daily log's obsidian-locations sidecar would (see brainJournal.js).
 * The durable, federated record of "I kept this" is the links entry.
 */

import { spawn } from '../lib/childProcess.js';
import { readdir, readFile, rm, stat, unlink } from 'fs/promises';
import { tmpdir } from 'os';
import { join } from 'path';
import { randomUUID } from 'crypto';
import { ServerError } from '../lib/errorHandler.js';
import {
  atomicWrite,
  copyFileGuarded,
  ensureDir,
  PATHS,
  readJSONFile,
  renameGuarded,
  shortId,
  tryReadFile,
  unlinkGuarded,
} from '../lib/fileUtils.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';
import { findFfmpeg } from '../lib/ffmpeg.js';
import { findYtDlp } from '../lib/ytdlp.js';
import { killWithEscalation } from '../lib/killWithEscalation.js';
import { safeChildProcessOptions } from '../lib/processEnv.js';
import { attachSseClient as attachSse, broadcastSse, closeJobAfterDelay } from '../lib/sseUtils.js';
import { vttToPlainText } from '../lib/vttTranscript.js';
import { createMutex } from '../lib/asyncMutex.js';
import { createSettingsStore } from '../lib/settingsStore.js';
import { cleanupYtDlpTemp, downloadAudioToTempMp3 } from './ytdlpAudioImport.js';
import { downloadVideoIntoLibrary } from './videoDownload.js';
import { YOUTUBE_VIDEO_URL_RE } from '../lib/youtubeUrl.js';
import { assertYoutubeVideoUrl } from '../lib/youtubeUrlAssert.js';
// The pure half of the ingest — yt-dlp metadata normalization, the Obsidian
// note body, the CoS agent prompt, and the index's Obsidian-pointer rule — lives
// in lib/ so it is unit-testable without this module's spawn/store graph (#6015).
import {
  buildAgentTaskContext,
  buildIngestNote,
  compareIngestsDesc,
  decodeIngestCursor,
  formatDuration,
  paginateIngests,
  parseVideoMetadata,
  resolveObsidianPointer,
  sanitizeFilename,
} from '../lib/youtubeIngestFormat.js';
import * as obsidian from './obsidian.js';
import * as brainStorage from './brainStorage.js';
import { createLinkFromUrl } from './brain.js';
import * as brainJournal from './brainJournal.js';
import { recordEvents } from './humanActivity.js';
// Straight from the task store rather than the cos.js facade — same reason
// referenceRepos.js does: it avoids pulling the whole Chief-of-Staff graph in
// behind a brain-side service. The `tasks:changed` event addTask emits is what
// cos.js listens on to spawn, so queueing behaves identically either way.
import { addTask } from './cosTaskStore.js';

// The accept rule and its validator are canonical in `lib/youtubeUrl.js` (#6014);
// these aliases keep the ingest's established names for existing importers.
export {
  YOUTUBE_VIDEO_URL_RE as YOUTUBE_INGEST_URL_RE,
  assertYoutubeVideoUrl as assertYoutubeIngestUrl,
};

// Bound resource use, same reasoning as the audio/video importers: a livestream
// archive or a 12-hour upload would otherwise download unbounded. Generous —
// conference talks and long-form podcasts are exactly what this feature is for.
export const INGEST_MAX_DURATION_SEC = 6 * 60 * 60; // 6 hours
export const INGEST_VIDEO_MAX_BYTES = 4 * 1024 * 1024 * 1024; // 4 GB
export const INGEST_AUDIO_MAX_BYTES = 512 * 1024 * 1024; // 512 MB

// `--dump-single-json` on a long video can be slow (yt-dlp negotiates formats);
// bound it so a wedged extractor can't hold a job open forever.
const METADATA_TIMEOUT_MS = 90_000;
const SUBTITLE_TIMEOUT_MS = 120_000;

const INGEST_DIR = join(PATHS.brain, 'youtube');
const INDEX_FILE = join(INGEST_DIR, 'index.json');
const SETTINGS_FILE = join(PATHS.brain, 'youtube-ingest-settings.json');

const DEFAULT_SETTINGS = {
  // null → inherit the Daily Log's vault, so a user who already pointed the
  // journal at Obsidian gets transcripts in the same vault with zero setup.
  obsidianVaultId: null,
  obsidianFolder: 'Consumed/YouTube',
  autoSync: true,
  // Quick-capture's advanced panel seeds its checkboxes from these.
  defaultCaptureTranscript: true,
  defaultDownloadVideo: false,
  defaultIngestAudio: false,
  taskPriority: 'MEDIUM',
};

// ─── Settings ──────────────────────────────────────────────────────────────

// Strict read + serialized PATCH live in the store (#4115): a corrupt file
// rejects instead of reading as DEFAULT_SETTINGS and being overwritten.
const settingsStore = createSettingsStore(SETTINGS_FILE, DEFAULT_SETTINGS);

export const getSettings = settingsStore.get;
export const updateSettings = settingsStore.update;

/**
 * Which Obsidian vault transcripts go to: the explicit setting when present,
 * otherwise the Daily Log's vault. Returns null when neither is configured (the
 * ingest still succeeds — the local transcript copy is the canonical one).
 */
async function resolveVaultId(settings) {
  if (settings.obsidianVaultId) return settings.obsidianVaultId;
  const journalSettings = await brainJournal.getSettings().catch(() => null);
  return journalSettings?.obsidianVaultId || null;
}

// ─── Local ingest index ────────────────────────────────────────────────────

// Serialize read→modify→write of the index so two ingests finishing at once
// can't clobber each other's entry (single-user, but a "transcript only" job
// and a slow video job legitimately overlap).
const indexMutex = createMutex();

async function loadIndex() {
  await ensureDir(INGEST_DIR);
  const loaded = await readJSONFile(INDEX_FILE, {}, { strict: true });
  return loaded && typeof loaded === 'object' && !Array.isArray(loaded) ? loaded : {};
}

// A first ingest only patches in the artifacts its switches produced, so seed
// the rest as explicit nulls — readers (and the client) get one record shape
// whether a video was ingested once for its transcript or three times for
// everything.
const NEW_INGEST = { transcript: null, obsidian: null, video: null, audio: null, taskId: null, agentPrompt: null, incomplete: null };

// The index record is the row that first names a landed transcript or audio file,
// so committing it takes the backup lease (#9982). Lease first, then the mutex,
// everywhere, so the two never invert.
async function putIngest(videoId, patch) {
  return withBackupAssetPublication(() => indexMutex(async () => {
    const index = await loadIndex();
    const existing = index[videoId] || null;
    const record = {
      ...(existing || NEW_INGEST),
      ...patch,
      videoId,
      ingestedAt: existing?.ingestedAt || new Date().toISOString(),
      updatedAt: new Date().toISOString(),
    };
    index[videoId] = record;
    await atomicWrite(INDEX_FILE, index);
    return record;
  }));
}

/**
 * Every ingest, newest first. Query-less callers (the default) get the full
 * legacy array. Passing `limit` opts into a bounded page ordered by
 * `ingestedAt` desc with `videoId` as a deterministic tie-breaker; pass back
 * a prior page's `nextCursor` to continue (#8267).
 */
export async function listIngests({ limit, cursor } = {}) {
  const index = await loadIndex();
  const sorted = Object.values(index).sort(compareIngestsDesc);
  if (limit === undefined) return sorted;
  let decodedCursor = null;
  if (cursor) {
    decodedCursor = decodeIngestCursor(cursor);
    if (!decodedCursor) throw new ServerError('Invalid ingest cursor', { status: 400, code: 'INVALID_CURSOR' });
  }
  return paginateIngests(sorted, { limit, cursor: decodedCursor });
}

export async function getIngest(videoId) {
  const index = await loadIndex();
  return index[videoId] || null;
}

/** The stored transcript markdown, or null when this video has none. */
export async function getTranscript(videoId) {
  const record = await getIngest(videoId);
  if (!record?.transcript?.path) return null;
  return tryReadFile(record.transcript.path);
}

/**
 * Forget an ingest: removes the local transcript/audio files and the Obsidian
 * note, then drops the index entry. Deliberately leaves the links record, the
 * downloaded video, and the activity event alone — each has its own lifecycle
 * and its own UI to delete from, and silently nuking a user's media library
 * entry from a "remove this transcript" action would be a nasty surprise.
 */
export async function deleteIngest(videoId) {
  // Read the record and drop it inside ONE lock, then clean up its files:
  // reading outside the lock would unlink paths from a record a concurrent
  // re-ingest had already replaced, and would parse the whole index twice.
  // The index is a data file a cut copies apart from the transcript and audio,
  // so the record drop and the unlinks hold one lease (#9982).
  const record = await withBackupAssetPublication(async () => {
    const dropped = await indexMutex(async () => {
      const index = await loadIndex();
      const found = index[videoId] || null;
      if (!found) return null;
      delete index[videoId];
      await atomicWrite(INDEX_FILE, index);
      return found;
    });
    if (!dropped) return null;
    if (dropped.transcript?.path) await unlink(dropped.transcript.path).catch(() => {});
    if (dropped.audio?.path) await unlink(dropped.audio.path).catch(() => {});
    return dropped;
  });
  if (!record) return false;

  if (record.obsidian?.path && record.obsidian?.vaultId) {
    await obsidian.deleteNote(record.obsidian.vaultId, record.obsidian.path).catch(() => {});
  }
  console.log(`🧹 Removed YouTube ingest ${videoId}`);
  return true;
}

// ─── yt-dlp helpers ────────────────────────────────────────────────────────

// Run yt-dlp to completion, collecting stdout. Used for the two invocations that
// produce no progress stream (metadata dump, subtitle fetch); the long-running
// media downloads go through the shared audio/video cores instead.
function runYtDlp(ytDlp, args, { timeoutMs, registerProcess }) {
  return new Promise((resolve) => {
    const proc = spawn(ytDlp, args, safeChildProcessOptions({ stdio: ['ignore', 'pipe', 'pipe'] }));
    registerProcess?.(proc);
    let stdout = '';
    let stderr = '';
    proc.stdout.on('data', (d) => { stdout += d.toString(); });
    proc.stderr.on('data', (d) => { stderr += d.toString(); });
    // Both a user cancel and this timeout end the child with a signal, so the
    // signal alone can't tell them apart — and reporting a 90-second extractor
    // stall as "ingest cancelled" sends the user looking for a cancel they
    // never made. Record which one fired.
    let timedOut = false;
    const timer = setTimeout(() => { timedOut = true; proc.kill('SIGKILL'); }, timeoutMs);
    proc.on('error', (err) => {
      clearTimeout(timer);
      registerProcess?.(null);
      resolve({ code: null, stdout, stderr, reason: `spawn failed: ${err.message}` });
    });
    proc.on('close', (code, signal) => {
      clearTimeout(timer);
      registerProcess?.(null);
      resolve({ code, signal, stdout, stderr, timedOut });
    });
  });
}

/**
 * Read the video's metadata and, when `subsDir` is given, write its English
 * caption files there — in ONE yt-dlp invocation.
 *
 * Doing them separately cost two full YouTube extractions (player-response +
 * format negotiation, several seconds each) of the same URL on the DEFAULT
 * ingest path. `--dump-single-json` normally implies `--simulate`, which would
 * skip writing the subtitle files, so `--no-simulate` is required alongside it
 * — the same yt-dlp quirk `--print` has, documented at length in
 * ytdlpAudioImport.js. Verified against a real video: the JSON lands on stdout
 * and the `.vtt` files land in `subsDir`.
 *
 * Returns the parsed metadata; the caller reads the caption files itself.
 */
async function fetchVideoInfo(ytDlp, url, registerProcess, { subsDir = null } = {}) {
  const result = await runYtDlp(
    ytDlp,
    [
      '--dump-single-json', '--no-playlist', '--no-warnings',
      ...(subsDir
        ? [
          '--no-simulate', '--skip-download',
          '--write-subs', '--write-auto-subs',
          // Prefer real English tracks; `en.*` also catches en-US/en-GB/en-orig.
          '--sub-langs', 'en.*,en',
          // yt-dlp can only *convert* subtitles with ffmpeg, but `--sub-format`
          // negotiates the download format directly — vtt is what YouTube serves
          // natively, so no conversion (and no ffmpeg dependency) is needed.
          '--sub-format', 'vtt/srt/best',
          '-o', join(subsDir, 'subs'),
        ]
        : ['--skip-download']),
      url,
    ],
    { timeoutMs: subsDir ? SUBTITLE_TIMEOUT_MS : METADATA_TIMEOUT_MS, registerProcess },
  );
  if (result.timedOut) {
    throw new Error(
      `yt-dlp timed out after ${Math.round((subsDir ? SUBTITLE_TIMEOUT_MS : METADATA_TIMEOUT_MS) / 1000)}s reading the video — YouTube may be throttling, or yt-dlp may need updating`,
    );
  }
  if (result.signal) throw new Error('canceled');
  if (result.code !== 0) {
    // yt-dlp's own stderr is the most actionable thing we have here (private
    // video, age gate, region block, extractor breakage all land differently).
    const detail = (result.stderr || result.reason || '').trim().split('\n').filter(Boolean).pop();
    throw new Error(detail || `yt-dlp exited ${result.code} while reading video metadata`);
  }
  return parseVideoMetadata(result.stdout);
}

/**
 * Read the caption files `fetchVideoInfo` wrote into `subsDir` and render them
 * as prose. Returns `{ text, language, source }` or null when the video had no
 * captions.
 *
 * yt-dlp names subtitle files `subs.<lang>.<ext>` and the language tag varies
 * (`en`, `en-US`, `en-orig`), so the produced file is discovered from disk
 * rather than assumed — the same "don't guess the filename" rule the video core
 * follows. `--sub-langs en.*,en` routinely writes MORE than one file (a real
 * upload yields both `subs.en-orig.vtt` and `subs.en.vtt`), so a plain `en`
 * track is preferred and the rest ignored, making the choice deterministic.
 *
 * `hasManualCaptions` comes from the same metadata dump — see parseVideoMetadata.
 */
async function readTranscriptFrom(subsDir, { hasManualCaptions } = {}) {
  const produced = (await readdir(subsDir).catch(() => [])).filter((n) => /\.(vtt|srt)$/i.test(n));
  const chosen = produced.find((n) => /\.en\.(vtt|srt)$/i.test(n)) || produced[0] || null;
  if (!chosen) return null;

  const raw = await readFile(join(subsDir, chosen), 'utf-8').catch(() => '');
  const text = vttToPlainText(raw);
  if (!text) return null;
  const langMatch = /\.([\w-]+)\.(vtt|srt)$/i.exec(chosen);
  return {
    text,
    language: langMatch ? langMatch[1] : 'en',
    source: hasManualCaptions ? 'captions' : 'auto-captions',
  };
}

// ─── Obsidian note ─────────────────────────────────────────────────────────

const buildNotePath = (settings, meta, capturedAt) => {
  const folder = (settings.obsidianFolder || '').replace(/^\/+|\/+$/g, '');
  const filename = `${capturedAt.slice(0, 10)} ${sanitizeFilename(meta.title)} (${meta.videoId}).md`;
  return folder ? `${folder}/${filename}` : filename;
};

// ─── Job orchestration ─────────────────────────────────────────────────────

// jobId -> { id, clients, lastPayload, process, canceled }
const ingestJobs = new Map();

export const attachIngestSseClient = (jobId, res) => attachSse(ingestJobs, jobId, res);

// Exposed for the same reason as videoDownload's — so the cancel path is
// reachable from a test instead of only from a live ingest.
export const __testing = { ingestJobs };

/**
 * Cancel an in-flight ingest. Returns false if the job is unknown or finished.
 *
 * The `canceled` flag matters independently of the child kill: an ingest is a
 * chain of steps, so cancelling between two of them (or during the non-spawn
 * persist phase) has no process to signal and is caught by the flag instead.
 */
export function cancelYoutubeIngest(jobId) {
  const job = ingestJobs.get(jobId);
  if (!job) return false;
  job.canceled = true;
  const proc = job.process;
  if (proc) {
    killWithEscalation(proc, { label: 'yt-dlp ingest', stillRunning: () => job.process === proc });
  }
  return true;
}

/**
 * Kick off an ingest. Returns `{ jobId, videoId }` immediately; the work runs
 * detached and streams progress over SSE. Terminal frames:
 * `{ type: 'complete', ingest }`, `{ type: 'error', error }`, `{ type: 'canceled' }`.
 *
 * @param {object}  opts
 * @param {string}  opts.url
 * @param {boolean} [opts.captureTranscript=true]
 * @param {boolean} [opts.downloadVideo=false]
 * @param {boolean} [opts.ingestAudio=false]
 * @param {string}  [opts.note]  Optional reason for keeping the link.
 * @param {string}  [opts.agentPrompt]  Non-empty → queue a CoS task pointed at the transcript.
 * @param {string[]} [opts.tags]
 * @param {string}  [opts.priority]     CoS task priority (defaults to the setting).
 */
export async function startYoutubeIngest({
  url,
  captureTranscript = true,
  downloadVideo = false,
  ingestAudio = false,
  note = '',
  agentPrompt = '',
  targetAppId, providerId, model, effort, workMode = 'issues',
  tags = [],
  priority,
} = {}) {
  const videoId = assertYoutubeVideoUrl(url);
  if (!captureTranscript && !downloadVideo && !ingestAudio) {
    throw new ServerError('Pick at least one of: transcript, video, audio', {
      status: 400,
      code: 'NOTHING_TO_INGEST',
    });
  }

  // Resolve explicit agent routing before starting downloads, so stale app
  // selections fail visibly and never silently dispatch into PortOS.
  let analysisApp = null;
  let filing = null;
  if (String(agentPrompt || '').trim()) {
    const { getAppById, PORTOS_APP_ID } = await import('./apps.js');
    analysisApp = await getAppById(targetAppId || PORTOS_APP_ID);
    if (!analysisApp?.repoPath || analysisApp.archived) {
      throw new ServerError('Selected analysis app is unavailable', { status: 400, code: 'APP_NOT_FOUND' });
    }
    if (workMode === 'issues') {
      const { resolveTrackerFilingBlock } = await import('../lib/workTracker.js');
      filing = await resolveTrackerFilingBlock(analysisApp, 'youtube-analysis');
    }
  }

  const ytDlp = await findYtDlp();
  if (!ytDlp) throw new ServerError('yt-dlp not found on PATH', { status: 500, code: 'YTDLP_MISSING' });
  // ffmpeg is only needed to merge video / transcode audio — a transcript-only
  // ingest must not fail on a machine that has yt-dlp but no ffmpeg.
  const ffmpeg = (downloadVideo || ingestAudio) ? await findFfmpeg() : null;
  if ((downloadVideo || ingestAudio) && !ffmpeg) {
    throw new ServerError('ffmpeg not found on PATH', { status: 500, code: 'FFMPEG_MISSING' });
  }

  const jobId = randomUUID();
  const job = { id: jobId, clients: [], process: null, canceled: false };
  ingestJobs.set(jobId, job);
  const cleanTags = tags.map((t) => String(t).trim()).filter(Boolean);
  const linkNote = typeof note === 'string' ? note.trim() : '';
  const prompt = String(agentPrompt || '').trim();
  console.log(`📺 YouTube ingest ${shortId(jobId)} — ${videoId} (transcript=${captureTranscript} video=${downloadVideo} audio=${ingestAudio})`);

  (async () => {
    const registerProcess = (proc) => { job.process = proc; };
    const stage = (name, percent) => broadcastSse(job, { type: 'progress', stage: name, ...(percent != null ? { percent } : {}) });
    const tempPrefix = ingestAudio ? `portos-yt-ingest-${jobId}` : null;
    // Copy the downloaded bytes to a unique sibling before atomically replacing
    // the durable name. A failed/partial copy must not truncate audio from a
    // prior ingest of this video.
    const audioStagingPath = ingestAudio ? join(INGEST_DIR, `.${jobId}.mp3.part`) : null;
    let terminalEvent = null;
    // Non-fatal outcomes the user still needs told about (no captions, duplicate
    // task). Broadcast live AND replayed on the terminal frame — a client using
    // the generic single-slot SSE hook only reads the terminal payload, so a
    // warning-only frame would otherwise vanish.
    const warnings = [];
    const warn = (message) => {
      warnings.push(message);
      broadcastSse(job, { type: 'warning', message });
    };
    // One timestamp for the whole ingest. Taken up front so the note's
    // `captured:` frontmatter, the note's filename date, and the activity
    // event's `happenedAt` can't disagree — an ingest that downloads a 3-hour
    // video legitimately crosses midnight.
    const capturedAt = new Date().toISOString();
    // A dedicated per-job temp dir rather than a shared-prefix scan of
    // os.tmpdir() (which routinely holds thousands of entries on macOS): the
    // captions are found with a 2-entry readdir and removed with one rm.
    const subsDir = captureTranscript ? join(tmpdir(), `portos-yt-subs-${jobId}`) : null;
    // What this run actually put on disk, accumulated as each stage lands.
    // Persisted even when a LATER stage fails: a transcript written before a
    // failed audio download is a real file in a real vault, and without an index
    // record it is invisible to the list and unreachable by deleteIngest —
    // an orphan the user has no way to clean up.
    const landed = { transcript: null, obsidian: null, video: null, audio: null };
    // The prior record for this video, if any — supplies the Obsidian note path
    // and link id to reuse so a re-ingest updates rather than forks.
    let prior = null;
    // The id the artifact FILES were written under — meta.videoId once the
    // metadata call names it, the URL-derived guess before that. The catch needs
    // it to key the partial record, and  is scoped to the try.
    let resolvedVideoId = videoId;

    // Every await in this fire-and-forget coordinator must sit INSIDE the try:
    // there is no request lifecycle to bubble to, so a rejection escaping it
    // takes the whole Node process down (root AGENTS.md, no-try/catch exception).
    try {
      // Read once: the vault/folder decision and the CoS task priority both come
      // from here, and an ingest should use one settings snapshot throughout.
      const settings = await getSettings();
      await ensureDir(INGEST_DIR);
      if (subsDir) await ensureDir(subsDir);

      // Metadata AND captions come back from a single yt-dlp extraction.
      stage(captureTranscript ? 'transcript' : 'metadata');
      const meta = await fetchVideoInfo(ytDlp, url, registerProcess, { subsDir });
      // yt-dlp's own id is authoritative; ours is a URL-shape guess.
      meta.videoId = meta.videoId || videoId;
      resolvedVideoId = meta.videoId;
      prior = await getIngest(meta.videoId);
      broadcastSse(job, { type: 'metadata', title: meta.title, channel: meta.channel, durationSec: meta.durationSec });
      if (job.canceled) throw new Error('canceled');

      // ── Transcript ──
      // Written AND mirrored before the media downloads: the note doesn't depend
      // on them, and doing it here lets the caption text (up to a few hundred KB)
      // be collected before a multi-GB video download rather than being pinned in
      // this closure across it.
      let transcriptPath = null;
      if (captureTranscript) {
        const transcript = await readTranscriptFrom(subsDir, { hasManualCaptions: meta.hasManualCaptions });
        if (transcript) {
          const markdown = buildIngestNote({ meta, url, transcript, tags: cleanTags, agentPrompt: prompt, capturedAt });
          transcriptPath = join(INGEST_DIR, `${meta.videoId}.md`);
          await atomicWrite(transcriptPath, markdown);
          landed.transcript = {
            path: transcriptPath,
            language: transcript.language,
            source: transcript.source,
            chars: transcript.text.length,
          };

          if (settings.autoSync) {
            const vaultId = await resolveVaultId(settings);
            // Re-ingesting on a later day must UPDATE the note this video
            // already has, not mint a second one at today's date — the index
            // tracks a single location, so a fresh path would orphan the old
            // note where deleteIngest can never reach it.
            const notePath = vaultId
              ? (prior?.obsidian?.vaultId === vaultId && prior.obsidian.path
                ? prior.obsidian.path
                : buildNotePath(settings, meta, capturedAt))
              : null;
            // Best-effort: the transcript is already stored locally, so a vault
            // on an unplugged drive must not fail the ingest.
            const written = notePath
              ? await obsidian.upsertNote(vaultId, notePath, markdown)
                .catch((err) => { console.error(`📓 Obsidian sync failed for ${meta.videoId}: ${err.message}`); return null; })
              : null;
            landed.obsidian = resolveObsidianPointer({ written, vaultId, notePath, prior });
          }
        } else {
          // Not fatal — the user still gets the link record, the activity event,
          // and whatever media they asked for. Say so explicitly rather than
          // completing with a silently empty transcript.
          warn('No captions were available for this video');
        }
      }

      // ── Video ──
      // Cancel is checked BEFORE each download, not only after: cancelling
      // while no child is running (during the transcript write or the Obsidian
      // round-trip) only sets the flag, and without this the job would go on to
      // start a multi-gigabyte download the user just cancelled.
      if (job.canceled) throw new Error('canceled');
      if (downloadVideo) {
        stage('video', 0);
        // Same call the Dev Tools downloader makes, so the video lands in the
        // media library identically — history entry, thumbnail, index event and
        // partial-file cleanup all included.
        const result = await downloadVideoIntoLibrary({
          url,
          ytDlp,
          ffmpeg,
          maxBytes: INGEST_VIDEO_MAX_BYTES,
          maxDurationSec: INGEST_MAX_DURATION_SEC,
          onProgress: ({ percent, stage: s }) => broadcastSse(job, { type: 'progress', stage: s || 'video', percent }),
          registerProcess,
        });
        if (result.outcome === 'canceled' || job.canceled) throw new Error('canceled');
        if (result.outcome === 'failed') throw new Error(`video download failed — ${result.reason}`);
        landed.video = { historyId: result.entry.id, filename: result.entry.filename };
      }

      // ── Audio ──
      if (job.canceled) throw new Error('canceled');
      if (ingestAudio) {
        stage('audio', 0);
        const result = await downloadAudioToTempMp3({
          url,
          ytDlp,
          ffmpeg,
          tempPrefix,
          maxBytes: INGEST_AUDIO_MAX_BYTES,
          maxDurationSec: INGEST_MAX_DURATION_SEC,
          onProgress: ({ percent, stage: s }) => broadcastSse(job, { type: 'progress', stage: s || 'audio', percent }),
          registerProcess,
        });
        if (result.outcome === 'canceled' || job.canceled) throw new Error('canceled');
        if (result.outcome === 'failed') throw new Error(`audio download failed — ${result.reason}`);
        // Kept next to the transcript rather than pushed into the music library:
        // a two-hour talk is reference material, not a track.
        const audioPath = join(INGEST_DIR, `${meta.videoId}.mp3`);
        // Copy to same-filesystem staging before publishing, so cross-device
        // yt-dlp output and failed copies cannot damage prior audio.
        await withBackupAssetPublication(async () => {
          await copyFileGuarded(result.outPath, audioStagingPath);
          if (job.canceled) throw new Error('canceled');
          await renameGuarded(audioStagingPath, audioPath);
        });
        const bytes = await stat(audioPath).then((s) => s.size).catch(() => null);
        landed.audio = { path: audioPath, bytes };
        await unlinkGuarded(result.outPath).catch(() => {});
      }

      // ── Persist the "I consumed and kept this" artifacts ──
      if (job.canceled) throw new Error('canceled');
      stage('saving');

      // The federated record of "saved for future reference". Re-ingesting a
      // video reuses the existing link instead of forking a duplicate. Prefer
      // the id this VIDEO already recorded over a URL string match: the same
      // video is reachable as youtu.be/<id> and youtube.com/watch?v=<id>, and
      // getLinkByUrl compares exact strings, so the second form would otherwise
      // mint a second link for a video the index already knows.
      const priorLink = prior?.linkId ? await brainStorage.getLinkById(prior.linkId).catch(() => null) : null;
      const existingLink = priorLink || await brainStorage.getLinkByUrl(url).catch(() => null);
      const linkFields = {
        title: meta.title,
        description: [meta.channel, formatDuration(meta.durationSec)].filter(Boolean).join(' · '),
        linkType: 'reference',
        tags: ['youtube', ...cleanTags],
      };
      if (linkNote) linkFields.note = linkNote;
      // A reused link still has to pick up THIS ingest's tags and the resolved
      // title/channel — otherwise tags typed on a re-ingest are silently dropped
      // and a link first saved as a bare URL keeps its hostname title forever.
      const link = existingLink
        ? (await brainStorage.updateLink(existingLink.id, {
          ...linkFields,
          tags: [...new Set([...(existingLink.tags || []), ...linkFields.tags])],
        }).catch(() => existingLink)) || existingLink
        : await createLinkFromUrl(url, { ...linkFields, autoClone: false });

      // Machine-local consumption log, so the video shows on the Timeline next
      // to passively-synced watch history. Best-effort: a DB hiccup must not
      // fail an ingest whose files are already on disk. The `ingest:` dedupe
      // prefix keeps this distinct from youtubeSync's watch-history rows.
      await recordEvents([{
        source: 'youtube',
        kind: 'media.watch',
        happenedAt: capturedAt,
        durationS: meta.durationSec,
        title: meta.title,
        summary: meta.channel,
        url,
        dedupeKey: `ingest:${meta.videoId}`,
        metadata: { videoId: meta.videoId, channel: meta.channel, ingest: true, tags: cleanTags },
      }]).catch((err) => console.error(`🗓️  Activity event failed for ${meta.videoId}: ${err.message}`));

      // ── Follow-up agent task ──
      let taskId = null;
      if (prompt) {
        stage('queueing');
        const task = await addTask({
          description: `Review ingested YouTube content: ${meta.title} [${meta.videoId}]`,
          app: analysisApp.id,
          ...(providerId ? { provider: providerId } : {}),
          ...(model ? { model } : {}),
          ...(effort ? { effort } : {}),
          ...(filing ? { workTracker: filing.workTracker, worktreeChangesExpected: filing.worktreeChangesExpected } : {}),
          context: buildAgentTaskContext({
            meta, url, agentPrompt: prompt, transcriptPath, notePath: landed.obsidian?.path, tags: cleanTags,
            hasTranscript: !!transcriptPath,
            appName: analysisApp.name, workMode,
            trackerInstructions: filing?.trackerInstructions
              .replace(/\{appName\}/g, () => analysisApp.name)
              .replace(/\{repoPath\}/g, () => analysisApp.repoPath),
          }),
          priority: priority || settings.taskPriority,
          useWorktree: workMode === 'implement',
          openPR: workMode === 'implement',
          simplify: false,
          reviewLoop: false,
        }, 'user').catch((err) => {
          console.error(`❌ CoS task for ingest ${meta.videoId} failed: ${err.message}`);
          return null;
        });
        if (task?.duplicate) {
          warn('A review task for this video is already queued');
        }
        taskId = task?.id || null;
      }

      // Artifact keys are written only for the switches this run actually ran.
      // putIngest merges over any prior record, so re-ingesting a video for its
      // audio alone must not null out the transcript entry whose file is still
      // sitting on disk — absent means "this run didn't touch it", not "gone".
      // Within a switch that DID run, an explicit null is meaningful (transcript
      // requested but the video had no captions), so it is written.
      const record = await putIngest(meta.videoId, {
        url,
        title: meta.title,
        channel: meta.channel,
        channelUrl: meta.channelUrl,
        durationSec: meta.durationSec,
        publishedAt: meta.publishedAt,
        ...(captureTranscript ? { transcript: landed.transcript, obsidian: landed.obsidian } : {}),
        ...(downloadVideo ? { video: landed.video } : {}),
        ...(ingestAudio ? { audio: landed.audio } : {}),
        ...(prompt ? { taskId, agentPrompt: prompt } : {}),
        linkId: link?.id || null,
        tags: cleanTags,
        // Clears the marker a previously-failed run left, so a record that has
        // since completed does not keep reporting itself as partial.
        incomplete: null,
      });

      console.log(`📺 YouTube ingest ${shortId(jobId)} complete — "${meta.title}"${taskId ? ` → task ${taskId}` : ''}`);
      terminalEvent = { type: 'complete', ingest: record, warnings };
    } catch (err) {
      const message = err?.message || String(err);
      // Record whatever DID land before the failure. A transcript written (and
      // mirrored into the vault) before a failed audio download is a real file
      // the user now owns; without an index entry it never shows in the list and
      // deleteIngest can never remove it — a permanent orphan. Keyed on the id
      // we actually resolved, so a failure before the metadata call (which is
      // what names the video) has nothing to write and correctly skips this.
      if (landed.transcript || landed.obsidian || landed.video || landed.audio) {
        await putIngest(resolvedVideoId, {
          url,
          ...(landed.transcript ? { transcript: landed.transcript, obsidian: landed.obsidian } : {}),
          ...(landed.video ? { video: landed.video } : {}),
          ...(landed.audio ? { audio: landed.audio } : {}),
          tags: cleanTags,
          incomplete: message === 'canceled' || job.canceled ? 'canceled' : message,
        }).catch((e) => console.error(`📺 Could not record partial ingest ${videoId}: ${e.message}`));
      }
      if (message === 'canceled' || job.canceled) {
        console.log(`🛑 YouTube ingest ${shortId(jobId)} cancelled`);
        terminalEvent = { type: 'canceled' };
      } else {
        console.error(`❌ YouTube ingest ${shortId(jobId)} failed: ${message}`);
        terminalEvent = { type: 'error', error: message };
      }
    } finally {
      // One rm regardless of outcome — the caption files are scratch, and on
      // cancel/failure yt-dlp may have written some of them.
      if (subsDir) await rm(subsDir, { recursive: true, force: true }).catch(() => {});
      // These paths are scoped to this job id. Remove only this job's exact
      // staging file and yt-dlp prefix; durable transcript/audio files are
      // intentionally left for the index/delete workflow to own.
      if (audioStagingPath) {
        await unlinkGuarded(audioStagingPath).catch((error) => {
          if (error.code !== 'ENOENT') {
            console.error(`🧹 YouTube ingest staging cleanup failed for ${shortId(jobId)}: ${error.message}`);
          }
        });
      }
      if (tempPrefix) {
        await cleanupYtDlpTemp(tempPrefix).catch((error) => {
          console.error(`🧹 YouTube ingest temp cleanup failed for ${shortId(jobId)}: ${error.message}`);
        });
      }
      if (terminalEvent) broadcastSse(job, terminalEvent);
      closeJobAfterDelay(ingestJobs, jobId);
    }
  })();

  return { jobId, videoId };
}