Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.1 KB
Raw Blame
/**
 * YouTube watch-history Takeout importer (#2153) — bulk historical backfill into
 * the machine-local human-activity timeline (#2150).
 *
 * YouTube killed the watch-history API years ago, so the reliable path for
 * historical data is Google Takeout: request "YouTube and YouTube Music →
 * history" and Google ships `watch-history.json` (a flat array, newest first)
 * covering the account lifetime. This importer takes that export (the raw ZIP or
 * the individual JSON) and maps every watch into a `media.watch` activity event
 * with the EXACT timestamp Takeout records.
 *
 * Takeout `watch-history.json` record shape:
 *   {
 *     header: "YouTube" | "YouTube Music",
 *     title:  "Watched <video title>",         // "Watched a video that has been removed" when gone
 *     titleUrl: "https://www.youtube.com/watch?v=VIDEOID",  // absent for removed videos / some ads
 *     subtitles: [{ name: "<channel>", url: "https://www.youtube.com/channel/UC..." }],
 *     time: "2024-01-05T04:32:10.123Z",
 *     products: ["YouTube"],
 *     activityControls: ["YouTube watch history"],
 *   }
 *
 * Dedupe reconciles the scrape and the backfill: BOTH sources key on
 * `yt:<videoId>:<localDay>` (the history page only shows a video once per day, so
 * day-bucketing is the coarsest-common-denominator that keeps a Takeout import and
 * a live scrape of the same watch from double-counting). The `happenedAt` we store
 * from Takeout is the exact timestamp; the scrape can only offer local midnight.
 * `recordEvents`'s `ON CONFLICT DO NOTHING` makes re-imports (and overlap with the
 * scrape) no-ops. No AI-provider calls; parsing is deterministic and LLM-free.
 */
import { readFile } from 'fs/promises';
import { collectZipEntries, isZipUpload } from '../lib/zipStream.js';
import { youtubeVideoIdFromUrl } from '../lib/youtubeUrl.js';
import { shortSummary, recordEvents, localDayKey } from './humanActivity.js';
import { getUserTimezone } from './userTimezone.js';

// ---------------------------------------------------------------------------
// Pure helpers (exported for unit tests — no DB, no filesystem, no side effects).
// Shared with youtubeSync.js so the scrape and the backfill agree on video-id
// extraction and dedupe-key construction.
// ---------------------------------------------------------------------------

// Video-id extraction is the canonical rule in `lib/youtubeUrl.js` — re-exported
// here so the importer's long-standing public surface keeps working (#6014).
export { youtubeVideoIdFromUrl };

// Resolve a Takeout watch timestamp to a UTC ISO string, or null if unparseable.
// Takeout's `time` is ISO-8601 with a `Z` (or an explicit offset), so a plain
// Date parse is correct — no OS-timezone ambiguity to correct for.
export function resolveYoutubeInstant(value) {
  if (!value) return null;
  const d = new Date(String(value).trim());
  return Number.isNaN(d.getTime()) ? null : d.toISOString();
}

// Strip Takeout's localized action prefix from a title. English exports prefix
// "Watched " (videos) / "Viewed " (some ads); other locales differ. We only strip
// the known English prefixes and otherwise keep the title verbatim — the video id
// (not the title) is what identifies the watch, so a non-English title still maps.
export function stripWatchedPrefix(title) {
  if (!title) return '';
  const s = String(title).trim();
  const m = /^(?:Watched|Viewed)\s+(.*)$/s.exec(s);
  return (m ? m[1] : s).trim();
}

// Map ONE raw Takeout watch record to a `media.watch` candidate, or null when it
// lacks a resolvable video id (removed videos / bare ad impressions carry no
// autobiographical signal) or a usable timestamp. `timezone` anchors the
// local-day dedupe bucket so a watch near local midnight keys to the right day.
export function takeoutWatchRecordToCandidate(record, timezone) {
  if (!record || typeof record !== 'object') return null;
  const happenedAt = resolveYoutubeInstant(record.time);
  if (!happenedAt) return null;
  const videoId = youtubeVideoIdFromUrl(record.titleUrl);
  if (!videoId) return null;

  const dayKey = localDayKey(happenedAt, timezone);
  if (!dayKey) return null;

  const title = stripWatchedPrefix(record.title) || '(untitled video)';
  const sub = Array.isArray(record.subtitles) ? record.subtitles[0] : null;
  const channel = sub?.name ? String(sub.name).trim() : null;
  const channelUrl = sub?.url ? String(sub.url).trim() : null;
  const product = typeof record.header === 'string' ? record.header : 'YouTube';

  return {
    source: 'youtube',
    kind: 'media.watch',
    happenedAt,
    title,
    summary: channel ? shortSummary(channel) : null,
    url: `https://www.youtube.com/watch?v=${videoId}`,
    // Shared with the scrape — one watch per video per local day (see file header).
    dedupeKey: `yt:${videoId}:${dayKey}`,
    metadata: {
      videoId,
      channel,
      channelUrl,
      product,
      dayBucket: dayKey,
    },
  };
}

// Map a batch of raw records to candidates, dropping the unmappable ones.
export function youtubeWatchActivityCandidates(records = [], timezone) {
  if (!Array.isArray(records)) return [];
  return records.map((r) => takeoutWatchRecordToCandidate(r, timezone)).filter(Boolean);
}

// Summarize a candidate batch for the import preview: date range, watch count,
// unique videos, and the top channels by watch count. Pure over candidates.
export function summarizeYoutubeCandidates(candidates = []) {
  const list = Array.isArray(candidates) ? candidates : [];
  let earliest = null;
  let latest = null;
  const videos = new Set();
  const channelCounts = new Map();
  for (const c of list) {
    if (c.happenedAt) {
      if (!earliest || c.happenedAt < earliest) earliest = c.happenedAt;
      if (!latest || c.happenedAt > latest) latest = c.happenedAt;
    }
    videos.add(c.metadata?.videoId || c.title);
    const channel = c.metadata?.channel;
    if (channel) channelCounts.set(channel, (channelCounts.get(channel) || 0) + 1);
  }
  const topChannels = [...channelCounts.entries()]
    .sort((a, b) => b[1] - a[1])
    .slice(0, 10)
    .map(([name, count]) => ({ name, count }));
  return {
    watches: list.length,
    uniqueVideos: videos.size,
    from: earliest,
    to: latest,
    topChannels,
  };
}

// Parse the text of a single Takeout watch-history JSON file into a record array.
// Tolerates a top-level array (the normal shape) and an `{ items: [...] }`
// wrapper; anything else yields []. Throws only on malformed JSON.
export function parseYoutubeJsonText(text) {
  const parsed = JSON.parse(text);
  if (Array.isArray(parsed)) return parsed;
  if (Array.isArray(parsed?.items)) return parsed.items;
  return [];
}

// ---------------------------------------------------------------------------
// File ingestion (ZIP or single JSON) → records.
// ---------------------------------------------------------------------------

// The watch-history JSON member inside a Takeout ZIP. Google nests it at
// `Takeout/YouTube and YouTube Music/history/watch-history.json`; match the
// filename anywhere so folder-naming/vintage differences don't miss it. The HTML
// variant (`watch-history.html`) is intentionally NOT matched — the JSON export
// is the supported, cheap-to-parse path.
const isWatchHistoryJsonEntry = (entryPath) =>
  /(?:^|\/)watch-history[^/]*\.json$/i.test(String(entryPath || ''));

// Read raw watch records from an uploaded file (ZIP export or single JSON array).
// The ZIP path delegates the whole streaming lifecycle (teardown, autodrain,
// per-entry await) to `collectZipEntries`, leaving only the match/parse callbacks.
export async function readYoutubeRecords(file) {
  if (!file?.path) return [];
  if (isZipUpload(file)) {
    const records = [];
    await collectZipEntries(file.path, {
      match: isWatchHistoryJsonEntry,
      onMatch: (buf) => {
        for (const r of parseYoutubeJsonText(buf.toString('utf-8'))) records.push(r);
      },
    });
    return records;
  }
  const text = await readFile(file.path, 'utf-8');
  return parseYoutubeJsonText(text);
}

// End-to-end import seam: read the file → map → (preview | record). Returns
// counts + a preview summary. `dryRun` parses and summarizes WITHOUT writing so
// the UI can show the user what will be imported before they commit. Because
// `recordEvents` is idempotent, committing (or re-committing) is always safe.
export async function importYoutubeHistory(file, { dryRun = false } = {}) {
  const timezone = await getUserTimezone();
  const records = await readYoutubeRecords(file);
  const candidates = youtubeWatchActivityCandidates(records, timezone);
  const summary = summarizeYoutubeCandidates(candidates);
  if (dryRun) {
    console.log(`📺 YouTube import preview: ${candidates.length} watch(es) from ${records.length} record(s)`);
    return { dryRun: true, parsed: records.length, mapped: candidates.length, recorded: 0, skipped: 0, summary };
  }
  const { recorded, skipped } = await recordEvents(candidates);
  console.log(`📺 YouTube import: ${recorded} new watch(es) recorded, ${skipped} duplicate/invalid (from ${records.length} record(s))`);
  return { dryRun: false, parsed: records.length, mapped: candidates.length, recorded, skipped, summary };
}