Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

16.1 KB
Raw Blame
/**
 * RSS/Atom Feed Ingestion Service
 *
 * Manages feed subscriptions, fetches/parses RSS and Atom feeds,
 * and stores items for reading within the Brain knowledge system.
 *
 * Data stored in data/feeds.json:
 *   { feeds: [...], items: [...] }
 */

import { randomUUID } from 'crypto';
import { join } from 'path';
import { PATHS, createCachedStore } from '../lib/fileUtils.js';
import { fetchPublicText } from '../lib/safeUrlFetch.js';
import { decodeXmlEntities } from '../lib/xmlEntities.js';

const FEEDS_FILE = join(PATHS.data, 'feeds.json');
const MAX_ITEMS_PER_FEED = 100;
const FETCH_TIMEOUT_MS = 15000;

const DEFAULT_DATA = { feeds: [], items: [] };
const store = createCachedStore(FEEDS_FILE, DEFAULT_DATA, { context: 'feeds' });

// Compare items newest-first by pubDate (or fetchedAt fallback). Both writes
// and the legacy normalization pass use this so getItems can skip the sort.
const itemSortKey = (item) => Date.parse(item.pubDate || item.fetchedAt) || 0;
const compareItemsNewestFirst = (a, b) => itemSortKey(b) - itemSortKey(a);

// Drop items with no link, a link already in `seen`, or a link repeated within
// the batch. Mutates `seen` so duplicates inside one payload collapse too.
const dedupeByLink = (items, seen = new Set()) =>
  items.filter(item => {
    if (!item.link || seen.has(item.link)) return false;
    seen.add(item.link);
    return true;
  });

// Sort runs on writes only; getItems trusts the invariant. Legacy unsorted
// state on disk gets normalized once on first read (see ensureItemsSorted).
// The invariant assumes this module owns all writes to feeds.json — out-of-band
// edits while the process is running won't retrigger normalization.
let _itemsSorted = false;
const sortItemsNewestFirst = (items) => {
  items.sort(compareItemsNewestFirst);
  _itemsSorted = true;
};

const isSortedNewestFirst = (items) => {
  for (let i = 1; i < items.length; i++) {
    if (compareItemsNewestFirst(items[i - 1], items[i]) > 0) return false;
  }
  return true;
};

async function ensureItemsSorted() {
  if (_itemsSorted) return;
  await store.mutate((data) => {
    if (!isSortedNewestFirst(data.items)) {
      sortItemsNewestFirst(data.items);
    } else {
      _itemsSorted = true;
    }
    return data;
  });
}

// ─── RSS/Atom Parser ────────────────────────────────────────────────────────

/**
 * Extract text content from an XML tag (first occurrence, handles CDATA).
 *
 * Index scanning rather than a regex: the old `\s*(?:CDATA)?([\s\S]*?)\s*</tag>`
 * pattern backtracked catastrophically when a fetched feed opened a tag and
 * never closed it (a 2 KB `<title>` plus spaces took 13 s), and feeds refresh
 * on a schedule from servers PortOS does not control.
 */
// ASCII-only fold keeps `lower` the same length as `xml`; a full toLowerCase()
// expands characters such as U+0130 and would skew the offsets below.
const asciiLower = (text) => text.replace(/[A-Z]+/g, (run) => run.toLowerCase());

const extractTag = (xml, tag) => {
  const lower = asciiLower(xml);
  const open = `<${asciiLower(tag)}`;
  const close = `</${asciiLower(tag)}>`;
  for (let from = 0; ; ) {
    const start = lower.indexOf(open, from);
    if (start === -1) return '';
    const after = start + open.length;
    const boundary = xml[after];
    if (boundary !== undefined && boundary !== '>' && boundary !== '/' && !/\s/.test(boundary)) { from = after; continue; }
    const gt = xml.indexOf('>', after);
    if (gt === -1) return '';
    const end = lower.indexOf(close, gt + 1);
    if (end === -1) return '';
    let inner = xml.slice(gt + 1, end).trim();
    if (inner.startsWith('<![CDATA[')) inner = inner.slice('<![CDATA['.length);
    if (inner.endsWith(']]>')) inner = inner.slice(0, -']]>'.length);
    return inner.trim();
  }
};

export const __testing = { extractTag };

/**
 * Extract href from an Atom <link> tag.
 */
const extractAtomLink = (xml) => {
  // Prefer rel="alternate" or no rel
  const altMatch = xml.match(/<link[^>]*rel=["']alternate["'][^>]*href=["']([^"']+)["']/i);
  if (altMatch) return altMatch[1];
  const anyMatch = xml.match(/<link[^>]*href=["']([^"']+)["']/i);
  return anyMatch ? anyMatch[1] : '';
};

/**
 * Strip HTML tags from a string for plain text display.
 */
const stripHtml = (html) => decodeXmlEntities(html.replace(/<[^>]+>/g, '')).trim();

/**
 * Parse RSS 2.0 feed XML into normalized items.
 */
const parseRSS = (xml) => {
  const channelMatch = xml.match(/<channel>([\s\S]*)<\/channel>/i);
  if (!channelMatch) return { title: '', items: [] };
  const channel = channelMatch[1];

  const feedTitle = extractTag(channel, 'title');

  // Split on <item> tags
  const itemBlocks = channel.match(/<item>([\s\S]*?)<\/item>/gi) || [];
  const items = itemBlocks.map(block => ({
    title: extractTag(block, 'title'),
    link: extractTag(block, 'link'),
    description: stripHtml(extractTag(block, 'description')).slice(0, 500),
    pubDate: extractTag(block, 'pubDate'),
    author: extractTag(block, 'author') || extractTag(block, 'dc:creator')
  }));

  return { title: feedTitle, items };
};

/**
 * Parse Atom feed XML into normalized items.
 */
const parseAtom = (xml) => {
  const feedTitle = extractTag(xml, 'title');

  const entryBlocks = xml.match(/<entry>([\s\S]*?)<\/entry>/gi) || [];
  const items = entryBlocks.map(block => ({
    title: extractTag(block, 'title'),
    link: extractAtomLink(block),
    description: stripHtml(extractTag(block, 'summary') || extractTag(block, 'content')).slice(0, 500),
    pubDate: extractTag(block, 'updated') || extractTag(block, 'published'),
    author: extractTag(block, 'name')
  }));

  return { title: feedTitle, items };
};

/**
 * Auto-detect feed format and parse.
 */
const parseFeed = (xml) => {
  if (/<feed[\s>]/i.test(xml)) return parseAtom(xml);
  return parseRSS(xml);
};

// ─── Feed Management ────────────────────────────────────────────────────────

export async function getFeeds() {
  const data = await store.load();
  // Pre-compute unread counts in one pass: O(I) instead of O(F×I)
  const unreadCounts = new Map();
  for (const item of data.items) {
    if (!item.read) unreadCounts.set(item.feedId, (unreadCounts.get(item.feedId) || 0) + 1);
  }
  return data.feeds.map(feed => ({
    ...feed,
    unreadCount: unreadCounts.get(feed.id) || 0
  }));
}

export async function addFeed(url) {
  // Fast pre-check for an obvious duplicate so we don't fetch the network for
  // a URL we're going to reject anyway. The authoritative re-check happens
  // inside the serialized mutate below.
  const existing = await store.load();
  if (existing.feeds.some(f => f.url === url)) {
    return { error: 'Feed URL already subscribed' };
  }

  // Fetch the feed to validate and get its title
  const xml = await fetchFeedXml(url);
  if (!xml) {
    return { error: 'Could not fetch feed — check the URL' };
  }

  const parsed = parseFeed(xml);
  const feed = {
    id: randomUUID(),
    url,
    title: parsed.title || new URL(url).hostname,
    addedAt: new Date().toISOString(),
    lastFetched: new Date().toISOString(),
    itemCount: parsed.items.length
  };

  // Add initial items
  const newItems = dedupeByLink(parsed.items).slice(0, MAX_ITEMS_PER_FEED).map(item => ({
    id: randomUUID(),
    feedId: feed.id,
    title: item.title,
    link: item.link,
    description: item.description,
    pubDate: item.pubDate,
    author: item.author,
    read: false,
    fetchedAt: new Date().toISOString()
  }));

  let dup = false;
  await store.mutate((data) => {
    if (data.feeds.some(f => f.url === url)) { dup = true; return data; }
    data.feeds.push(feed);
    data.items.push(...newItems);
    sortItemsNewestFirst(data.items);
    return data;
  });
  if (dup) return { error: 'Feed URL already subscribed' };

  console.log(`📡 Feed added: ${feed.title} (${newItems.length} items)`);

  return { feed: { ...feed, unreadCount: newItems.length } };
}

export async function removeFeed(id) {
  let removedFeed = null;
  await store.mutate((data) => {
    const idx = data.feeds.findIndex(f => f.id === id);
    if (idx === -1) return data;
    removedFeed = data.feeds[idx];
    data.feeds.splice(idx, 1);
    data.items = data.items.filter(i => i.feedId !== id);
    return data;
  });

  if (!removedFeed) return { error: 'Feed not found' };
  console.log(`🗑️ Feed removed: ${removedFeed.title}`);
  return { removed: true };
}

export async function refreshFeed(id) {
  const snapshot = await store.load();
  const feedSnapshot = snapshot.feeds.find(f => f.id === id);
  if (!feedSnapshot) return { error: 'Feed not found' };

  // Network fetch happens outside the write lock; the parsed result is applied
  // under mutate so a concurrent write can't clobber the merge (applyParsedFeed
  // is the shared no-I/O merge — dedupe, trim, bump lastFetched/itemCount/title).
  const xml = await fetchFeedXml(feedSnapshot.url);
  if (!xml) return { error: 'Could not fetch feed' };

  const parsed = parseFeed(xml);

  let result = null;
  await store.mutate((data) => {
    const feed = data.feeds.find(f => f.id === id);
    if (!feed) return data; // removed between fetch and apply
    const newCount = applyParsedFeed(data, feed, parsed);
    sortItemsNewestFirst(data.items);
    const unreadCount = data.items.filter(i => i.feedId === id && !i.read).length;
    result = { feed: { ...feed, unreadCount }, newCount };
    return data;
  });

  if (!result) return { error: 'Feed not found' };
  console.log(`🔄 Feed refreshed: ${result.feed.title} (+${result.newCount} new)`);
  return result;
}

// Fetch and parse XML for a feed without touching the store (safe to run in parallel)
async function fetchAndParseFeed(feed) {
  const xml = await fetchFeedXml(feed.url);
  if (!xml) return { feed, parsed: null, error: 'Could not fetch feed' };
  return { feed, parsed: parseFeed(xml), error: null };
}

// Apply a parsed feed result to a loaded data object (no I/O — safe to call serially)
function applyParsedFeed(data, feed, parsed) {
  const feedRecord = data.feeds.find(f => f.id === feed.id);
  if (!feedRecord) return 0;

  const existingLinks = new Set(data.items.filter(i => i.feedId === feed.id).map(i => i.link));
  const newItems = dedupeByLink(parsed.items, existingLinks)
    .slice(0, MAX_ITEMS_PER_FEED)
    .map(item => ({
      id: randomUUID(),
      feedId: feed.id,
      title: item.title,
      link: item.link,
      description: item.description,
      pubDate: item.pubDate,
      author: item.author,
      read: false,
      fetchedAt: new Date().toISOString()
    }));

  data.items.push(...newItems);

  // Trim to MAX_ITEMS_PER_FEED per feed (keep newest)
  const feedItems = data.items
    .filter(i => i.feedId === feed.id)
    .sort(compareItemsNewestFirst);
  if (feedItems.length > MAX_ITEMS_PER_FEED) {
    const keepIds = new Set(feedItems.slice(0, MAX_ITEMS_PER_FEED).map(i => i.id));
    data.items = data.items.filter(i => i.feedId !== feed.id || keepIds.has(i.id));
  }

  feedRecord.lastFetched = new Date().toISOString();
  feedRecord.itemCount = data.items.filter(i => i.feedId === feed.id).length;
  if (parsed.title) feedRecord.title = parsed.title;

  return newItems.length;
}

// refreshAllFeeds defers sort to a single end-of-batch pass — applyParsedFeed
// runs per feed inside the loop and we'd otherwise re-sort the whole array
// N times for an N-feed refresh.
export async function refreshAllFeeds() {
  const snapshot = await store.load();
  const feedList = [...snapshot.feeds];
  const CONCURRENCY = 5;
  let totalFailed = 0;

  // Fetch in parallel batches (no store I/O), collecting parsed results; the
  // merge into the store happens once under mutate so a concurrent write path
  // can't clobber the batch.
  const parsedResults = [];
  for (let i = 0; i < feedList.length; i += CONCURRENCY) {
    const batch = feedList.slice(i, i + CONCURRENCY);
    const fetched = await Promise.allSettled(batch.map(f => fetchAndParseFeed(f)));
    for (const r of fetched) {
      if (r.status === 'rejected') {
        totalFailed++;
        console.warn(`⚠️ feeds: fetch error: ${r.reason?.message || r.reason}`);
        continue;
      }
      const { feed, parsed, error } = r.value;
      if (error || !parsed) {
        totalFailed++;
        console.warn(`⚠️ feeds: refresh failed for ${feed.id} (${feed.url}): ${error}`);
        continue;
      }
      parsedResults.push({ feed, parsed });
    }
  }

  let totalNew = 0;
  let feedCount = 0;
  await store.mutate((data) => {
    for (const { feed, parsed } of parsedResults) {
      const newCount = applyParsedFeed(data, feed, parsed);
      console.log(`🔄 Feed refreshed: ${feed.title || feed.id} (+${newCount} new)`);
      totalNew += newCount;
    }
    if (totalNew > 0) sortItemsNewestFirst(data.items);
    feedCount = data.feeds.length;
    return data;
  });

  console.log(`📡 All feeds refreshed: +${totalNew} new items, ${totalFailed} failures`);
  return { refreshed: feedCount, newItems: totalNew, failures: totalFailed };
}

export async function getItems({ feedId, unreadOnly, limit, offset = 0 } = {}) {
  await ensureItemsSorted();
  const data = await store.load();
  let items = data.items;

  if (feedId) items = items.filter(i => i.feedId === feedId);
  if (unreadOnly) items = items.filter(i => !i.read);

  if (limit != null) return items.slice(offset, offset + limit);
  return items.slice(offset);
}

export async function markItemRead(itemId) {
  let found = false;
  await store.mutate((data) => {
    const item = data.items.find(i => i.id === itemId);
    if (!item) return data;
    item.read = true;
    found = true;
    return data;
  });
  if (!found) return { error: 'Item not found' };
  return { updated: true };
}

export async function markAllRead(feedId) {
  let count = 0;
  await store.mutate((data) => {
    for (const item of data.items) {
      if (feedId && item.feedId !== feedId) continue;
      if (!item.read) {
        item.read = true;
        count++;
      }
    }
    return data;
  });
  console.log(`✅ Marked ${count} items as read${feedId ? ` for feed ${feedId}` : ''}`);
  return { marked: count };
}

export async function getFeedStats() {
  const data = await store.load();
  // Single-pass per-feed unread tally (O(I)) — mirrors getFeeds()'s approach so
  // the widget can surface top unread feed names without a second computation.
  const unreadCounts = new Map();
  for (const item of data.items) {
    if (!item.read) unreadCounts.set(item.feedId, (unreadCounts.get(item.feedId) || 0) + 1);
  }
  const topUnread = data.feeds
    .map(feed => ({ id: feed.id, title: feed.title, unread: unreadCounts.get(feed.id) || 0 }))
    .filter(f => f.unread > 0)
    .sort((a, b) => b.unread - a.unread)
    .slice(0, 5);
  return {
    totalFeeds: data.feeds.length,
    totalItems: data.items.length,
    unreadItems: data.items.filter(i => !i.read).length,
    topUnread
  };
}

// ─── Internal Helpers ───────────────────────────────────────────────────────

const FEED_HEADERS = {
  'User-Agent': 'PortOS Feed Reader/1.0',
  Accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml',
};

// Fetch a feed's XML through the shared SSRF-guarded fetch. `blockPrivate: true`
// keeps feeds' historical block-all-private posture (an arbitrary user-added
// feed URL must not be aimable at the home network) while gaining the
// connect-time IP PIN that closes the DNS-rebinding TOCTOU (#2046): the
// hostname is resolved once and undici connects to exactly that vetted address,
// re-validated and re-pinned on every redirect hop. `throwOnUnsafe: false`
// preserves this module's contract of returning null on any failure (so addFeed
// surfaces the friendly "Could not fetch feed" message rather than a bubbled
// 400) for an unsafe/private host, network error, non-ok status, or blocked
// redirect.
async function fetchFeedXml(url) {
  return fetchPublicText(url, {
    timeoutMs: FETCH_TIMEOUT_MS,
    headers: FEED_HEADERS,
    blockPrivate: true,
    throwOnUnsafe: false,
  });
}