Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

9.3 KB
Raw Blame
/**
 * Mood Board — Pinterest importer (link + manual sync).
 *
 * A mood board can be linked to a public Pinterest board's RSS feed. "Sync now"
 * fetches the feed, downloads each NEW pin's image into the local gallery
 * (`data/images/`, so it federates as a normal board-item asset and survives
 * Pinterest link rot), and appends it as an image item — deduping by the pin
 * permalink stored as the item's `source`. The very first sync on a freshly
 * linked board pulls everything the feed exposes; later syncs add only what's
 * new. NOTE: Pinterest's RSS returns the most-recent pins only (~25), NOT the
 * full board — the UI states this; we never claim a complete backfill.
 *
 * Network I/O (feed fetch + image downloads) runs OUTSIDE the board row lock;
 * the resulting items land in a single locked `appendPinterestItems` write.
 * Re-exported through index.js (the public moodBoard entry) so the route imports
 * one module; the federation emit is fired here after each store mutation.
 */

import { createHash } from 'crypto';
import { join } from 'path';
import { ServerError } from '../../lib/errorHandler.js';
import { PATHS, ensureDir, detectImageFormat, writeFileGuarded } from '../../lib/fileUtils.js';
import { normalizePinterestFeedUrl, parsePinterestRss } from '../../lib/pinterestFeed.js';
import { fetchPublicText, fetchPublicBinary } from '../../lib/safeUrlFetch.js';
import { withBackupAssetPublication } from '../../lib/backupSnapshotBoundary.js';
import { emitRecordUpdated } from '../sharing/recordEvents.js';
import { MAX_ITEMS_PER_BOARD } from './logic.js';
import * as store from './db.js';
import { resolvePinterestBoardUrl } from './pinterestUrl.js';

const FEED_TIMEOUT_MS = 20000;
const IMAGE_TIMEOUT_MS = 20000;
const MAX_IMAGE_BYTES = 12 * 1024 * 1024;
// Pinterest 403s obviously-bot user-agents on its CDN; present a browsery one.
const PINTEREST_UA = 'Mozilla/5.0 (compatible; PortOS Mood Board/1.0; +https://github.com/atomantic/PortOS)';
const FEED_HEADERS = { 'User-Agent': PINTEREST_UA, Accept: 'application/rss+xml, application/xml, text/xml' };
const IMAGE_HEADERS = { 'User-Agent': PINTEREST_UA, Accept: 'image/*' };
// One origin (i.pinimg.com) serves every pin and 403s bot-ish traffic, so keep
// the download fan-out small — fast enough for a 25-pin first sync, polite enough
// to avoid rate-limiting. The caller ensures data/images/ exists once up front.
const DOWNLOAD_CONCURRENCY = 3;

/**
 * Download a pin's image into data/images/ and return its served path
 * (`/data/images/<file>`), or null on failure. Tries the upsized (736x) URL
 * first, falling back to the original feed URL if it 404s. The filename is keyed
 * on the pin permalink so re-downloading the same pin is idempotent. Format +
 * extension are sniffed from the actual bytes (via detectImageFormat), which
 * also rejects a non-image body (an error/HTML page served with 200) — more
 * trustworthy than Pinterest's Content-Type header.
 */
export async function downloadPinImage({ pinUrl, imageUrl, imageUrlOriginal }) {
  let res = await fetchPublicBinary(imageUrl, { timeoutMs: IMAGE_TIMEOUT_MS, headers: IMAGE_HEADERS, maxBytes: MAX_IMAGE_BYTES });
  if (!res?.buffer?.length && imageUrlOriginal && imageUrlOriginal !== imageUrl) {
    res = await fetchPublicBinary(imageUrlOriginal, { timeoutMs: IMAGE_TIMEOUT_MS, headers: IMAGE_HEADERS, maxBytes: MAX_IMAGE_BYTES });
  }
  if (!res?.buffer?.length) return null;
  const fmt = detectImageFormat(res.buffer);
  if (!fmt) return null;
  const filename = `pinterest-${createHash('sha1').update(pinUrl).digest('hex').slice(0, 16)}${fmt.ext}`;
  // Pin-keyed, so a re-download rewrites bytes a board may already name: hold
  // the backup lease (#9982) so the in-place write never overlaps a snapshot.
  await withBackupAssetPublication(() => writeFileGuarded(join(PATHS.images, filename), res.buffer));
  return `/data/images/${filename}`;
}

/**
 * Link a board to a Pinterest board URL (or `.rss` feed URL). Validates +
 * normalizes the URL (throws 400 on a non-Pinterest host) and stores the link.
 */
export async function linkPinterestBoard(boardId, { url }) {
  const { feedUrl, boardUrl, isSection } = await resolvePinterestBoardUrl(url);
  const board = await store.setPinterestLink(boardId, { feedUrl, boardUrl });
  emitRecordUpdated('moodBoard', boardId);
  console.log(`📌 Pinterest link set: board ${boardId} → ${feedUrl}${isSection ? ' (section URL — feed covers the whole board; Pinterest has no per-section RSS)' : ''}`);
  return board;
}

export async function unlinkPinterestBoard(boardId) {
  const board = await store.clearPinterestLink(boardId);
  emitRecordUpdated('moodBoard', boardId);
  return board;
}

/**
 * Sync the linked Pinterest board: fetch the feed, download every NEW pin's
 * image, append them in one locked write. Returns `{ board, added, skipped,
 * feedCount }`. Throws 404 (no board), 400 (not linked), or 502 (feed
 * unreachable).
 */
export async function syncPinterestBoard(boardId) {
  let board = await store.getBoard(boardId);
  if (!board) throw new ServerError('Mood board not found', { status: 404, code: 'NOT_FOUND' });
  if (!board.pinterest?.feedUrl) throw new ServerError('This board is not linked to a Pinterest board', { status: 400, code: 'NOT_LINKED' });

  // Self-heal links saved before section detection: a section URL was stored with
  // a `/user/board/section.rss` feed that Pinterest serves as HTML (0 pins). Re-
  // normalize the stored link (preferring the displayed boardUrl, which retains
  // the section path) and persist the corrected board-level feed in place. The
  // heal is guarded under lock (healPinterestFeed) so it can't resurrect/clobber
  // a link the user unlinked or repointed concurrently — same re-entrancy guard
  // the locked append below uses via expectedFeedUrl.
  const staleFeedUrl = board.pinterest.feedUrl;
  const { feedUrl: healedFeedUrl, boardUrl: healedBoardUrl } =
    normalizePinterestFeedUrl(board.pinterest.boardUrl || staleFeedUrl);
  if (healedFeedUrl !== staleFeedUrl) {
    const { board: healed, changed } = await store.healPinterestFeed(boardId, {
      fromFeedUrl: staleFeedUrl, feedUrl: healedFeedUrl, boardUrl: healedBoardUrl,
    });
    board = healed;
    if (changed) {
      // The heal persisted a feed-URL correction — emit now so federation/UI
      // pick it up even if the fetch below fails (otherwise peers keep the stale
      // section feed until a later successful sync).
      emitRecordUpdated('moodBoard', boardId);
      console.log(`📌 Pinterest sync: board ${boardId} feed corrected to ${healedFeedUrl} (section URL has no RSS)`);
    }
  }
  // This sync is committed to `healedFeedUrl` (== staleFeedUrl when no heal was
  // needed). If a concurrent unlink/repoint during the heal left the persisted
  // board on a different feed, abort rather than sync a feed this run wasn't
  // started for — same guard the locked append applies via expectedFeedUrl.
  const feedUrl = healedFeedUrl;
  if (board.pinterest?.feedUrl !== feedUrl) {
    console.log(`📌 Pinterest sync: board ${boardId} aborted — link changed mid-sync`);
    return { board, added: 0, feedCount: 0, aborted: true };
  }

  const xml = await fetchPublicText(feedUrl, { timeoutMs: FEED_TIMEOUT_MS, headers: FEED_HEADERS });
  if (!xml) throw new ServerError('Could not fetch the Pinterest feed (it may be private or rate-limited)', { status: 502, code: 'FEED_FETCH_FAILED' });

  const pins = parsePinterestRss(xml);
  const items = Array.isArray(board.items) ? board.items : [];
  const seen = new Set(items.map((it) => it?.source).filter(Boolean));
  const capacity = Math.max(0, MAX_ITEMS_PER_BOARD - items.length);
  // Pre-dedupe by pin permalink BEFORE downloading so we don't fetch images we'd
  // discard, and cap to remaining board capacity to avoid orphan downloads.
  const candidates = pins.filter((p) => p.pinUrl && !seen.has(p.pinUrl)).slice(0, capacity);

  // Download in small concurrent batches (preserving feed order) instead of one
  // at a time — the candidates are independent, sha1-named by permalink.
  await ensureDir(PATHS.images);
  const imported = [];
  for (let i = 0; i < candidates.length; i += DOWNLOAD_CONCURRENCY) {
    const batch = candidates.slice(i, i + DOWNLOAD_CONCURRENCY);
    const results = await Promise.all(batch.map(async (pin) => {
      const localUrl = await downloadPinImage(pin);
      // A pin's title and description are someone else's words, so none is kept.
      return localUrl ? { imageUrl: localUrl, source: pin.pinUrl } : null;
    }));
    for (const r of results) if (r) imported.push(r);
  }

  // Pass the feed we actually fetched so the locked append aborts if the user
  // unlinked / repointed the board while downloads ran (fetch is outside the lock).
  const syncedAt = new Date().toISOString();
  // The append first names the downloaded pins, so it commits under the lease.
  const { board: nextBoard, added, aborted } = await withBackupAssetPublication(
    () => store.appendPinterestItems(boardId, imported, { syncedAt, expectedFeedUrl: feedUrl }),
  );
  if (aborted) {
    console.log(`📌 Pinterest sync: board ${boardId} aborted — link changed mid-sync`);
    return { board: nextBoard, added: 0, feedCount: pins.length, aborted: true };
  }
  emitRecordUpdated('moodBoard', boardId);
  console.log(`📌 Pinterest sync: board ${boardId} +${added} new (${pins.length} pins in feed)`);
  return { board: nextBoard, added, feedCount: pins.length };
}