Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

15.5 KB
Raw Blame
/**
 * Catalog ingest sources — URL / file / voice-memo / brain-bridge entry points.
 *
 * Each source produces raw text + a scrap with a typed `source_kind`, then
 * hands off to the SAME extraction pipeline the textarea-paste flow uses
 * (`extractIngredients`). So they emit identical `catalog:extract:progress`
 * frames and the client lands on the same paste→review→commit UI.
 *
 *   url          — fetch the page via the browser service, pull main text,
 *                  source_kind 'url', metadata { url, title }.
 *   file         — text already read client-side (.txt/.md), source_kind 'file',
 *                  metadata { filename, mime }.
 *   voice-memo   — base64 WAV → Whisper transcript, audio persisted under
 *                  data/audio to mint a media_key, source_kind 'voice-memo',
 *                  metadata { mediaKey, mimeType, durationApproxMs? }.
 *   brain-bridge — an existing brain record (idea / memory / project / admin /
 *                  person), its prose folded into one rawText block,
 *                  source_kind 'brain-bridge', metadata { brainType, brainId,
 *                  brainTitle, brainTags }. This is the ingestion bridge that
 *                  makes the catalog the universal sink for captured thoughts.
 *
 * Keeping this orchestration out of the route handlers lets the unit tests
 * mock the network/Whisper boundaries without spinning up Express.
 */

import { randomUUID } from 'crypto';
import { join } from 'path';
import * as catalogDB from './catalogDB.js';
import * as brainStorage from './brainStorage.js';
import { extractIngredientsForScrap } from './catalogExtraction.js';
import { transcribe } from './voice/stt.js';
import { navigateToUrlPinned } from './browserService.js';
import { lookup } from 'dns/promises';
import { PATHS, ensureDir, safeJSONParse, writeFileGuarded } from '../lib/fileUtils.js';
import { isSafeIngestUrl, isBlockedIngestHost } from '../lib/catalogValidation.js';
import { withBackupAssetPublication } from '../lib/backupSnapshotBoundary.js';

// Cap fetched/transcribed bodies at the scrap column boundary (the Zod
// rawText max). A runaway page or a multi-hour memo can't blow past the DB.
const RAW_TEXT_MAX = 2_000_000;

// Browser settle delay between navigate and innerText read. The CDP
// `/json/new` resolves as soon as the tab exists, not when the DOM is ready,
// so a same-tick innerText read returns an empty/loading body.
const PAGE_SETTLE_MS = 2_500;

const clampText = (s) => (typeof s === 'string' ? s.slice(0, RAW_TEXT_MAX) : '');


/**
 * Shared tail for every source: create a typed scrap and run the SAME
 * extraction pipeline the textarea-paste flow uses, returning `{ scrap, draft }`
 * (the exact shape the existing /scraps/:id/extract route returns, so the
 * client review flow is identical). The three source functions differ only in
 * how they PRODUCE rawText / title / metadata; this codifies the "one pipeline"
 * contract the module header describes. `log` receives the created scrap.
 */
async function createScrapAndExtract({ title, rawText, sourceKind, metadata, providerOverride, log, namesNewAsset = false }) {
  // Chunk long pastes identically to the textarea flow — createChunkedScrap
  // returns the PARENT scrap; extraction budgets its complete text for the selected model.
  const createScrap = () => catalogDB.createChunkedScrap({ title, rawText, sourceKind, metadata });
  // A scrap that first names bytes written just before it commits under the
  // backup lease (#9982); the extraction that follows stays outside it.
  const scrap = await (namesNewAsset ? withBackupAssetPublication(createScrap) : createScrap());
  const draft = await extractIngredientsForScrap({ scrapId: scrap.id, providerOverride });
  if (log) console.log(log(scrap));
  return { scrap, draft };
}

/**
 * Full SSRF gate for an ingest URL: scheme + blocked-literal check (sync), AND
 * a DNS resolve so a hostname whose A record points at a blocked address
 * (cloud metadata / loopback / link-local) is rejected too. Applied to BOTH the
 * request-body URL and the post-redirect landed URL — a redirect to such a
 * hostname must be caught the same way as the first hop. (Residual TOCTOU —
 * Chrome re-resolves independently — is acceptable on a single-user private
 * network; this closes the realistic record-points-at-metadata vector.)
 */
async function assertIngestUrlSafe(target) {
  if (!isSafeIngestUrl(target)) {
    throw new Error('refusing to ingest a non-http(s) or loopback/link-local URL');
  }
  const { hostname } = new URL(target);
  const isIpLiteral = /^[\d.]+$/.test(hostname) || hostname.includes(':');
  if (!isIpLiteral) {
    const resolved = await lookup(hostname).catch(() => null);
    if (resolved?.address && isBlockedIngestHost(resolved.address)) {
      throw new Error('refusing to ingest a host that resolves to a blocked (loopback/link-local) address');
    }
  }
}

/**
 * Fetch a URL through the browser service and return its main text + title.
 * Uses CDP (the same headed/headless Chrome the rest of PortOS drives) so
 * JS-rendered pages and login-walled content the user is already signed into
 * resolve correctly — a bare `fetch()` would only see the server-rendered
 * HTML shell of an SPA.
 *
 * SSRF pin: Chrome re-resolves DNS independently of our pre-check, so a rebinding
 * DNS answer could steer Chrome's connection to a blocked address after
 * `assertIngestUrlSafe` vetted the hostname (a TOCTOU). `navigateToUrlPinned`
 * closes that window by verifying, against Chrome's OWN reported
 * `remoteIPAddress`, that EVERY main-frame hop — the initial navigation, each
 * HTTP redirect, AND any client-side navigation during the `settleMs` window
 * before the DOM is read — connected to an allowed address, refusing (and
 * tearing the tab down) otherwise. The `isBlockedIngestHost` blocklist is the
 * SAME criterion `assertIngestUrlSafe` vets with, so the pin and the pre-check
 * agree. The pin owns the settle wait, so there is no separate stale-DNS re-vet
 * (which would itself be rebindable) after it returns.
 *
 * Returns `{ text, title, finalUrl }`. Throws when the browser can't reach
 * the page or extracts nothing usable, so the route surfaces a clean 4xx/5xx.
 */
export async function fetchUrlMainText(url, { settleMs = PAGE_SETTLE_MS } = {}) {
  await assertIngestUrlSafe(url);

  // Prefer the page's own <article>/<main> if present (skips nav/footer chrome),
  // falling back to document body text. Runs in the page; returns a plain string
  // so returnByValue marshals it cleanly.
  const expression = `(() => {
    const pick = document.querySelector('article') || document.querySelector('main') || document.body;
    const title = (document.title || '').trim();
    const text = (pick && pick.innerText ? pick.innerText : '').trim();
    return JSON.stringify({ title, text });
  })()`;

  // Navigate in a fresh tab and PIN: verify Chrome's actual per-hop connection
  // IP is not a blocked (loopback / link-local / metadata) address for the
  // initial nav, every redirect, and any client-side nav during the settle
  // window — closing the rebinding gap between our dns.lookup above and Chrome's
  // own resolution. The DOM read runs on the SAME pinned CDP session (via
  // evaluateExpression), re-verified after the read, so no late navigation slips
  // between "stop monitoring" and "read". Fails closed (tab torn down, throws)
  // on any disallowed hop. Supplying `evaluateExpression` also makes it tear the
  // tab down on SUCCESS — the read already happened on that CDP session, so the
  // tab is scratch and `page.id` names a closed tab by the time we get here.
  // Every ingested URL would otherwise leave one open in the user's browser.
  const page = await navigateToUrlPinned(url, {
    verifyRemoteIp: (ip) => !isBlockedIngestHost(ip),
    settleMs,
    evaluateExpression: expression,
  });

  const parsed = page.evalResult ? safeJSONParse(page.evalResult, null) : null;
  const text = clampText(parsed?.text || '');
  if (!text.trim()) throw new Error('no readable text extracted from page');
  return { text, title: parsed?.title || page.title || '', finalUrl: page.url || url };
}

/**
 * Ingest from a URL: fetch + extract main text, create a 'url' scrap, run the
 * extraction pipeline. Returns `{ scrap, draft }` — same shape as the existing
 * /scraps/:id/extract route so the client review flow is unchanged.
 */
export async function ingestFromUrl({ url, providerOverride, settleMs } = {}) {
  const { text, title, finalUrl } = await fetchUrlMainText(url, settleMs !== undefined ? { settleMs } : {});
  return createScrapAndExtract({
    title: title || finalUrl,
    rawText: text,
    sourceKind: 'url',
    metadata: { url: finalUrl, title: title || null },
    providerOverride,
    log: (s) => `🌐 Catalog URL ingest: ${finalUrl} → scrap ${s.id} (${text.length} chars)`,
  });
}

/**
 * Ingest from an uploaded text file. The client reads .txt/.md text locally
 * and posts it with the original filename/mime; we record provenance in scrap
 * metadata and run the pipeline.
 */
export async function ingestFromFile({ text, filename, mime, providerOverride } = {}) {
  const rawText = clampText(text);
  if (!rawText.trim()) throw new Error('file contained no extractable text');
  return createScrapAndExtract({
    title: filename,
    rawText,
    sourceKind: 'file',
    metadata: { filename, mime: mime || null },
    providerOverride,
    log: (s) => `📄 Catalog file ingest: ${filename} → scrap ${s.id} (${rawText.length} chars)`,
  });
}

/**
 * Persist a recorded memo's audio under data/audio and return its filename —
 * the `media_key` the catalog uses to reference media without storing bytes
 * in the scrap. Mirrors the audio path PATHS.audio is already used for.
 */
async function persistVoiceMemoAudio(audioBuffer, mimeType) {
  // WAV is the only format the client encodes (whisper.cpp accepts WAV only),
  // so default the extension to .wav; honor an explicit webm/mp3 mime if the
  // client ever sends a pre-encoded blob.
  const ext = mimeType?.includes('webm') ? 'webm' : mimeType?.includes('mpeg') ? 'mp3' : 'wav';
  const mediaKey = `voice-memo-${randomUUID()}.${ext}`;
  await ensureDir(PATHS.audio);
  await writeFileGuarded(join(PATHS.audio, mediaKey), audioBuffer);
  return mediaKey;
}

/**
 * Ingest from a recorded voice memo: decode base64 WAV → Whisper transcript →
 * persist audio (mint media_key) → 'voice-memo' scrap → pipeline. The audio
 * media_key rides in scrap metadata so a later commit can attach it to the
 * resulting ingredient via the media-refs table.
 *
 * `deps` lets tests inject a fake transcriber / persister without a running
 * Whisper server or touching disk.
 */
export async function ingestFromVoice(
  { audioBase64, mimeType = 'audio/wav', title, providerOverride } = {},
  { transcribeFn = transcribe, persistFn = persistVoiceMemoAudio } = {},
) {
  const audioBuffer = Buffer.from(audioBase64, 'base64');
  if (audioBuffer.byteLength === 0) throw new Error('voice memo audio was empty');

  const { text } = await transcribeFn(audioBuffer, { mimeType });
  const transcript = clampText((text || '').trim());
  if (!transcript) throw new Error('voice memo produced an empty transcript');

  // Persist the audio AFTER a successful transcription so a failed/empty memo
  // doesn't litter data/audio with orphan files.
  const mediaKey = await persistFn(audioBuffer, mimeType);

  const { scrap, draft } = await createScrapAndExtract({
    title: title?.trim() || 'Voice memo',
    rawText: transcript,
    sourceKind: 'voice-memo',
    metadata: { mediaKey, mimeType },
    providerOverride,
    namesNewAsset: true,
    log: (s) => `🎙️ Catalog voice ingest: ${mediaKey} → scrap ${s.id} (${transcript.length} chars)`,
  });
  return { scrap, draft, mediaKey };
}

// Supported brain record types → their by-id storage getter. Mirrors
// `CATALOG_BRAIN_INGEST_TYPES` in catalogValidation.js (the route validates the
// type against that enum before we get here; this map is the resolution side).
// We import the leaf `brainStorage` rather than `brain.js` so this stays a pure
// read — no AI/classification side effects ride along.
const BRAIN_INGEST_GETTERS = {
  ideas: brainStorage.getIdeaById,
  memories: brainStorage.getMemoryEntryById,
  projects: brainStorage.getProjectById,
  admin: brainStorage.getAdminById,
  people: brainStorage.getPersonById,
};

/**
 * Fold a brain record's prose into a single ingest-ready text block. Each brain
 * type keeps its body in different fields (an idea's `oneLiner` + `notes`, a
 * memory's `content`, a project/admin's `nextAction` + `notes`, a person's
 * `context` + `followUps`), so map per-type rather than guessing. The record's
 * title/name is prepended as context for the extractor, and any record tags ride
 * along in metadata so the user can keep them on the committed ingredients.
 * Returns `{ title, rawText, tags }`.
 */
export function brainRecordToIngestText(brainType, record) {
  const title = (record?.title || record?.name || '').trim();
  const parts = [];
  const push = (v) => { if (typeof v === 'string' && v.trim()) parts.push(v.trim()); };
  switch (brainType) {
    case 'ideas':
      push(record?.oneLiner);
      push(record?.notes);
      break;
    case 'memories':
      push(record?.content);
      break;
    case 'projects':
    case 'admin':
      if (record?.nextAction) push(`Next: ${record.nextAction}`);
      push(record?.notes);
      break;
    case 'people':
      push(record?.context);
      if (Array.isArray(record?.followUps) && record.followUps.length) {
        push(`Follow-ups: ${record.followUps.join('; ')}`);
      }
      break;
    default:
      // Best-effort for any future type the enum admits before this map does.
      for (const k of ['oneLiner', 'content', 'notes', 'context', 'summary', 'description']) {
        push(record?.[k]);
      }
  }
  const body = parts.join('\n\n').trim();
  const rawText = clampText(title ? `${title}\n\n${body}`.trim() : body);
  const tags = Array.isArray(record?.tags) ? record.tags.filter((t) => typeof t === 'string') : [];
  return { title: title || 'Brain entry', rawText, tags };
}

/**
 * Ingest from an existing brain record. Resolves the record by `{ brainType,
 * brainId }`, folds its prose into rawText, and runs the same scrap→extract
 * pipeline as every other source — returning `{ scrap, draft }` so the client
 * lands on the identical review phase. This is the brain → catalog bridge: a
 * captured idea/journal/etc. becomes typed catalog ingredients the user reviews
 * and commits, making the catalog the universal sink for creative atoms.
 */
export async function ingestFromBrain({ brainType, brainId, providerOverride } = {}) {
  const getter = BRAIN_INGEST_GETTERS[brainType];
  if (!getter) throw new Error(`unsupported brain type for catalog ingest: ${brainType}`);
  const record = await getter(brainId);
  if (!record) throw new Error(`brain ${brainType} entry not found: ${brainId}`);
  // rawText is already trimmed by brainRecordToIngestText (and folds in the
  // title), so a falsy check is sufficient to catch a record with no usable
  // text at all (no title AND no body). A title-only record is intentionally
  // ingestible — the user explicitly chose to send it.
  const { title, rawText, tags } = brainRecordToIngestText(brainType, record);
  if (!rawText) throw new Error('brain entry had no text to ingest');
  return createScrapAndExtract({
    title,
    rawText,
    sourceKind: 'brain-bridge',
    metadata: { brainType, brainId, brainTitle: title, brainTags: tags },
    providerOverride,
    log: (s) => `🧠 Catalog brain ingest: ${brainType}/${brainId} → scrap ${s.id} (${rawText.length} chars)`,
  });
}