Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.8 KB
Raw Blame
/**
 * Pure parsers for the catalog bulk-import path. Dependency-free —
 * markdown / CSV / JSON each normalize to a list of
 * `{ type, name, payload, tags }` entries that the route then validates
 * one-by-one against `catalogIngredientCreateSchema` before insert.
 *
 * The parsers are intentionally lenient (best-effort); the Zod gate after
 * parse is what guarantees DB-safe shape. Each parser throws on structural
 * failure (not on per-row content errors) — those bubble up as the
 * `BULK_IMPORT_PARSE_FAILED` error.
 *
 * `INGREDIENT_TYPES` + `PRIMARY_CONTENT_KEY_BY_TYPE` derive from the shared
 * type registry (`catalogTypes.js`, a pure no-side-effect module) so a new
 * type flows through bulk import/export automatically. A bulk-import row's
 * body paragraph / CSV `description` column lands in each type's
 * `primaryContentKey`.
 *
 * Also exports a minimal `toYamlString()` serializer used by the export
 * route — handwritten so we don't pull js-yaml just for the catalog bundle.
 */

import { CATALOG_TYPES, INGREDIENT_TYPE_IDS } from './catalogTypes.js';

export const INGREDIENT_TYPES = INGREDIENT_TYPE_IDS;

export const PRIMARY_CONTENT_KEY_BY_TYPE = Object.freeze(
  Object.fromEntries(CATALOG_TYPES.map((t) => [t.id, t.primaryContentKey])),
);

const TYPE_LOWER = new Set(INGREDIENT_TYPES);

function normalizeType(raw) {
  if (typeof raw !== 'string') return null;
  const t = raw.trim().toLowerCase();
  return TYPE_LOWER.has(t) ? t : null;
}

function normalizeTags(raw) {
  if (Array.isArray(raw)) {
    return raw
      .map((t) => (typeof t === 'string' ? t.trim() : ''))
      .filter(Boolean);
  }
  if (typeof raw === 'string') {
    return raw
      .split(',')
      .map((s) => s.trim())
      .filter(Boolean);
  }
  return [];
}

/**
 * Normalize one freeform entry into the import shape.
 * Returns `{ type, name, payload, tags }` or throws.
 */
function normalizeEntry(input, index) {
  if (!input || typeof input !== 'object') {
    throw new Error(`entry[${index}]: not an object`);
  }
  const type = normalizeType(input.type);
  if (!type) {
    throw new Error(`entry[${index}]: invalid or missing type (got ${JSON.stringify(input.type)})`);
  }
  const name = typeof input.name === 'string' ? input.name.trim() : '';
  if (!name) {
    throw new Error(`entry[${index}]: name is required`);
  }
  // Caller may supply payload directly OR a top-level `description` /
  // `summary` shortcut that flows into the type's primary content key.
  const explicitPayload = input.payload && typeof input.payload === 'object' && !Array.isArray(input.payload)
    ? { ...input.payload }
    : {};
  if (typeof input.description === 'string' && input.description.trim()) {
    const key = PRIMARY_CONTENT_KEY_BY_TYPE[type];
    // Don't overwrite an explicit payload key — the explicit one wins.
    if (!(key in explicitPayload)) explicitPayload[key] = input.description.trim();
  }
  return {
    type,
    name,
    payload: explicitPayload,
    tags: normalizeTags(input.tags),
  };
}

/**
 * JSON: accepts either a bare array of entries OR a full export bundle
 * (`{ version, ref, ingredients: [...] }`) so the export → edit → re-import
 * round trip works without the user having to strip the wrapper.
 */
export function parseJsonBulk(payload) {
  if (typeof payload !== 'string') {
    throw new Error('json payload must be a string');
  }
  let parsed;
  try {
    parsed = JSON.parse(payload);
  } catch (err) {
    throw new Error(`invalid JSON: ${err.message}`);
  }
  const isBundle = !Array.isArray(parsed) && parsed && Array.isArray(parsed.ingredients);
  const rawEntries = Array.isArray(parsed)
    ? parsed
    : (isBundle ? parsed.ingredients : null);
  if (!rawEntries) {
    throw new Error('json payload must be an array of entries or an export bundle with an `ingredients` array');
  }
  const entries = rawEntries.map((entry, i) => {
    const normalized = normalizeEntry(entry, i);
    // Export identity and attachments are validated by the bulk-entry schema.
    for (const key of ['id', 'media', 'scraps']) {
      if (entry[key] !== undefined) attachNonEnumerable(normalized, key, entry[key]);
    }
    // Preserve the per-row role the export bundle stamped so the bulk-import
    // route can re-create each ingredient's ref link with its original role,
    // instead of collapsing every row onto the batch-level default. Rides as
    // a non-enumerable field so deep-equal tests on the entry shape still pass.
    if (entry && typeof entry.roleForExportedRef === 'string' && entry.roleForExportedRef.trim()) {
      attachNonEnumerable(normalized, 'roleForExportedRef', entry.roleForExportedRef.trim());
    }
    return normalized;
  });
  // When the input is a full export bundle, surface its `ref` so the route can
  // fall back to it for ref-links (re-import recreates the bundle's slice
  // membership). Non-enumerable, like markdown's `warnings`, so array-indexing
  // callers are unaffected.
  if (isBundle && parsed.ref && typeof parsed.ref === 'object'
      && typeof parsed.ref.kind === 'string' && parsed.ref.id != null) {
    attachNonEnumerable(entries, 'bundleRef', { kind: parsed.ref.kind, id: String(parsed.ref.id) });
  }
  return entries;
}

/**
 * CSV: dependency-free parser. Header row required; supports basic
 * double-quoted fields ("" = literal quote, embedded commas allowed).
 * Recognized headers: `type`, `name`, `description`, `tags`. Unknown
 * columns are folded into `payload[colName]`.
 */
export function parseCsvBulk(payload) {
  if (typeof payload !== 'string') {
    throw new Error('csv payload must be a string');
  }
  const rows = tokenizeCsv(payload);
  if (rows.length < 2) {
    throw new Error('csv requires a header row and at least one data row');
  }
  const headers = rows[0].map((h) => h.trim());
  const typeIdx = headers.indexOf('type');
  const nameIdx = headers.indexOf('name');
  if (typeIdx < 0 || nameIdx < 0) {
    throw new Error('csv headers must include `type` and `name`');
  }
  const out = [];
  for (let r = 1; r < rows.length; r++) {
    const row = rows[r];
    // Skip blank lines (tokenizer leaves a single empty string per blank).
    if (row.length === 1 && row[0] === '') continue;
    const entry = { type: row[typeIdx], name: row[nameIdx], payload: {}, tags: [] };
    for (let c = 0; c < headers.length; c++) {
      const h = headers[c];
      const v = row[c] ?? '';
      if (h === 'type' || h === 'name') continue;
      if (h === 'tags') {
        entry.tags = normalizeTags(v);
      } else if (h === 'description') {
        entry.description = v;
      } else if (h) {
        // Unknown column → stash in payload so users can pre-populate
        // canon-shape keys (e.g. `personality,role,significance,…`) without
        // a custom format.
        if (v !== '') entry.payload[h] = v;
      }
    }
    out.push(normalizeEntry(entry, r - 1));
  }
  if (out.length === 0) {
    throw new Error('csv produced zero data rows');
  }
  return out;
}

function tokenizeCsv(text) {
  const rows = [];
  let row = [];
  let cur = '';
  let inQuotes = false;
  let i = 0;
  const n = text.length;
  while (i < n) {
    const ch = text[i];
    if (inQuotes) {
      if (ch === '"') {
        if (text[i + 1] === '"') {
          cur += '"';
          i += 2;
          continue;
        }
        inQuotes = false;
        i++;
        continue;
      }
      cur += ch;
      i++;
      continue;
    }
    if (ch === '"') { inQuotes = true; i++; continue; }
    if (ch === ',') { row.push(cur); cur = ''; i++; continue; }
    if (ch === '\r') {
      // Swallow lone \r; handle \r\n via the \n branch
      if (text[i + 1] === '\n') { i++; continue; }
      // Treat bare \r as line terminator
      row.push(cur); cur = ''; rows.push(row); row = []; i++;
      continue;
    }
    if (ch === '\n') {
      row.push(cur); cur = ''; rows.push(row); row = []; i++;
      continue;
    }
    cur += ch;
    i++;
  }
  // Final cell + row (no trailing newline case).
  row.push(cur);
  // If the only thing in the final row is an empty cell AND there were prior
  // rows, drop it (trailing newline artifact).
  if (!(row.length === 1 && row[0] === '' && rows.length > 0)) {
    rows.push(row);
  }
  return rows;
}

/**
 * Markdown convention:
 *   ## <Type>: <Name>
 *   <body paragraph(s) — primary content for the type>
 *   tags: <comma, separated, tags>
 *
 * The `tags:` line is optional and case-insensitive; everything else above
 * it (until the next `## ` heading) becomes the body. Body lands in the
 * type's primary content key per PRIMARY_CONTENT_KEY_BY_TYPE.
 *
 * This mirrors `ingredientToMarkdown` for everything that format emits, so it
 * also recognizes the two structured sections that exporter writes inside a
 * `## ` block:
 *   - a ` ```json … ``` ` fence carrying the non-primary payload keys, which
 *     is parsed back into `payload.*` (merged under the body's primary key);
 *   - a `### Scraps` subsection of `- (kind) text` bullets, preserved as a
 *     sibling `scraps[]` array for a lossless round-trip.
 * (Per-row export role rides only the JSON bundle's `roleForExportedRef`; the
 * markdown format does not carry it, so markdown re-import falls back to the
 * route's default role — lossy on role by design.)
 * Both terminate the body so re-importing an export doesn't pollute the
 * primary content key with fence text or scrap bullets.
 */
export function parseMarkdownBulk(payload) {
  if (typeof payload !== 'string') {
    throw new Error('markdown payload must be a string');
  }
  const lines = payload.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n');
  // Heading regex: `## <Type>: <Name>` — case-insensitive on the type, name
  // is everything to the right of the first `:`. Heading must start the line.
  const headingRe = /^##\s+([A-Za-z][A-Za-z-]*)\s*:\s*(.+?)\s*$/;
  // `### Scraps` subsection header (case-insensitive).
  const scrapsHeadingRe = /^###\s+scraps\s*$/i;
  // `- (sourceKind) raw text…` scrap bullet emitted by ingredientToMarkdown.
  const scrapBulletRe = /^-\s+\(([^)]*)\)\s*(.*)$/;
  const out = [];
  const warnings = [];
  let current = null;
  const flush = () => {
    if (!current) return;
    const bodyText = current.bodyLines.join('\n').trim();
    if (bodyText) {
      const key = PRIMARY_CONTENT_KEY_BY_TYPE[current.type];
      // Don't clobber a fence-supplied key with the body, and vice-versa: the
      // fence holds the non-primary keys, so the body's primary key wins here.
      current.payload[key] = bodyText;
    }
    out.push({
      type: current.type,
      name: current.name,
      payload: current.payload,
      tags: current.tags,
      scraps: current.scraps,
    });
  };
  for (const line of lines) {
    const m = headingRe.exec(line);
    if (m) {
      flush();
      const type = normalizeType(m[1]);
      const name = m[2].trim();
      if (!type) {
        // Skip unrecognized headings (e.g. `## Notes:`, typos like `## Plce:`);
        // they'd otherwise open a malformed section. Surface as a warning so
        // the user notices typos; the body until the next valid heading is
        // dropped.
        warnings.push(`Unknown type "${m[1]}" in heading "${line.trim()}" — section skipped`);
        current = null;
        continue;
      }
      current = { type, name, bodyLines: [], tags: [], payload: {}, scraps: [], section: 'body', fence: null };
      continue;
    }
    if (!current) continue;
    // Inside a ```json fence: accumulate raw lines until the closing ```.
    if (current.fence) {
      if (/^```\s*$/.test(line)) {
        const raw = current.fence.lines.join('\n');
        let parsedFence = null;
        try { parsedFence = JSON.parse(raw); } catch { parsedFence = null; }
        if (parsedFence && typeof parsedFence === 'object' && !Array.isArray(parsedFence)) {
          // Merge the non-primary payload keys back in. The body's primary
          // key is applied in flush() and wins on collision.
          for (const [k, v] of Object.entries(parsedFence)) current.payload[k] = v;
        } else {
          warnings.push(`Malformed JSON fence in "## ${capitalize(current.type)}: ${current.name}" — skipped`);
        }
        current.fence = null;
      } else {
        current.fence.lines.push(line);
      }
      continue;
    }
    // Open a ```json fence — the ONLY fence `ingredientToMarkdown` emits,
    // carrying the non-primary payload keys. Other fences (```js, ```text,
    // bare ```) are intentionally left in the body so a hand-authored markdown
    // import doesn't silently lose fenced content. Fence state is driven by
    // `current.fence`; `section` only gates scraps, so the body resumes after
    // the closing ```.
    const fenceOpen = /^```(\w*)\s*$/.exec(line);
    if (fenceOpen && fenceOpen[1].toLowerCase() === 'json') {
      current.fence = { lines: [] };
      continue;
    }
    // `### Scraps` switches us into the scraps subsection — body is done.
    if (scrapsHeadingRe.test(line)) {
      current.section = 'scraps';
      continue;
    }
    if (current.section === 'scraps') {
      const bullet = scrapBulletRe.exec(line);
      if (bullet) {
        const sourceKind = bullet[1].trim() || 'unknown';
        const rawText = bullet[2].trim();
        current.scraps.push({ sourceKind, rawText });
      }
      // Non-bullet lines inside the scraps subsection (blanks) are ignored.
      continue;
    }
    // `tags: a, b, c` (case-insensitive, anywhere in the body section).
    const tagMatch = /^\s*tags\s*:\s*(.*)$/i.exec(line);
    if (tagMatch) {
      current.tags = normalizeTags(tagMatch[1]);
      continue;
    }
    current.bodyLines.push(line);
  }
  flush();
  if (out.length === 0) {
    throw new Error('markdown produced zero `## <Type>: <Name>` sections');
  }
  const entries = out.map((entry, i) => {
    const normalized = normalizeEntry(entry, i);
    // Keep parser output compatible with callers consuming only draft fields.
    // The bulk-import route explicitly validates and persists this metadata.
    if (Array.isArray(entry.scraps) && entry.scraps.length > 0) {
      attachNonEnumerable(normalized, 'scraps', entry.scraps);
    }
    return normalized;
  });
  // Warnings ride as a non-enumerable property on the returned array so
  // existing callers that array-index `result[i]` (or deep-equal it in
  // tests) keep working; the bulk-import handler reads `result.warnings`
  // to surface typos back to the user.
  attachNonEnumerable(entries, 'warnings', warnings);
  return entries;
}

/**
 * Top-level dispatch — `format` is the trusted enum (already Zod-validated
 * at the route boundary).
 */
export function parseBulkPayload(format, payload) {
  if (format === 'json') return parseJsonBulk(payload);
  if (format === 'csv') return parseCsvBulk(payload);
  if (format === 'markdown') return parseMarkdownBulk(payload);
  throw new Error(`unknown bulk-import format: ${format}`);
}

// === Export serializers ===================================================

/**
 * Render a single ingredient as a `## <Type>: <Name>` block with body +
 * tags + scraps. Inverse of parseMarkdownBulk (lossy on payload keys that
 * aren't the primary content key — they're still dumped as a JSON code
 * fence so round-tripping power users don't lose data).
 */
export function ingredientToMarkdown(ing) {
  const lines = [];
  lines.push(`## ${capitalize(ing.type)}: ${ing.name}`);
  lines.push('');
  const primaryKey = PRIMARY_CONTENT_KEY_BY_TYPE[ing.type];
  const payload = ing.payload || {};
  const primaryBody = typeof payload[primaryKey] === 'string' ? payload[primaryKey].trim() : '';
  if (primaryBody) {
    lines.push(primaryBody);
    lines.push('');
  }
  if (Array.isArray(ing.tags) && ing.tags.length > 0) {
    lines.push(`tags: ${ing.tags.join(', ')}`);
    lines.push('');
  }
  // Dump remaining payload keys as a JSON fence so power users keep the
  // full canon shape on round-trip. Empty {} is suppressed.
  const remainder = { ...payload };
  delete remainder[primaryKey];
  if (Object.keys(remainder).length > 0) {
    lines.push('```json');
    lines.push(JSON.stringify(remainder, null, 2));
    lines.push('```');
    lines.push('');
  }
  if (Array.isArray(ing.scraps) && ing.scraps.length > 0) {
    lines.push('### Scraps');
    lines.push('');
    for (const s of ing.scraps) {
      lines.push(`- (${s.sourceKind || 'unknown'}) ${truncate(s.rawText || '', 500)}`);
    }
    lines.push('');
  }
  return lines.join('\n');
}

// Attach metadata (warnings, bundleRef, roleForExportedRef) that rides
// alongside the parsed shape without affecting array-indexing callers or
// deep-equal tests on the entry/array shape.
function attachNonEnumerable(target, key, value) {
  Object.defineProperty(target, key, { value, enumerable: false });
}

function capitalize(s) {
  if (!s) return '';
  return s.charAt(0).toUpperCase() + s.slice(1);
}

function truncate(s, n) {
  return s.length > n ? `${s.slice(0, n).trimEnd()}…` : s;
}

/**
 * Bundle the export envelope as markdown — top heading + per-ingredient
 * blocks via ingredientToMarkdown.
 */
export function bundleToMarkdown(bundle) {
  const out = [];
  out.push(`# Catalog Export — ${bundle.ref.kind}/${bundle.ref.id}`);
  out.push('');
  out.push(`Exported at ${bundle.exportedAt} (${bundle.ingredients.length} ingredient${bundle.ingredients.length === 1 ? '' : 's'})`);
  out.push('');
  for (const ing of bundle.ingredients) {
    out.push(ingredientToMarkdown(ing));
  }
  return out.join('\n');
}

/**
 * Minimal YAML serializer for the export bundle's JSON shape. Supports the
 * value types we actually emit (strings, numbers, booleans, null, arrays,
 * objects). Strings always quote with double-quotes to sidestep YAML's
 * special-token landmines (`yes`, `no`, `null`, leading-`-`, etc.).
 *
 * Power users who need a richer YAML round-trip can convert from the JSON
 * format instead; we don't pull js-yaml just for this.
 */
export function toYamlString(value) {
  // Top-level shape: emit as a mapping if it's a plain object, else fall
  // back to a list/scalar at depth 0.
  if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
    return emitMapping(value, 0).join('\n') + '\n';
  }
  if (Array.isArray(value)) {
    return emitSequence(value, 0).join('\n') + '\n';
  }
  return emitScalar(value) + '\n';
}

function isScalar(v) {
  return v === null || v === undefined || typeof v !== 'object';
}

function emitScalar(value) {
  if (value === null || value === undefined) return 'null';
  if (typeof value === 'boolean') return value ? 'true' : 'false';
  if (typeof value === 'number') return Number.isFinite(value) ? String(value) : 'null';
  if (typeof value === 'string') return yamlQuote(value);
  return yamlQuote(String(value));
}

function emitMapping(obj, depth) {
  const pad = '  '.repeat(depth);
  const keys = Object.keys(obj);
  if (keys.length === 0) return [`${pad}{}`];
  const out = [];
  for (const k of keys) {
    const v = obj[k];
    const keyStr = yamlKey(k);
    if (isScalar(v)) {
      out.push(`${pad}${keyStr}: ${emitScalar(v)}`);
      continue;
    }
    if (Array.isArray(v)) {
      if (v.length === 0) {
        out.push(`${pad}${keyStr}: []`);
      } else {
        out.push(`${pad}${keyStr}:`);
        out.push(...emitSequence(v, depth));
      }
      continue;
    }
    // Object (non-null, non-array)
    if (Object.keys(v).length === 0) {
      out.push(`${pad}${keyStr}: {}`);
    } else {
      out.push(`${pad}${keyStr}:`);
      out.push(...emitMapping(v, depth + 1));
    }
  }
  return out;
}

function emitSequence(arr, depth) {
  const pad = '  '.repeat(depth);
  const out = [];
  for (const v of arr) {
    if (isScalar(v)) {
      out.push(`${pad}- ${emitScalar(v)}`);
      continue;
    }
    if (Array.isArray(v)) {
      if (v.length === 0) {
        out.push(`${pad}- []`);
      } else {
        out.push(`${pad}-`);
        out.push(...emitSequence(v, depth + 1));
      }
      continue;
    }
    // Object: inline first key after the `- ` then indent the rest.
    const keys = Object.keys(v);
    if (keys.length === 0) {
      out.push(`${pad}- {}`);
      continue;
    }
    const mapLines = emitMapping(v, depth + 1);
    // The first mapping line already carries depth+1 indentation; rewrite
    // it to use `- ` at depth instead so the list item is properly aligned.
    const firstPad = '  '.repeat(depth + 1);
    const first = mapLines[0].startsWith(firstPad) ? mapLines[0].slice(firstPad.length) : mapLines[0];
    out.push(`${pad}- ${first}`);
    for (let i = 1; i < mapLines.length; i++) out.push(mapLines[i]);
  }
  return out;
}

function yamlKey(k) {
  // Quote keys that aren't safe bare identifiers; covers spaces, colons,
  // special tokens. Cheap and correct for our shape.
  if (/^[A-Za-z_][A-Za-z0-9_-]*$/.test(k) && !['null', 'true', 'false', 'yes', 'no'].includes(k.toLowerCase())) {
    return k;
  }
  return yamlQuote(k);
}

function yamlQuote(s) {
  // JSON.stringify already gives us a YAML-compatible double-quoted form
  // for the subset of strings we'll see (escapes \, ", \n, control chars).
  return JSON.stringify(s);
}