Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

16.2 KB
Raw Blame
/**
 * Brain Graph Data Service
 *
 * Computes a graph of brain entities with edges derived from:
 * - Semantic similarity (via CoS memory embeddings through the bridge)
 * - Shared tags (Jaccard similarity)
 * - Explicit CoS memory links (through the bridge)
 *
 * Bounded by design: the full graph at scale (~1500 nodes, >1M shared-tag
 * pairs) crashes the browser, so this service never returns the whole graph.
 * Callers either fetch the lightweight search index (`getBrainGraphSearchIndex`),
 * a bounded overview of the most-connected nodes (`getBrainGraphOverview`), or
 * one node's neighborhood (`getBrainGraphNeighborhood`). Shared-tag edges are
 * only ever computed *among a bounded node set* and capped per node, so the
 * O(n²) tag explosion can't happen.
 *
 * The two edge-bearing views need every record's tags and summary, so they go
 * through `loadNodes()` (the cached field projections for each entity store).
 * The search index needs only `{ id, label, brainType }`, so it reads those
 * same projections in `brainSearchIndex` instead — focusing the graph's search box
 * no longer walks and JSON-parses every record body and Daily Log entry on disk
 * (issue #3507).
 */

import * as memoryBackend from './memoryBackend.js';
import { loadBridgeMap, bridgeKey } from './brainMemoryBridge.js';
import { getBrainProjections } from './brainSearchIndex.js';
import { getGoals } from './identity.js';

const ENTITY_TYPES = ['people', 'projects', 'ideas', 'admin', 'memories', 'songs', 'threads'];

// Edges shown per node are capped so a densely-tagged hub can't reintroduce the
// combinatorial blow-up inside a bounded view.
const SHARED_TAG_CAP_PER_NODE = 8;
const SHARED_TAG_MIN_JACCARD = 0.3;
const OVERVIEW_DEFAULT_LIMIT = 100;
const NEIGHBORHOOD_DEFAULT_LIMIT = 80;
const MAX_LIMIT = 250;

function jaccardSimilarity(tagsA, tagsB) {
  if (!tagsA?.length || !tagsB?.length) return 0;
  const setA = new Set(tagsA);
  const setB = new Set(tagsB);
  let intersection = 0;
  for (const t of setA) {
    if (setB.has(t)) intersection++;
  }
  const union = setA.size + setB.size - intersection;
  return union === 0 ? 0 : intersection / union;
}

const clampLimit = (limit, fallback) => {
  const n = Number(limit);
  if (!Number.isFinite(n) || n <= 0) return fallback;
  return Math.min(Math.floor(n), MAX_LIMIT);
};

// Node identity — shared by the full-record path (`loadNodes`) and the
// projection-backed search index, so the two can never drift on what a node is
// called or which records are skipped.
const entityLabel = (record) => record.name || record.title || '(untitled)';

// One-line gist for the node tooltip / detail panel. Every candidate is
// type-checked rather than `||`-chained straight through, because `content` is
// a STRING on memories/journals but an OBJECT (`{ format, text }`) on a
// SongBook record — an untyped chain would put `[object Object]` in the
// tooltip. `artist` sits ahead of `notes` so a song reads as "who plays it"
// rather than whatever practice note happens to be attached.
//
// Kept identical to `BODY_FIELDS` in client/src/components/brain/tabs/BrainGraph.jsx:
// the detail panel falls back to this server-computed summary when the full
// record hasn't loaded, so a different field order would make the text jump on
// load. `description` is inert here (no entity store carries it — goals set
// their summary directly below) but stays in the list so the two can't drift.
const SUMMARY_FIELDS = ['description', 'context', 'oneLiner', 'artist', 'notes', 'content'];
const entitySummary = (record) => {
  for (const field of SUMMARY_FIELDS) {
    if (typeof record[field] === 'string' && record[field]) return record[field];
  }
  return '';
};
const goalLabel = (goal) => goal.title || '(untitled goal)';
const isInactiveGoal = (goal) => goal.status === 'completed' || goal.status === 'abandoned';
const journalDate = (entry) => entry.id || entry.date;

// Load all non-archived brain entities as graph nodes (no edges).
async function loadNodes() {
  const nodes = [];
  const perType = await Promise.all(ENTITY_TYPES.map((type) => getBrainProjections(type, { ranked: false })));
  ENTITY_TYPES.forEach((type, i) => {
    for (const record of perType[i]) {
      if (record.archived || record.status === 'archived') continue;
      nodes.push({
        id: record.id,
        brainType: type,
        label: entityLabel(record),
        summary: entitySummary(record),
        tags: record.tags || [],
        importance: 0.6,
        status: record.status
      });
    }
  });

  // Goals (identity system): active goals only (completed/abandoned excluded)
  const goalsData = await getGoals().catch(() => null);
  for (const goal of goalsData?.goals ?? []) {
    if (isInactiveGoal(goal)) continue;
    nodes.push({
      id: goal.id,
      brainType: 'goals',
      label: goalLabel(goal),
      summary: goal.description || '',
      tags: goal.tags || [],
      importance: goal.status === 'active' ? 0.75 : 0.5,
      status: goal.status,
      progress: goal.progress
    });
  }

  // Journals (Daily Log): non-empty dated entries as graph nodes
  const journals = await getBrainProjections('journals', { ranked: false }).catch(() => []);
  for (const entry of journals) {
    if (!entry.hasBody) continue;
    const date = journalDate(entry);
    if (!date) continue;
    nodes.push({
      id: date,
      brainType: 'journals',
      label: date,
      summary: entry.summary,
      tags: [],
      importance: 0.4
    });
  }

  return nodes;
}

/**
 * Lightweight index for the client-side search box — every node, no edges.
 *
 * Reads the same cached projections as `loadNodes()`, retaining only the
 * search fields. Node set, labels, and order are identical — the label/skip
 * rules are shared above, and `ranked: false` keeps each store's natural order.
 *
 * Goals still come from `getGoals()`: identity keeps them in ONE file, so
 * there is no per-record walk to avoid.
 */
export async function getBrainGraphSearchIndex() {
  const nodes = [];

  // Independent stores — read them together, then emit in ENTITY_TYPES order so
  // the node order matches the `loadNodes()` derivation. A store that fails
  // still rejects the request, exactly as the full-record path does.
  const perType = await Promise.all(ENTITY_TYPES.map((type) => getBrainProjections(type, { ranked: false })));
  perType.forEach((records, i) => {
    for (const record of records) {
      if (record.archived || record.status === 'archived') continue;
      nodes.push({ id: record.id, label: entityLabel(record), brainType: ENTITY_TYPES[i] });
    }
  });

  const goalsData = await getGoals().catch(() => null);
  for (const goal of goalsData?.goals ?? []) {
    if (isInactiveGoal(goal)) continue;
    nodes.push({ id: goal.id, label: goalLabel(goal), brainType: 'goals' });
  }

  const journals = await getBrainProjections('journals', { ranked: false }).catch(() => []);
  for (const entry of journals) {
    if (!entry.hasBody) continue;
    const date = journalDate(entry);
    if (!date) continue;
    nodes.push({ id: date, label: date, brainType: 'journals' });
  }

  return { nodes };
}

// Build the shared graph context once: nodes + tag index + remapped
// similar/linked edges (from CoS memory, capped per node already) + adjacency.
// This deliberately omits shared-tag edges — those are computed per bounded set.
async function loadGraphContext() {
  const nodes = await loadNodes();
  const nodeById = new Map(nodes.map(n => [n.id, n]));
  const brainIdSet = new Set(nodeById.keys());

  // tag → array of node ids
  const tagIndex = new Map();
  for (const n of nodes) {
    for (const tag of n.tags || []) {
      if (!tagIndex.has(tag)) tagIndex.set(tag, []);
      tagIndex.get(tag).push(n.id);
    }
  }

  // Reverse bridge: memoryId → brainId
  const bridgeMap = await loadBridgeMap();
  const reverseBridge = {};
  for (const [bKey, memId] of Object.entries(bridgeMap)) {
    reverseBridge[memId] = bKey.split(':')[1];
  }

  // Only memories bridged to a CURRENT node can contribute an edge, so scope the
  // memory graph read to them (#10953): similarity work drops from scoring every
  // embedded memory as a source to scoring just these. The backend keeps the
  // global candidate population, so the edges are identical to the full graph's
  // filtered to these endpoints. Fewer than two participating nodes cannot form
  // an edge (self-edges are dropped below), so skip the read entirely.
  const sourceIds = [];
  const participating = new Set();
  for (const [memId, brainId] of Object.entries(reverseBridge)) {
    if (!brainIdSet.has(brainId)) continue;
    sourceIds.push(memId);
    participating.add(brainId);
  }

  // Remap CoS similar/linked edges to brain ids. ~3 similar/node + links — not
  // the explosion (that's shared_tag, computed per bounded set below).
  const remappedEdges = [];
  const seen = new Set();
  let hasEmbeddings = false;
  const cosGraph = participating.size < 2
    ? null
    : await memoryBackend.getGraphData({ sourceIds }).catch(() => null);
  if (cosGraph) {
    for (const edge of cosGraph.edges) {
      const source = reverseBridge[edge.source];
      const target = reverseBridge[edge.target];
      if (!source || !target || source === target) continue;
      if (!brainIdSet.has(source) || !brainIdSet.has(target)) continue;
      const key = [source, target].sort().join('-');
      if (seen.has(key)) continue;
      seen.add(key);
      const type = edge.type === 'linked' ? 'linked' : 'similar';
      if (type === 'similar') hasEmbeddings = true;
      remappedEdges.push({ source, target, type, weight: edge.weight });
    }
  }

  // adjacency: id → array of { otherId, type, weight }
  const adjacency = new Map();
  const addAdj = (a, b, type, weight) => {
    if (!adjacency.has(a)) adjacency.set(a, []);
    adjacency.get(a).push({ otherId: b, type, weight });
  };
  for (const e of remappedEdges) {
    addAdj(e.source, e.target, e.type, e.weight);
    addAdj(e.target, e.source, e.type, e.weight);
  }

  return { nodes, nodeById, tagIndex, remappedEdges, adjacency, hasEmbeddings };
}

// Shared-tag edges among a bounded set of node ids, Jaccard-gated and capped per
// node (union of each endpoint's top-K) so a hub can't dominate the view.
function sharedTagEdgesAmong(ids, nodeById) {
  const idSet = new Set(ids);
  // Collect candidate weighted pairs.
  const perNode = new Map(); // id → array of { other, weight }
  for (let i = 0; i < ids.length; i++) {
    const a = nodeById.get(ids[i]);
    if (!a?.tags?.length) continue;
    for (let j = i + 1; j < ids.length; j++) {
      const b = nodeById.get(ids[j]);
      if (!b?.tags?.length) continue;
      const w = jaccardSimilarity(a.tags, b.tags);
      if (w < SHARED_TAG_MIN_JACCARD) continue;
      if (!perNode.has(a.id)) perNode.set(a.id, []);
      if (!perNode.has(b.id)) perNode.set(b.id, []);
      perNode.get(a.id).push({ other: b.id, weight: w });
      perNode.get(b.id).push({ other: a.id, weight: w });
    }
  }
  // Keep an edge if it survives the top-K cap on *either* endpoint.
  const kept = new Set();
  for (const [, list] of perNode) {
    list.sort((x, y) => y.weight - x.weight);
  }
  const edges = [];
  for (const [id, list] of perNode) {
    for (const { other, weight } of list.slice(0, SHARED_TAG_CAP_PER_NODE)) {
      if (!idSet.has(other)) continue;
      const key = [id, other].sort().join('-');
      if (kept.has(key)) continue;
      kept.add(key);
      const [source, target] = [id, other].sort();
      edges.push({ source, target, type: 'shared_tag', weight });
    }
  }
  return edges;
}

// Edges induced among a bounded node set: explicit/semantic first, then capped
// shared-tag for pairs not already connected.
function inducedEdges(idSet, ctx) {
  const ids = [...idSet];
  const edges = [];
  const seen = new Set();
  for (const e of ctx.remappedEdges) {
    if (!idSet.has(e.source) || !idSet.has(e.target)) continue;
    const key = [e.source, e.target].sort().join('-');
    if (seen.has(key)) continue;
    seen.add(key);
    edges.push(e);
  }
  for (const e of sharedTagEdgesAmong(ids, ctx.nodeById)) {
    const key = [e.source, e.target].sort().join('-');
    if (seen.has(key)) continue; // an explicit/semantic edge already connects them
    seen.add(key);
    edges.push(e);
  }
  return edges;
}

// Cheap connectivity proxy for ranking the overview: degree in the
// similar/linked graph (weighted higher) plus a bounded co-tag participation
// score derived from tag-bucket sizes — neither enumerates all pairs.
function connectivityScores(ctx) {
  const score = new Map();
  for (const n of ctx.nodes) score.set(n.id, 0);
  for (const [id, neighbors] of ctx.adjacency) {
    score.set(id, (score.get(id) || 0) + neighbors.length * 2);
  }
  for (const [, ids] of ctx.tagIndex) {
    const inc = Math.min(ids.length - 1, 5); // cap a giant bucket's contribution
    if (inc <= 0) continue;
    for (const id of ids) score.set(id, (score.get(id) || 0) + inc);
  }
  return score;
}

export async function getBrainGraphOverview({ limit } = {}) {
  const cap = clampLimit(limit, OVERVIEW_DEFAULT_LIMIT);
  const ctx = await loadGraphContext();
  if (!ctx.nodes.length) {
    return { nodes: [], edges: [], hasEmbeddings: false, mode: 'overview' };
  }
  const score = connectivityScores(ctx);
  const topIds = [...ctx.nodes]
    .sort((a, b) => (score.get(b.id) || 0) - (score.get(a.id) || 0))
    .slice(0, cap)
    .map(n => n.id);
  const idSet = new Set(topIds);
  const edges = inducedEdges(idSet, ctx);
  const nodes = topIds.map(id => ctx.nodeById.get(id));
  return { nodes, edges, hasEmbeddings: ctx.hasEmbeddings, mode: 'overview' };
}

export async function getBrainGraphNeighborhood({ focusId, limit } = {}) {
  const cap = clampLimit(limit, NEIGHBORHOOD_DEFAULT_LIMIT);
  const ctx = await loadGraphContext();
  const focus = ctx.nodeById.get(focusId);
  if (!focus) {
    return { nodes: [], edges: [], hasEmbeddings: ctx.hasEmbeddings, mode: 'neighborhood', focusId, notFound: true };
  }

  // Gather neighbor candidates with a priority weight: linked > similar > tag.
  const candidates = new Map(); // otherId → { type, weight, priority }
  const consider = (otherId, type, weight) => {
    if (otherId === focusId || !ctx.nodeById.has(otherId)) return;
    const priority = type === 'linked' ? 2 : type === 'similar' ? 1 : 0;
    const existing = candidates.get(otherId);
    if (!existing || priority > existing.priority || (priority === existing.priority && weight > existing.weight)) {
      candidates.set(otherId, { type, weight, priority });
    }
  };
  for (const { otherId, type, weight } of ctx.adjacency.get(focusId) || []) {
    consider(otherId, type, weight);
  }
  // Shared-tag neighbors of the focus (bounded by the focus's own tag buckets).
  const tagCandidateIds = new Set();
  for (const tag of focus.tags || []) {
    for (const id of ctx.tagIndex.get(tag) || []) tagCandidateIds.add(id);
  }
  for (const id of tagCandidateIds) {
    if (id === focusId) continue;
    const w = jaccardSimilarity(focus.tags, ctx.nodeById.get(id)?.tags);
    if (w >= SHARED_TAG_MIN_JACCARD) consider(id, 'shared_tag', w);
  }

  // Rank candidates and take the top `cap` neighbors. Final tiebreak on id keeps
  // which neighbors survive the cap deterministic when priority+weight are equal
  // (e.g. many identically-tagged neighbors at Jaccard 1.0).
  const neighborIds = [...candidates.entries()]
    .sort((a, b) => (b[1].priority - a[1].priority) || (b[1].weight - a[1].weight) || a[0].localeCompare(b[0]))
    .slice(0, cap)
    .map(([id]) => id);

  const idSet = new Set([focusId, ...neighborIds]);

  // Always include the focus↔neighbor edge for every selected neighbor (so none
  // are orphaned by the shared-tag cap), then add neighbor↔neighbor context.
  const edges = [];
  const seen = new Set();
  for (const id of neighborIds) {
    const c = candidates.get(id);
    const [source, target] = [focusId, id].sort();
    const key = `${source}-${target}`;
    seen.add(key);
    edges.push({ source, target, type: c.type, weight: c.weight });
  }
  for (const e of inducedEdges(idSet, ctx)) {
    const key = [e.source, e.target].sort().join('-');
    if (seen.has(key)) continue;
    seen.add(key);
    edges.push(e);
  }

  const nodes = [...idSet].map(id => ctx.nodeById.get(id));
  return { nodes, edges, hasEmbeddings: ctx.hasEmbeddings, mode: 'neighborhood', focusId };
}