Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Brain Graph Data Service
*
* Computes a graph of brain entities with edges derived from:
* - Semantic similarity (via CoS memory embeddings through the bridge)
* - Shared tags (Jaccard similarity)
* - Explicit CoS memory links (through the bridge)
*
* Bounded by design: the full graph at scale (~1500 nodes, >1M shared-tag
* pairs) crashes the browser, so this service never returns the whole graph.
* Callers either fetch the lightweight search index (`getBrainGraphSearchIndex`),
* a bounded overview of the most-connected nodes (`getBrainGraphOverview`), or
* one node's neighborhood (`getBrainGraphNeighborhood`). Shared-tag edges are
* only ever computed *among a bounded node set* and capped per node, so the
* O(n²) tag explosion can't happen.
*
* The two edge-bearing views need every record's tags and summary, so they go
* through `loadNodes()` (the cached field projections for each entity store).
* The search index needs only `{ id, label, brainType }`, so it reads those
* same projections in `brainSearchIndex` instead — focusing the graph's search box
* no longer walks and JSON-parses every record body and Daily Log entry on disk
* (issue #3507).
*/
import * as memoryBackend from './memoryBackend.js';
import { loadBridgeMap, bridgeKey } from './brainMemoryBridge.js';
import { getBrainProjections } from './brainSearchIndex.js';
import { getGoals } from './identity.js';
const ENTITY_TYPES = ['people', 'projects', 'ideas', 'admin', 'memories', 'songs', 'threads'];
// Edges shown per node are capped so a densely-tagged hub can't reintroduce the
// combinatorial blow-up inside a bounded view.
const SHARED_TAG_CAP_PER_NODE = 8;
const SHARED_TAG_MIN_JACCARD = 0.3;
const OVERVIEW_DEFAULT_LIMIT = 100;
const NEIGHBORHOOD_DEFAULT_LIMIT = 80;
const MAX_LIMIT = 250;
function jaccardSimilarity(tagsA, tagsB) {
if (!tagsA?.length || !tagsB?.length) return 0;
const setA = new Set(tagsA);
const setB = new Set(tagsB);
let intersection = 0;
for (const t of setA) {
if (setB.has(t)) intersection++;
}
const union = setA.size + setB.size - intersection;
return union === 0 ? 0 : intersection / union;
}
const clampLimit = (limit, fallback) => {
const n = Number(limit);
if (!Number.isFinite(n) || n <= 0) return fallback;
return Math.min(Math.floor(n), MAX_LIMIT);
};
// Node identity — shared by the full-record path (`loadNodes`) and the
// projection-backed search index, so the two can never drift on what a node is
// called or which records are skipped.
const entityLabel = (record) => record.name || record.title || '(untitled)';
// One-line gist for the node tooltip / detail panel. Every candidate is
// type-checked rather than `||`-chained straight through, because `content` is
// a STRING on memories/journals but an OBJECT (`{ format, text }`) on a
// SongBook record — an untyped chain would put `[object Object]` in the
// tooltip. `artist` sits ahead of `notes` so a song reads as "who plays it"
// rather than whatever practice note happens to be attached.
//
// Kept identical to `BODY_FIELDS` in client/src/components/brain/tabs/BrainGraph.jsx:
// the detail panel falls back to this server-computed summary when the full
// record hasn't loaded, so a different field order would make the text jump on
// load. `description` is inert here (no entity store carries it — goals set
// their summary directly below) but stays in the list so the two can't drift.
const SUMMARY_FIELDS = ['description', 'context', 'oneLiner', 'artist', 'notes', 'content'];
const entitySummary = (record) => {
for (const field of SUMMARY_FIELDS) {
if (typeof record[field] === 'string' && record[field]) return record[field];
}
return '';
};
const goalLabel = (goal) => goal.title || '(untitled goal)';
const isInactiveGoal = (goal) => goal.status === 'completed' || goal.status === 'abandoned';
const journalDate = (entry) => entry.id || entry.date;
// Load all non-archived brain entities as graph nodes (no edges).
async function loadNodes() {
const nodes = [];
const perType = await Promise.all(ENTITY_TYPES.map((type) => getBrainProjections(type, { ranked: false })));
ENTITY_TYPES.forEach((type, i) => {
for (const record of perType[i]) {
if (record.archived || record.status === 'archived') continue;
nodes.push({
id: record.id,
brainType: type,
label: entityLabel(record),
summary: entitySummary(record),
tags: record.tags || [],
importance: 0.6,
status: record.status
});
}
});
// Goals (identity system): active goals only (completed/abandoned excluded)
const goalsData = await getGoals().catch(() => null);
for (const goal of goalsData?.goals ?? []) {
if (isInactiveGoal(goal)) continue;
nodes.push({
id: goal.id,
brainType: 'goals',
label: goalLabel(goal),
summary: goal.description || '',
tags: goal.tags || [],
importance: goal.status === 'active' ? 0.75 : 0.5,
status: goal.status,
progress: goal.progress
});
}
// Journals (Daily Log): non-empty dated entries as graph nodes
const journals = await getBrainProjections('journals', { ranked: false }).catch(() => []);
for (const entry of journals) {
if (!entry.hasBody) continue;
const date = journalDate(entry);
if (!date) continue;
nodes.push({
id: date,
brainType: 'journals',
label: date,
summary: entry.summary,
tags: [],
importance: 0.4
});
}
return nodes;
}
/**
* Lightweight index for the client-side search box — every node, no edges.
*
* Reads the same cached projections as `loadNodes()`, retaining only the
* search fields. Node set, labels, and order are identical — the label/skip
* rules are shared above, and `ranked: false` keeps each store's natural order.
*
* Goals still come from `getGoals()`: identity keeps them in ONE file, so
* there is no per-record walk to avoid.
*/
export async function getBrainGraphSearchIndex() {
const nodes = [];
// Independent stores — read them together, then emit in ENTITY_TYPES order so
// the node order matches the `loadNodes()` derivation. A store that fails
// still rejects the request, exactly as the full-record path does.
const perType = await Promise.all(ENTITY_TYPES.map((type) => getBrainProjections(type, { ranked: false })));
perType.forEach((records, i) => {
for (const record of records) {
if (record.archived || record.status === 'archived') continue;
nodes.push({ id: record.id, label: entityLabel(record), brainType: ENTITY_TYPES[i] });
}
});
const goalsData = await getGoals().catch(() => null);
for (const goal of goalsData?.goals ?? []) {
if (isInactiveGoal(goal)) continue;
nodes.push({ id: goal.id, label: goalLabel(goal), brainType: 'goals' });
}
const journals = await getBrainProjections('journals', { ranked: false }).catch(() => []);
for (const entry of journals) {
if (!entry.hasBody) continue;
const date = journalDate(entry);
if (!date) continue;
nodes.push({ id: date, label: date, brainType: 'journals' });
}
return { nodes };
}
// Build the shared graph context once: nodes + tag index + remapped
// similar/linked edges (from CoS memory, capped per node already) + adjacency.
// This deliberately omits shared-tag edges — those are computed per bounded set.
async function loadGraphContext() {
const nodes = await loadNodes();
const nodeById = new Map(nodes.map(n => [n.id, n]));
const brainIdSet = new Set(nodeById.keys());
// tag → array of node ids
const tagIndex = new Map();
for (const n of nodes) {
for (const tag of n.tags || []) {
if (!tagIndex.has(tag)) tagIndex.set(tag, []);
tagIndex.get(tag).push(n.id);
}
}
// Reverse bridge: memoryId → brainId
const bridgeMap = await loadBridgeMap();
const reverseBridge = {};
for (const [bKey, memId] of Object.entries(bridgeMap)) {
reverseBridge[memId] = bKey.split(':')[1];
}
// Only memories bridged to a CURRENT node can contribute an edge, so scope the
// memory graph read to them (#10953): similarity work drops from scoring every
// embedded memory as a source to scoring just these. The backend keeps the
// global candidate population, so the edges are identical to the full graph's
// filtered to these endpoints. Fewer than two participating nodes cannot form
// an edge (self-edges are dropped below), so skip the read entirely.
const sourceIds = [];
const participating = new Set();
for (const [memId, brainId] of Object.entries(reverseBridge)) {
if (!brainIdSet.has(brainId)) continue;
sourceIds.push(memId);
participating.add(brainId);
}
// Remap CoS similar/linked edges to brain ids. ~3 similar/node + links — not
// the explosion (that's shared_tag, computed per bounded set below).
const remappedEdges = [];
const seen = new Set();
let hasEmbeddings = false;
const cosGraph = participating.size < 2
? null
: await memoryBackend.getGraphData({ sourceIds }).catch(() => null);
if (cosGraph) {
for (const edge of cosGraph.edges) {
const source = reverseBridge[edge.source];
const target = reverseBridge[edge.target];
if (!source || !target || source === target) continue;
if (!brainIdSet.has(source) || !brainIdSet.has(target)) continue;
const key = [source, target].sort().join('-');
if (seen.has(key)) continue;
seen.add(key);
const type = edge.type === 'linked' ? 'linked' : 'similar';
if (type === 'similar') hasEmbeddings = true;
remappedEdges.push({ source, target, type, weight: edge.weight });
}
}
// adjacency: id → array of { otherId, type, weight }
const adjacency = new Map();
const addAdj = (a, b, type, weight) => {
if (!adjacency.has(a)) adjacency.set(a, []);
adjacency.get(a).push({ otherId: b, type, weight });
};
for (const e of remappedEdges) {
addAdj(e.source, e.target, e.type, e.weight);
addAdj(e.target, e.source, e.type, e.weight);
}
return { nodes, nodeById, tagIndex, remappedEdges, adjacency, hasEmbeddings };
}
// Shared-tag edges among a bounded set of node ids, Jaccard-gated and capped per
// node (union of each endpoint's top-K) so a hub can't dominate the view.
function sharedTagEdgesAmong(ids, nodeById) {
const idSet = new Set(ids);
// Collect candidate weighted pairs.
const perNode = new Map(); // id → array of { other, weight }
for (let i = 0; i < ids.length; i++) {
const a = nodeById.get(ids[i]);
if (!a?.tags?.length) continue;
for (let j = i + 1; j < ids.length; j++) {
const b = nodeById.get(ids[j]);
if (!b?.tags?.length) continue;
const w = jaccardSimilarity(a.tags, b.tags);
if (w < SHARED_TAG_MIN_JACCARD) continue;
if (!perNode.has(a.id)) perNode.set(a.id, []);
if (!perNode.has(b.id)) perNode.set(b.id, []);
perNode.get(a.id).push({ other: b.id, weight: w });
perNode.get(b.id).push({ other: a.id, weight: w });
}
}
// Keep an edge if it survives the top-K cap on *either* endpoint.
const kept = new Set();
for (const [, list] of perNode) {
list.sort((x, y) => y.weight - x.weight);
}
const edges = [];
for (const [id, list] of perNode) {
for (const { other, weight } of list.slice(0, SHARED_TAG_CAP_PER_NODE)) {
if (!idSet.has(other)) continue;
const key = [id, other].sort().join('-');
if (kept.has(key)) continue;
kept.add(key);
const [source, target] = [id, other].sort();
edges.push({ source, target, type: 'shared_tag', weight });
}
}
return edges;
}
// Edges induced among a bounded node set: explicit/semantic first, then capped
// shared-tag for pairs not already connected.
function inducedEdges(idSet, ctx) {
const ids = [...idSet];
const edges = [];
const seen = new Set();
for (const e of ctx.remappedEdges) {
if (!idSet.has(e.source) || !idSet.has(e.target)) continue;
const key = [e.source, e.target].sort().join('-');
if (seen.has(key)) continue;
seen.add(key);
edges.push(e);
}
for (const e of sharedTagEdgesAmong(ids, ctx.nodeById)) {
const key = [e.source, e.target].sort().join('-');
if (seen.has(key)) continue; // an explicit/semantic edge already connects them
seen.add(key);
edges.push(e);
}
return edges;
}
// Cheap connectivity proxy for ranking the overview: degree in the
// similar/linked graph (weighted higher) plus a bounded co-tag participation
// score derived from tag-bucket sizes — neither enumerates all pairs.
function connectivityScores(ctx) {
const score = new Map();
for (const n of ctx.nodes) score.set(n.id, 0);
for (const [id, neighbors] of ctx.adjacency) {
score.set(id, (score.get(id) || 0) + neighbors.length * 2);
}
for (const [, ids] of ctx.tagIndex) {
const inc = Math.min(ids.length - 1, 5); // cap a giant bucket's contribution
if (inc <= 0) continue;
for (const id of ids) score.set(id, (score.get(id) || 0) + inc);
}
return score;
}
export async function getBrainGraphOverview({ limit } = {}) {
const cap = clampLimit(limit, OVERVIEW_DEFAULT_LIMIT);
const ctx = await loadGraphContext();
if (!ctx.nodes.length) {
return { nodes: [], edges: [], hasEmbeddings: false, mode: 'overview' };
}
const score = connectivityScores(ctx);
const topIds = [...ctx.nodes]
.sort((a, b) => (score.get(b.id) || 0) - (score.get(a.id) || 0))
.slice(0, cap)
.map(n => n.id);
const idSet = new Set(topIds);
const edges = inducedEdges(idSet, ctx);
const nodes = topIds.map(id => ctx.nodeById.get(id));
return { nodes, edges, hasEmbeddings: ctx.hasEmbeddings, mode: 'overview' };
}
export async function getBrainGraphNeighborhood({ focusId, limit } = {}) {
const cap = clampLimit(limit, NEIGHBORHOOD_DEFAULT_LIMIT);
const ctx = await loadGraphContext();
const focus = ctx.nodeById.get(focusId);
if (!focus) {
return { nodes: [], edges: [], hasEmbeddings: ctx.hasEmbeddings, mode: 'neighborhood', focusId, notFound: true };
}
// Gather neighbor candidates with a priority weight: linked > similar > tag.
const candidates = new Map(); // otherId → { type, weight, priority }
const consider = (otherId, type, weight) => {
if (otherId === focusId || !ctx.nodeById.has(otherId)) return;
const priority = type === 'linked' ? 2 : type === 'similar' ? 1 : 0;
const existing = candidates.get(otherId);
if (!existing || priority > existing.priority || (priority === existing.priority && weight > existing.weight)) {
candidates.set(otherId, { type, weight, priority });
}
};
for (const { otherId, type, weight } of ctx.adjacency.get(focusId) || []) {
consider(otherId, type, weight);
}
// Shared-tag neighbors of the focus (bounded by the focus's own tag buckets).
const tagCandidateIds = new Set();
for (const tag of focus.tags || []) {
for (const id of ctx.tagIndex.get(tag) || []) tagCandidateIds.add(id);
}
for (const id of tagCandidateIds) {
if (id === focusId) continue;
const w = jaccardSimilarity(focus.tags, ctx.nodeById.get(id)?.tags);
if (w >= SHARED_TAG_MIN_JACCARD) consider(id, 'shared_tag', w);
}
// Rank candidates and take the top `cap` neighbors. Final tiebreak on id keeps
// which neighbors survive the cap deterministic when priority+weight are equal
// (e.g. many identically-tagged neighbors at Jaccard 1.0).
const neighborIds = [...candidates.entries()]
.sort((a, b) => (b[1].priority - a[1].priority) || (b[1].weight - a[1].weight) || a[0].localeCompare(b[0]))
.slice(0, cap)
.map(([id]) => id);
const idSet = new Set([focusId, ...neighborIds]);
// Always include the focus↔neighbor edge for every selected neighbor (so none
// are orphaned by the shared-tag cap), then add neighbor↔neighbor context.
const edges = [];
const seen = new Set();
for (const id of neighborIds) {
const c = candidates.get(id);
const [source, target] = [focusId, id].sort();
const key = `${source}-${target}`;
seen.add(key);
edges.push({ source, target, type: c.type, weight: c.weight });
}
for (const e of inducedEdges(idSet, ctx)) {
const key = [e.source, e.target].sort().join('-');
if (seen.has(key)) continue;
seen.add(key);
edges.push(e);
}
const nodes = [...idSet].map(id => ctx.nodeById.get(id));
return { nodes, edges, hasEmbeddings: ctx.hasEmbeddings, mode: 'neighborhood', focusId };
}