Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Memory Store (P2 — Memory Studio engine)
// =========================================
// The real, tiered memory engine shared by api/agent-memory.js and api/memory/**.
//
// Tiers (Letta/MemGPT model):
// working — the small, always-in-context core (pinned + the agent's identity
// and active rules). Token-budgeted.
// recall — recent interactions/trades, searchable.
// archival — long-term store; semantic search over real embeddings.
//
// Embeddings are produced by api/_lib/embeddings.js (free NIM lane first, OpenAI
// backstop), stored as JSONB vectors tagged with the embedder that made them,
// and scored strictly within their own vector space (scoreRowsBySpace). No fake
// similarity: when no provider is configured, search degrades to substring +
// salience, never a fabricated cosine score.
//
// Entity graph (Zep/Graphiti-style temporal KG): every memory is mined for the
// mints, tickers, wallets, people and strategies it mentions; entities are
// upserted as graph nodes and linked to their source memory. Edges between
// entities are derived at read time from co-occurrence within a memory.
import { sql, sqlValues } from './db.js';
import {
embeddingsConfigured,
defaultIngestEmbedderTag,
embedPassages,
scoreRowsBySpace,
} from './embeddings.js';
import { extractEntities } from './memory-entities.js';
import { outcomeWeight, groupOutcomes } from './memory-outcomes.js';
export const MEMORY_TIERS = ['working', 'recall', 'archival'];
// The four kinds a memory can be. Mirrors the CHECK constraint on
// agent_memories.type, so a caller filtering by type gets a 400 at the boundary
// instead of a silently empty result set from a typo.
export const MEMORY_TYPES = ['user', 'feedback', 'project', 'reference'];
// The working core is kept small on purpose — this is what's always paged into
// the model's context. ~4 chars/token is the platform's standard estimate
// (matches widget knowledge chunking).
export const WORKING_TOKEN_BUDGET = 2000;
export const CHARS_PER_TOKEN = 4;
// How many un-embedded / un-extracted rows a single read-triggered lazy pass
// processes. Bounds the work per request; repeated reads converge the backlog.
const EMBED_CAP = 24;
const ENTITY_CAP = 40;
export function estimateTokens(text) {
return Math.ceil(String(text || '').length / CHARS_PER_TOKEN);
}
// ── Decoration ──────────────────────────────────────────────────────────────
export function decorateMemory(row, extra = {}) {
const createdMs = row.created_at ? new Date(row.created_at).getTime() : Date.now();
return {
id: row.id,
agent_id: row.agent_id,
type: row.type,
content: row.content,
tags: row.tags || [],
context: row.context || {},
salience: row.salience,
tier: row.tier || 'recall',
pinned: !!row.pinned,
embedder: row.embedder || null,
hasEmbedding: row.has_embedding ?? row.embedding != null,
accessCount: row.access_count ?? 0,
isPublic: row.is_public ?? false,
tokens: estimateTokens(row.content),
createdAt: createdMs,
updatedAt: row.updated_at ? new Date(row.updated_at).getTime() : createdMs,
lastAccessedAt: row.last_accessed_at ? new Date(row.last_accessed_at).getTime() : null,
expiresAt: row.expires_at ? new Date(row.expires_at).getTime() : null,
// Portable & Verifiable Brain — authorship + storage provenance. Present
// on rows selected with the brain columns; absent (undefined) otherwise.
contentHash: row.content_hash ?? undefined,
signature: row.signature ?? undefined,
signerAddress: row.signer_address ?? undefined,
signedAt: row.signed_at ? new Date(row.signed_at).getTime() : (row.signed_at === null ? null : undefined),
storageMode: row.storage_mode ?? undefined,
ipfsCid: row.ipfs_cid ?? undefined,
...extra,
};
}
// ── Lazy embedding ────────────────────────────────────────────────────────────
/**
* Embed up to EMBED_CAP not-yet-embedded memories for an agent. Best-effort:
* returns counts and never throws (a provider outage just leaves rows for the
* next pass). Covers every write path uniformly — no writer needs to embed.
* @returns {Promise<{configured:boolean, embedded:number, remaining:number}>}
*/
export async function ensureEmbeddings(agentId, cap = EMBED_CAP) {
if (!embeddingsConfigured()) return { configured: false, embedded: 0, remaining: 0 };
const tag = defaultIngestEmbedderTag();
if (!tag) return { configured: false, embedded: 0, remaining: 0 };
let rows;
try {
rows = await sql`
SELECT id, content FROM agent_memories
WHERE agent_id = ${agentId} AND embedding IS NULL
AND (expires_at IS NULL OR expires_at > now())
ORDER BY created_at DESC
LIMIT ${cap + 1}
`;
} catch {
return { configured: true, embedded: 0, remaining: 0 };
}
if (!rows.length) return { configured: true, embedded: 0, remaining: 0 };
const capped = rows.length > cap;
const batch = capped ? rows.slice(0, cap) : rows;
let vectors;
try {
vectors = await embedPassages(tag, batch.map((r) => r.content || ''));
} catch (err) {
console.warn('[memory-store] embed batch failed', err?.code || err?.message);
return { configured: true, embedded: 0, remaining: rows.length };
}
let embedded = 0;
for (let i = 0; i < batch.length; i++) {
const vec = vectors[i];
if (!vec) continue;
try {
await sql`
UPDATE agent_memories
SET embedding = ${JSON.stringify(Array.from(vec))}::jsonb, embedder = ${tag}
WHERE id = ${batch[i].id}
`;
embedded++;
} catch {
/* leave for next pass */
}
}
return { configured: true, embedded, remaining: capped ? 1 : 0 };
}
// ── Semantic search (mem0 search()) ────────────────────────────────────────────
/**
* Rank an agent's memories against a query. Embeds the query in each stored
* vector space (never crossing spaces), filters by minScore, and fills any
* shortfall with a salience-ranked substring match so recall is never empty
* just because a provider is down.
*
* @param {string} agentId
* @param {string} query
* @param {{ topK?:number, minScore?:number, tiers?:string[], type?:string, bump?:boolean }} [opts]
* @returns {Promise<{ results:Array<object>, provider:boolean, scored:number }>}
*/
export async function searchMemories(agentId, query, opts = {}) {
const { topK = 8, minScore = 0.25, tiers, type, bump = true } = opts;
const q = String(query || '').trim();
if (!q) return { results: [], provider: false, scored: 0 };
await ensureEmbeddings(agentId).catch(() => {});
const tierList = Array.isArray(tiers) && tiers.length ? tiers.filter((t) => MEMORY_TIERS.includes(t)) : null;
const rows = await sql`
SELECT id, agent_id, type, content, tags, context, salience, tier, pinned,
embedder, embedding, access_count, is_public, created_at, updated_at,
last_accessed_at, expires_at
FROM agent_memories
WHERE agent_id = ${agentId}
AND (expires_at IS NULL OR expires_at > now())
AND (${type ?? null}::text IS NULL OR type = ${type ?? null})
AND (${tierList}::text[] IS NULL OR tier = ANY(${tierList}))
ORDER BY created_at DESC
LIMIT 1500
`;
if (!rows.length) return { results: [], provider: embeddingsConfigured(), scored: 0 };
// Realized-outcome weights for the recalled-then-traded memories. Best-effort:
// before the memory_outcomes migration lands, or on a read failure, every weight
// is a neutral 1 and ranking is the plain semantic/lexical order.
let outcomesById = new Map();
try {
const orows = await sql`
SELECT memory_id, pnl_usd, win, closed_at FROM memory_outcomes
WHERE agent_id = ${agentId}
`;
outcomesById = groupOutcomes(orows);
} catch {
/* unweighted recall */
}
const weigh = (r) => {
const o = outcomeWeight(outcomesById.get(String(r.id)) || []);
return { weight: o.weight, samples: o.samples, wins: o.wins, losses: o.losses, netPnlUsd: o.net_pnl_usd, lastClosedAt: o.last_closed_at };
};
const embedded = rows.filter((r) => Array.isArray(r.embedding) && r.embedding.length);
let semantic = [];
let usedProvider = false;
if (embedded.length && embeddingsConfigured()) {
try {
const { scored } = await scoreRowsBySpace(embedded, q);
semantic = scored
.filter((r) => r.score >= minScore)
.map((r) => ({ r, outcome: weigh(r) }))
.sort((a, b) => b.r.score * b.outcome.weight - a.r.score * a.outcome.weight);
usedProvider = true;
} catch (err) {
console.warn('[memory-store] scoring failed', err?.code || err?.message);
}
}
const ranked = [];
const seen = new Set();
for (const { r, outcome } of semantic) {
if (seen.has(r.id)) continue;
seen.add(r.id);
ranked.push(decorateMemory(r, {
score: Number(r.score.toFixed(4)),
rankScore: Number((r.score * outcome.weight).toFixed(4)),
outcome,
match: 'semantic',
}));
if (ranked.length >= topK) break;
}
// Substring + salience fill so a query always surfaces obvious lexical hits
// (and so recall works at all before embeddings backfill / when unconfigured).
if (ranked.length < topK) {
const ql = q.toLowerCase();
const lexical = rows
.filter((r) => !seen.has(r.id) && String(r.content || '').toLowerCase().includes(ql))
.map((r) => ({ r, outcome: weigh(r) }))
.sort((a, b) => (b.r.salience || 0) * b.outcome.weight - (a.r.salience || 0) * a.outcome.weight);
for (const { r, outcome } of lexical) {
if (seen.has(r.id)) continue;
seen.add(r.id);
ranked.push(decorateMemory(r, {
score: null,
rankScore: Number(((r.salience || 0) * outcome.weight).toFixed(4)),
outcome,
match: 'lexical',
}));
if (ranked.length >= topK) break;
}
}
if (bump && ranked.length) {
const ids = ranked.map((r) => r.id);
// Reinforcement: accessed memories gain a little salience (capped) and
// record the access — the recency/decay loop the Letta tiering relies on.
sql`
UPDATE agent_memories
SET access_count = access_count + 1,
last_accessed_at = now(),
salience = LEAST(1.0, salience + 0.02)
WHERE id = ANY(${ids}::uuid[])
`.catch(() => {});
}
return { results: ranked, provider: usedProvider, scored: semantic.length };
}
// ── Entity extraction + graph ──────────────────────────────────────────────────
/**
* Mine up to ENTITY_CAP not-yet-processed memories for entities, upsert them as
* graph nodes, and link each to its source memory. Idempotent + best-effort.
* @returns {Promise<{processed:number, remaining:number}>}
*/
export async function ensureEntities(agentId, cap = ENTITY_CAP) {
let rows;
try {
rows = await sql`
SELECT id, content, tags, context FROM agent_memories
WHERE agent_id = ${agentId} AND entities_extracted = false
AND (expires_at IS NULL OR expires_at > now())
ORDER BY created_at DESC
LIMIT ${cap + 1}
`;
} catch {
return { processed: 0, remaining: 0 };
}
if (!rows.length) return { processed: 0, remaining: 0 };
const capped = rows.length > cap;
const batch = capped ? rows.slice(0, cap) : rows;
// Mine every memory first, then write in three bulk statements instead of the
// 2× INSERT + UPDATE per entity-per-row the loop used to issue.
// 1. Upsert entities, aggregated by (kind, normalized): mention_count and the
// capped salience bump are summed over occurrences in this batch (net-equal
// to applying the per-occurrence upsert sequentially); label/meta take the
// last occurrence, matching the original EXCLUDED.* "last writer wins".
// 2. Insert the (entity, memory) links, resolving entity ids from the upsert's
// RETURNING, with ON CONFLICT DO NOTHING.
// 3. Mark the processed memories entities_extracted.
const byKey = new Map(); // "kind\u0000normalized" → { kind, label, normalized, meta, count }
const mentions = []; // { key, memoryId } per occurrence — drives the links
for (const row of batch) {
const entities = extractEntities(row.content, row.tags || [], row.context || {});
for (const e of entities) {
const key = `${e.kind}\u0000${e.normalized}`;
const agg = byKey.get(key);
if (agg) {
agg.count += 1;
agg.label = e.label;
agg.meta = e.meta || {};
} else {
byKey.set(key, { kind: e.kind, label: e.label, normalized: e.normalized, meta: e.meta || {}, count: 1 });
}
mentions.push({ key, memoryId: row.id });
}
}
if (!byKey.size) {
// No entities in any memory — still mark them processed so reads converge.
try {
const ids = batch.map((r) => r.id);
await sql`UPDATE agent_memories SET entities_extracted = true WHERE id = ANY(${ids}::uuid[])`;
return { processed: batch.length, remaining: capped ? 1 : 0 };
} catch (err) {
console.warn('[memory-store] entity extraction failed', err?.message);
return { processed: 0, remaining: capped ? 1 : 0 };
}
}
try {
const entityRows = [...byKey.values()].map((e) => [
agentId, e.kind, e.label, e.normalized, e.count, JSON.stringify(e.meta || {}),
]);
// `v.agent_id::uuid` is load-bearing, exactly like the ::uuid casts on the
// link insert below. A parameter inside a standalone `VALUES` list has no
// column to infer its type from, so Postgres resolves it to text, and
// text is not assignable to a uuid column: this statement raised
// `column "agent_id" is of type uuid but expression is of type text` on
// every call. The catch below swallowed it as a warning, so nothing 500'd
// and nothing looked broken from the outside; instead `entities_extracted`
// was never set, no entity row was ever written, and every agent's
// knowledge graph (graph(), entities(), memoriesFor()) came back
// permanently empty. The sibling insert never had the bug because it
// casts. Keep both casts.
const upserted = await sql`
INSERT INTO agent_memory_entities (agent_id, kind, label, normalized, mention_count, salience, meta)
SELECT v.agent_id::uuid, v.kind, v.label, v.normalized, v.cnt::int,
LEAST(1.0, 0.5 + 0.05 * (v.cnt::int - 1)), v.meta::jsonb
FROM ( VALUES ${sqlValues(entityRows)} ) AS v(agent_id, kind, label, normalized, cnt, meta)
ON CONFLICT (agent_id, kind, normalized) DO UPDATE
SET mention_count = agent_memory_entities.mention_count + EXCLUDED.mention_count,
last_seen_at = now(),
label = EXCLUDED.label,
salience = LEAST(1.0, agent_memory_entities.salience + 0.05 * EXCLUDED.mention_count)
RETURNING id, kind, normalized
`;
const idByKey = new Map();
for (const r of upserted) idByKey.set(`${r.kind}\u0000${r.normalized}`, r.id);
// Dedup link pairs (the PK is (entity_id, memory_id)) before the bulk insert.
const linkSet = new Set();
const linkRows = [];
for (const m of mentions) {
const entityId = idByKey.get(m.key);
if (!entityId) continue;
const pair = `${entityId}\u0000${m.memoryId}`;
if (linkSet.has(pair)) continue;
linkSet.add(pair);
linkRows.push([entityId, m.memoryId]);
}
if (linkRows.length) {
await sql`
INSERT INTO agent_memory_entity_links (entity_id, memory_id)
SELECT v.entity_id::uuid, v.memory_id::uuid
FROM ( VALUES ${sqlValues(linkRows)} ) AS v(entity_id, memory_id)
ON CONFLICT DO NOTHING
`;
}
const ids = batch.map((r) => r.id);
await sql`UPDATE agent_memories SET entities_extracted = true WHERE id = ANY(${ids}::uuid[])`;
return { processed: batch.length, remaining: capped ? 1 : 0 };
} catch (err) {
console.warn('[memory-store] entity extraction failed', err?.message);
return { processed: 0, remaining: capped ? 1 : 0 };
}
}
/**
* Build the agent's knowledge graph: entity nodes + co-occurrence edges.
* @returns {Promise<{nodes:Array<object>, edges:Array<object>, stats:object}>}
*/
export async function buildGraph(agentId, { limit = 200 } = {}) {
await ensureEntities(agentId).catch(() => {});
const entities = await sql`
SELECT id, kind, label, normalized, salience, mention_count, first_seen_at, last_seen_at, meta
FROM agent_memory_entities
WHERE agent_id = ${agentId}
ORDER BY mention_count DESC, last_seen_at DESC
LIMIT ${limit}
`;
if (!entities.length) return { nodes: [], edges: [], stats: { entities: 0, edges: 0 } };
const ids = entities.map((e) => e.id);
const links = await sql`
SELECT entity_id, memory_id FROM agent_memory_entity_links
WHERE entity_id = ANY(${ids}::uuid[])
`;
// Group entities by the memory they co-occur in; every pair in a memory is an edge.
const byMemory = new Map();
for (const l of links) {
if (!byMemory.has(l.memory_id)) byMemory.set(l.memory_id, []);
byMemory.get(l.memory_id).push(l.entity_id);
}
const edgeWeights = new Map();
for (const entIds of byMemory.values()) {
const uniq = [...new Set(entIds)];
for (let i = 0; i < uniq.length; i++) {
for (let j = i + 1; j < uniq.length; j++) {
const [a, b] = uniq[i] < uniq[j] ? [uniq[i], uniq[j]] : [uniq[j], uniq[i]];
const key = `${a}|${b}`;
edgeWeights.set(key, (edgeWeights.get(key) || 0) + 1);
}
}
}
const nodes = entities.map((e) => ({
id: e.id,
kind: e.kind,
label: e.label,
salience: e.salience,
mentions: e.mention_count,
firstSeenAt: e.first_seen_at ? new Date(e.first_seen_at).getTime() : null,
lastSeenAt: e.last_seen_at ? new Date(e.last_seen_at).getTime() : null,
meta: e.meta || {},
}));
const edges = [...edgeWeights.entries()].map(([key, weight]) => {
const [source, target] = key.split('|');
return { source, target, weight };
});
return { nodes, edges, stats: { entities: nodes.length, edges: edges.length } };
}
/**
* Memories that mention a given entity (by id) — powers "what does my agent
* remember about <mint>?" and graph-node drilldown.
*/
export async function memoriesForEntity(agentId, entityId, { limit = 50 } = {}) {
const rows = await sql`
SELECT m.id, m.agent_id, m.type, m.content, m.tags, m.context, m.salience, m.tier,
m.pinned, m.embedder, m.embedding, m.access_count, m.is_public, m.created_at,
m.updated_at, m.last_accessed_at, m.expires_at
FROM agent_memories m
JOIN agent_memory_entity_links l ON l.memory_id = m.id
JOIN agent_memory_entities e ON e.id = l.entity_id
WHERE e.id = ${entityId} AND e.agent_id = ${agentId} AND m.agent_id = ${agentId}
AND (m.expires_at IS NULL OR m.expires_at > now())
ORDER BY m.created_at DESC
LIMIT ${limit}
`;
return rows.map((r) => decorateMemory(r));
}
/**
* True for a turn written by the public /api/agent-ask concierge. Those rows hold
* anonymous visitor text, so the owner's chat must never recall them into its
* system prompt.
*/
export function isVisitorQaMemory(m) {
const tags = Array.isArray(m?.tags) ? m.tags : [];
return tags.includes('qa') && tags.some((t) => typeof t === 'string' && t.startsWith('qa:'));
}
// ── Working context + token budget ──────────────────────────────────────────────
/**
* The "in-context now" working set: pinned and working-tier memories, ordered by
* salience, with a live token-budget accounting so the UI can show exactly what
* the model carries and whether the core has overflowed its budget.
*/
export async function computeContext(agentId) {
const rows = await sql`
SELECT id, agent_id, type, content, tags, context, salience, tier, pinned,
embedder, embedding, access_count, is_public, created_at, updated_at,
last_accessed_at, expires_at
FROM agent_memories
WHERE agent_id = ${agentId}
AND (expires_at IS NULL OR expires_at > now())
AND (pinned = true OR tier = 'working')
ORDER BY pinned DESC, salience DESC, created_at DESC
`;
let tokens = 0;
const entries = rows.map((r) => {
const dec = decorateMemory(r);
tokens += dec.tokens;
return dec;
});
const [counts] = await sql`
SELECT
COUNT(*)::int AS total,
COUNT(*) FILTER (WHERE tier = 'working')::int AS working,
COUNT(*) FILTER (WHERE tier = 'recall')::int AS recall,
COUNT(*) FILTER (WHERE tier = 'archival')::int AS archival,
COUNT(*) FILTER (WHERE embedding IS NOT NULL)::int AS embedded
FROM agent_memories
WHERE agent_id = ${agentId} AND (expires_at IS NULL OR expires_at > now())
`;
return {
entries,
tokens,
budget: WORKING_TOKEN_BUDGET,
overBudget: tokens > WORKING_TOKEN_BUDGET,
counts: counts || { total: 0, working: 0, recall: 0, archival: 0, embedded: 0 },
};
}
// ── Tier assignment helper (used at write time) ──────────────────────────────────
/**
* Default tier for a new memory: pinned/working when the caller pins it,
* archival for low-salience reference material, recall otherwise.
*/
export function defaultTier({ pinned, salience = 0.5, type } = {}) {
if (pinned) return 'working';
if (type === 'reference' && salience < 0.5) return 'archival';
return 'recall';
}