Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Provider-agnostic embedding service.
*
* Reads `settings.embeddings = { provider, model }` and routes embedText() to
* the configured backend (Ollama or LM Studio). When `provider === 'none'` or
* unset, returns `{ skipped: true }` — callers persist the row without an
* embedding and a future re-embed admin action backfills.
*
* Vector dim is pinned to 768 (matches the `vector(768)` column on memories +
* catalog_ingredients). Output dim is validated; a mismatch surfaces clearly
* rather than corrupting the index by silently inserting the wrong-shape vec.
*/
import { getSettings } from './settings.js';
import * as ollama from './ollamaManager.js';
import * as lmstudio from './lmStudioManager.js';
import { mapWithConcurrency } from '../lib/mapWithConcurrency.js';
export const EMBEDDING_DIM = 768;
/**
* Read the configured provider + model.
* Defaults to `{ provider: 'none' }` so a missing settings slice cleanly
* degrades to no-embedding mode (rows persist; semantic search returns empty).
*/
export async function getEmbeddingsConfig() {
const settings = await getSettings();
const cfg = settings?.embeddings || {};
return {
provider: cfg.provider || 'none',
model: cfg.model || null,
};
}
/**
* Embed a single text. Returns:
* - `{ skipped: true, reason }` when provider is 'none'/unset
* - `{ success: true, embedding, model, provider, dimensions }` on success
* - `{ success: false, error, provider, model }` on failure
*/
export async function embedText(text, options = {}) {
if (!text || typeof text !== 'string' || text.trim().length === 0) {
return { skipped: true, reason: 'empty-text' };
}
const cfg = await getEmbeddingsConfig();
const provider = options.provider || cfg.provider;
const model = options.model || cfg.model || undefined;
if (provider === 'none' || !provider) {
return { skipped: true, reason: 'provider-disabled' };
}
let raw;
if (provider === 'ollama') {
raw = await ollama.getEmbeddings(text, { model, timeout: options.timeout });
} else if (provider === 'lmstudio') {
raw = await lmstudio.getEmbeddings(text, { model, timeout: options.timeout });
} else {
return { success: false, error: `Unknown embeddings provider: ${provider}`, provider, model };
}
if (!raw?.success) {
return { success: false, error: raw?.error || 'Embedding request failed', provider, model: raw?.model || model };
}
const embedding = raw.embedding || [];
if (embedding.length !== EMBEDDING_DIM) {
return {
success: false,
error: `Embedding model returned dim=${embedding.length}, expected ${EMBEDDING_DIM}. Pick a 768-dim model (e.g. nomic-embed-text).`,
provider,
model: raw.model || model,
};
}
return {
success: true,
embedding,
model: raw.model || model,
provider,
dimensions: embedding.length,
};
}
/**
* Build the embedding seed text for an ingredient — name + the few payload
* fields that carry narrative weight. Used by every catalog write path so
* the seed is consistent across commit, manual create, manual edit, and
* the admin backfill.
*
* Character canon shape uses `physicalDescription` + `personality` (per
* `sanitizeCharacter` in storyBible.js); without these in the seed, a
* character's main descriptive text is never embedded and semantic search
* misses every character whose content was captured through the inline New
* form or the writers-room/bible extractor — only the name would match.
*
* Returns `''` when there's nothing worth embedding so callers can skip.
*/
export function ingredientEmbedSeed({ name, payload } = {}) {
const p = payload || {};
// Include every narrative-weight field across the six ingredient types:
// character → role/physicalDescription/personality/background/motivations;
// place → slugline/era/description; object → description/significance;
// light types (idea/scene/concept) → summary/description/notes. Missing any
// of these leaves an ingredient effectively name-only in vector search.
return [
name,
p.role,
p.description,
p.physicalDescription,
p.personality,
p.background,
p.motivations,
p.summary,
p.notes,
p.slugline,
p.era,
p.significance,
]
.filter(Boolean)
.join(' ')
.slice(0, 4000);
}
/**
* Convenience wrapper: build the seed, embed it, and return the
* `{ embedding, embeddingModel }` slice ready to spread into createIngredient
* / updateIngredient. Returns `{}` when there's nothing to embed, the
* provider is unavailable, or the embed failed — the spread becomes a no-op.
*
* Logs non-skipped failures (provider down, dim mismatch, unknown error) so
* a quiet "search is broken" symptom has a paper trail. The batch path in
* embedBatch already logs; this single-text path was silent before.
*/
export async function embedIngredient(ingredient) {
const seed = ingredientEmbedSeed(ingredient);
if (!seed) return {};
const out = await embedText(seed).catch((err) => {
console.error(`🧬 embedIngredient threw: ${err?.message || err}`);
return null;
});
if (!out) return {};
if (out.skipped) return {};
if (!out.success) {
console.error(`🧬 embedIngredient failed: ${out.error || 'unknown'} (provider=${out.provider}, model=${out.model || 'unset'})`);
return {};
}
return { embedding: out.embedding, embeddingModel: out.model };
}
/**
* Embed an array of texts in parallel with a concurrency cap.
*
* Used by the catalog backfill and the `/api/catalog/embeddings/backfill`
* admin action. Per-text failures don't abort the batch — the corresponding
* slot in the result is `null`, the caller decides whether to skip or retry.
*/
export async function embedBatch(texts, options = {}) {
const concurrency = Math.max(1, options.concurrency || 4);
return mapWithConcurrency(texts, concurrency, async (text, i) => {
const out = await embedText(text, options);
if (out.success) {
return { embedding: out.embedding, model: out.model, provider: out.provider };
}
if (!out.skipped) {
console.error(`🧬 embedBatch[${i}] failed: ${out.error}`);
}
return null;
});
}