Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Universe Character — vision-driven structured expansion.
*
* The text-based sibling `universeCharacterExpand.js` fleshes out a character's
* blank fields from its EXISTING data. This one does the same job from one or
* more REFERENCE IMAGES: a vision-capable model looks at the image(s) and
* returns a JSON object of the character's structured attributes (palette,
* physical/visual notes, expressions, …), which we merge no-clobber onto the
* canon entry.
*
* Review-then-apply, WITHOUT a second LLM round-trip: this service only
* PROPOSES. It runs the vision model once, computes the no-clobber merge
* against the character's still-blank fields, and returns the proposed field
* values for review. The client shows them (editable) and, on Apply, writes the
* kept values through the normal entry-PATCH path — so user edits are honored,
* the write is serialized by the existing universe write queue, and we never
* pay for a second non-deterministic LLM call.
*
* Vision is an API-provider-only capability (only the toolkit's executeApiRun
* base64-inlines images). We resolve an API provider up front and reject a
* fallback to a non-vision provider so the model is guaranteed to actually see
* the references rather than hallucinating attributes from the text prompt.
*
* Merge semantics mirror the text expand exactly (`applyExpansion`):
* - key absent → preserve existing
* - key present but empty → no-op
* - key present, non-empty → fill ONLY when the target field is blank
*/
import { getUniverse } from './universeBuilder.js';
import { applyExpansion, STRING_FIELDS, LIST_FIELDS } from './universeCharacterExpand.js';
import { bibleFieldIsBlank } from '../lib/universeBibleCompleteness.js';
import { resolveAPIProvider, parseLLMJSON } from './aiProvider.js';
import { runPromptThroughProvider, assertProvider, assertVisionRunUsedImages } from './promptRunner.js';
import { ServerError } from '../lib/errorHandler.js';
import { shortId } from '../lib/fileUtils.js';
// Cap mirrors VISION_MAX_IMAGES in universeVisionDescribe.js — the runner
// base64-inlines every image into a single request body, so a large batch
// balloons the prompt and the provider's context window.
export const VISION_EXPAND_MAX_IMAGES = 8;
// Per-list-field row-shape guidance so the model returns rows that survive
// `sanitizeCharacter` (which drops rows missing the required key). Keep in sync
// with the sanitizer's row shapes in storyBible.js.
const LIST_FIELD_SHAPES = {
stats: '{ "label": "<attribute name>", "value": "<value>" }',
colorPalette: '{ "name": "<swatch name>", "hex": "#rrggbb", "role": "<where it appears>" }',
props: '{ "name": "<prop name>", "purpose": "<why they carry it>", "materials": "<materials>", "notes": "<notes>" }',
expressions: '{ "name": "<expression>", "description": "<how it reads on their face>" }',
handGestures: '{ "name": "<gesture>", "description": "<the hand pose>" }',
};
// Non-visual character-framework fields (CWQE Phase 10, #2175). They live in the
// shared STRING_FIELDS / LIST_FIELDS lists (the text/vision expand flows share
// one source of truth) but a vision model CANNOT infer a character's Ghost, Lie,
// Want, Need, or secrets from an image — those are narrative, not renderable. So
// the vision prompt excludes them (they'd only invite hallucinated backstory,
// and `secrets` has no LIST_FIELD_SHAPES row shape to emit). arcType / sliders
// are already absent from both shared lists (merged specially in the text flow),
// so they never reach the vision prompt either.
const NON_VISUAL_FRAMEWORK_FIELDS = new Set(['ghost', 'wound', 'lie', 'want', 'need', 'secrets']);
// The structured psychology profile (#6414) is the same kind of material one
// level deeper — a theory of control and its survival/connection/status drives
// are an interior, not a rendered surface. It never reaches the prompt (it is
// an `objects` field, so it is in neither shared list), but the shared
// `applyExpansion` WOULD merge it, so a model that volunteers the key anyway
// must not be able to author a character's psychology from a picture. Stripped
// from the payload before the merge rather than trusted to the prompt.
const NON_VISUAL_PROPOSAL_KEYS = Object.freeze(['psychology']);
/**
* Build the vision prompt. The model is told to return ONE JSON object keyed by
* the structured character fields, populating only those it can infer from the
* image(s) and focusing on the still-blank ones. No prose, no markdown.
*/
export function buildVisionExpandPrompt({ name, context, imageCount = 1, blankFields = [] }) {
const subject = name ? `the character "${name}"` : 'this character';
const intro = imageCount > 1
? `You are looking at ${imageCount} reference images of the same character${name ? ` (${subject})` : ''}. They show the same subject under different conditions (angle, lighting, framing, background). Read the visual traits that stay CONSISTENT across all of them and describe that single shared subject — ignore differences incidental to any one shot.`
: `You are looking at a reference image of ${subject}.`;
const ctx = context && context.trim()
? `\n\nKnown context (use it to disambiguate, do not contradict the images): ${context.trim()}`
: '';
// Narrow the ask to the fields that are still blank when we have that list;
// otherwise offer the full set. Populated fields are no-clobber server-side
// regardless, so this is purely to focus the model.
// Non-visual framework fields (ghost/lie/…/secrets) are never asked of a
// vision model — filter them out of both the full-set and blank-narrowed asks.
const visualStringFields = STRING_FIELDS.filter((f) => !NON_VISUAL_FRAMEWORK_FIELDS.has(f));
const visualListFields = LIST_FIELDS.filter((f) => !NON_VISUAL_FRAMEWORK_FIELDS.has(f));
const stringTargets = blankFields.length
? visualStringFields.filter((f) => blankFields.includes(f))
: visualStringFields;
const listTargets = blankFields.length
? visualListFields.filter((f) => blankFields.includes(f))
: visualListFields;
const stringLines = stringTargets.map((f) => ` "${f}": "<string>"`).join(',\n');
const listLines = listTargets
.map((f) => ` "${f}": [ ${LIST_FIELD_SHAPES[f]} ]`)
.join(',\n');
const schemaBody = [stringLines, listLines].filter(Boolean).join(',\n');
return `${intro}
You are a graphic-novelist's character-reference analyst. Extract the character's renderable visual + design attributes from the image(s) so they can seed a story bible.
Return ONLY a single valid JSON object in this shape (replace every <…> with real content; do NOT output the literal angle-bracket text). OMIT any key you cannot confidently infer from the image(s) — an omitted key means "no opinion" and is preferred over a guess:
{
${schemaBody}
}
Rules:
- Base every value on what is VISIBLE in the image(s). Do not invent backstory, plot, or facts the image cannot show.
- String fields: concise, concrete, renderable phrasing (comma-separated descriptive phrases work well). No markdown, no headings, no commentary about the images themselves.
- List fields: return an array of objects in the exact shape shown. Omit the whole field rather than emitting rows you cannot fill.
- Output only the JSON object — no preamble, no code fences, no trailing prose.${ctx}`;
}
/**
* Expand a universe character's structured attributes from reference image(s).
*
* @param {object} args
* @param {string} args.universeId
* @param {string} args.entryId — character id
* @param {string} [args.name] — entity name, for prompt context
* @param {string} [args.context] — extra known context to disambiguate
* @param {string[]} args.screenshots — image paths the runner can load (bare
* filenames under data/screenshots, or absolute paths for gallery images)
* @param {string} [args.providerId] — preferred API provider id
* @param {string} [args.model] — preferred model id
* @returns {Promise<object>} `{ fields, updatedFields, llm }` where `fields` is
* a `{ name: value }` map of the blank attributes the model proposes filling
* (already merged + sanitized), or `{ locked: true, updatedFields: [] }` when
* the character is locked.
*/
export async function expandEntityFromImages({
universeId, entryId, name, context, screenshots, providerId, model,
} = {}) {
const images = Array.isArray(screenshots) ? screenshots.filter((s) => typeof s === 'string' && s.trim()) : [];
if (images.length === 0) {
throw new ServerError('At least one image is required', { status: 400, code: 'VALIDATION_ERROR' });
}
if (images.length > VISION_EXPAND_MAX_IMAGES) {
throw new ServerError(`Too many images — analyze at most ${VISION_EXPAND_MAX_IMAGES} at once`, {
status: 400, code: 'VALIDATION_ERROR',
});
}
const universe = await getUniverse(universeId);
const list = Array.isArray(universe.characters) ? universe.characters : [];
const idx = list.findIndex((e) => e.id === entryId);
if (idx < 0) {
throw new ServerError(`Character ${entryId} not found in universe`, {
status: 404, code: 'UNIVERSE_CANON_NOT_FOUND',
});
}
const target = list[idx];
if (target.locked === true) {
return { universe, entry: target, locked: true, updatedFields: [] };
}
// Same blank predicate the completeness scan and the text expand use — a
// second implementation here is how a pre-migration character carrying only
// the legacy `description` alias ended up in the "still blank" ask.
const blankFields = [...STRING_FIELDS, ...LIST_FIELDS].filter((f) => bibleFieldIsBlank(target, f));
const provider = await resolveAPIProvider(providerId);
assertProvider(provider, {
message:
'Analyzing an image needs an API-based AI provider with a vision-capable model (e.g. Ollama with a llava/qwen-vl model, LM Studio, or an OpenAI-compatible endpoint). Configure one under Settings → Providers.',
code: 'NO_API_PROVIDER',
status: 503,
});
const prompt = buildVisionExpandPrompt({
name, context, imageCount: images.length, blankFields,
});
const result = await runPromptThroughProvider({
provider,
prompt,
source: 'universe-vision-expand',
model: model || undefined,
screenshots: images,
});
// A CLI/TUI fallback silently drops the images and would return attributes
// hallucinated from the text prompt alone — reject it outright.
const ranProvider = assertVisionRunUsedImages(result, provider);
let content;
try {
content = parseLLMJSON(result.text || '');
} catch (e) {
throw new ServerError(e.message, { status: 502, code: 'UNIVERSE_VISION_EXPAND_BAD_JSON' });
}
if (!content || typeof content !== 'object' || Array.isArray(content)) {
throw new ServerError('The vision model returned a non-object response — try a different model or clearer images.', {
status: 502, code: 'UNIVERSE_VISION_EXPAND_BAD_JSON',
});
}
const llm = { provider: ranProvider.id || provider.id, model: result.model || null };
// Compute the no-clobber merge against the loaded target — `applyExpansion`
// fills only blank fields and sanitizes list rows. Surface just the fields it
// would fill (name → merged value) so the modal can present them for review;
// the client applies the kept/edited values via the normal entry-PATCH path.
for (const key of NON_VISUAL_PROPOSAL_KEYS) delete content[key];
const { merged, updatedFields } = applyExpansion(target, content);
const fields = {};
for (const f of updatedFields) fields[f] = merged[f];
if (updatedFields.length > 0) {
console.log(`✨ Universe vision expand proposed — universe=${shortId(universeId)} entry=${shortId(entryId)} fields=${updatedFields.length}`);
}
return { fields, updatedFields, llm };
}
export const __testing = { buildVisionExpandPrompt };