Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Vision-to-prose / vision-to-attributes modal for a canon entry.
*
* Add reference images — uploaded from disk OR picked from the image gallery —
* pick an (API/vision-capable) provider + model on demand, and either:
* - "Describe" (all kinds): turn the image(s) into an image-gen-ready prose
* description that seeds the entry's descriptor field. A single image
* describes that subject; multiple images return the description COMMON to
* all of them (the same subject from several angles, …).
* - "Build character details" (characters only): have a vision model PROPOSE
* values for the character's still-blank structured fields (palette, visual
* notes, expressions, …). The proposals are shown for review (string fields
* editable) and applied via the entry-patch path.
*
* The modal is stateless w.r.t. the universe — it returns the generated prose
* via `onApply(description)` and the reviewed attribute patch via
* `onApplyFields(patch)`, and lets the parent write them into the entry.
*/
import { useState, useCallback } from 'react';
import { Loader2, ImagePlus, Images, Sparkles, X } from 'lucide-react';
import Modal from '../ui/Modal';
import FilePickerButton from '../ui/FilePickerButton';
import toast from '../ui/Toast';
import GalleryImagePicker from '../imageGen/GalleryImagePicker';
import VisionProviderPicker from './VisionProviderPicker';
import { useAsyncAction } from '../../hooks/useAsyncAction';
import { processScreenshotUploads } from '../../services/apiMedia';
import { describeEntityFromImages, expandEntityFromImages } from '../../services/apiUniverseBuilder';
// Mirror server VISION_MAX_IMAGES so the UI stops the user before the request
// 400s.
const MAX_IMAGES = 8;
// Shared with CorrectiveReferenceModal, which offers the same vision actions
// against a different (corrective, not blank-fill) analysis.
export const KIND_NOUN = { character: 'character', place: 'place', object: 'object' };
// camelCase field name → human label for the attribute review list.
const humanizeField = (f) => f
.replace(/([a-z0-9])([A-Z])/g, '$1 $2')
.replace(/^./, (c) => c.toUpperCase());
// Unique key per reference image so a gallery image and an upload sharing a
// name don't collide in the thumbnail list or the remove handler.
const imageKey = (img) => `${img.source}:${img.filename}`;
export default function VisionDescribeModal({
open, kind, entryName, universeId, entryId, onApply, onApplyFields, onClose,
}) {
// Each: { source: 'upload'|'gallery', filename, preview }. `filename` is what
// we send to the server (an uploaded screenshot name or a gallery name);
// `preview` is the URL for the thumbnail.
const [images, setImages] = useState([]);
const [uploading, setUploading] = useState(false);
const [galleryOpen, setGalleryOpen] = useState(false);
const [result, setResult] = useState('');
// Structured attribute proposals (characters): { fields, updatedFields }.
const [proposed, setProposed] = useState(null);
// Per-field include set + edited string values for the review list.
const [selectedFields, setSelectedFields] = useState(() => new Set());
const [fieldEdits, setFieldEdits] = useState({});
// Optional "known context" the user can type to disambiguate the subject.
const [context, setContext] = useState('');
// Current vision provider/model selection, lifted from VisionProviderPicker.
const [vision, setVision] = useState({ providerId: '', model: '', hasProviders: false, noVisionModel: false });
const noun = KIND_NOUN[kind] || 'subject';
const isCharacter = kind === 'character';
const addImages = useCallback((next) => {
setImages((prev) => {
const seen = new Set(prev.map(imageKey));
const merged = [...prev];
for (const img of next) {
if (merged.length >= MAX_IMAGES) break;
if (seen.has(imageKey(img))) continue;
seen.add(imageKey(img));
merged.push(img);
}
return merged;
});
}, []);
const handleFiles = useCallback(async (fileList) => {
// Respect the cap across the already-selected set. Slice BEFORE uploading
// so over-cap files never hit the screenshots dir as orphans.
const room = MAX_IMAGES - images.length;
if (room <= 0) {
toast.error(`Up to ${MAX_IMAGES} images at a time`);
return;
}
const files = Array.from(fileList || []);
if (files.length === 0) return;
if (files.length > room) toast.error(`Only adding ${room} — max ${MAX_IMAGES} images`);
setUploading(true);
await processScreenshotUploads(files.slice(0, room), {
onSuccess: (info) => addImages([{ source: 'upload', filename: info.filename, preview: info.preview }]),
onError: (msg) => toast.error(msg),
});
setUploading(false);
}, [images.length, addImages]);
const handleGalleryPick = useCallback((item) => {
if (!item?.filename) return;
if (images.length >= MAX_IMAGES) {
toast.error(`Up to ${MAX_IMAGES} images at a time`);
return;
}
addImages([{ source: 'gallery', filename: item.filename, preview: item.previewUrl }]);
}, [images.length, addImages]);
const removeImage = useCallback((key) => {
setImages((prev) => prev.filter((img) => imageKey(img) !== key));
}, []);
// Payload shared by both vision calls.
const imagePayload = () => ({
images: images.map((img) => ({ source: img.source, filename: img.filename })),
providerId: vision.providerId || undefined,
model: vision.model || undefined,
});
const [runDescribe, describing] = useAsyncAction(async () => {
const res = await describeEntityFromImages({
kind,
name: entryName || undefined,
context: context.trim() || undefined,
...imagePayload(),
}, { silent: true });
if (res?.description) setResult(res.description);
return res;
}, { errorMessage: 'Failed to describe image(s)' });
const [runBuildAttributes, building] = useAsyncAction(async () => {
const res = await expandEntityFromImages(universeId, entryId, {
name: entryName || undefined,
context: context.trim() || undefined,
...imagePayload(),
}, { silent: true });
// Keep any prose description that's already shown — both the description and
// the structured proposals stay open so the user can apply each in turn.
if (res?.locked) {
toast.error(`${entryName || 'This character'} is locked — unlock it to fill from an image`);
return res;
}
const fields = res?.fields && typeof res.fields === 'object' ? res.fields : {};
const updated = Array.isArray(res?.updatedFields) ? res.updatedFields : [];
setProposed({ fields, updatedFields: updated });
setSelectedFields(new Set(updated));
// Seed editable string values; list fields apply their proposed value as-is.
const edits = {};
for (const f of updated) if (typeof fields[f] === 'string') edits[f] = fields[f];
setFieldEdits(edits);
if (updated.length === 0) toast(`No new details to fill for ${entryName || `this ${noun}`}`);
return res;
}, { errorMessage: 'Failed to build details from image(s)' });
// Whether the structured-attributes panel still has unsaved proposals.
const hasPendingAttributes = !!proposed && proposed.updatedFields.length > 0;
const applyDescription = () => {
onApply(result.trim());
toast.success(`Applied description to ${entryName || `this ${noun}`}`);
// Clear just the description (it's saved); keep the modal open if the
// structured proposals are still unsaved so the user can apply those too.
setResult('');
if (!hasPendingAttributes) onClose();
};
const toggleField = (f) => {
setSelectedFields((prev) => {
const next = new Set(prev);
if (next.has(f)) next.delete(f); else next.add(f);
return next;
});
};
const applyAttributes = () => {
if (!proposed) return;
const patch = {};
for (const f of proposed.updatedFields) {
if (!selectedFields.has(f)) continue;
const value = typeof proposed.fields[f] === 'string'
? (fieldEdits[f] ?? proposed.fields[f])
: proposed.fields[f];
patch[f] = value;
}
if (Object.keys(patch).length === 0) {
toast.error('Select at least one detail to apply');
return;
}
onApplyFields(patch);
toast.success(`Applied ${Object.keys(patch).length} detail${Object.keys(patch).length > 1 ? 's' : ''} to ${entryName || `this ${noun}`}`);
// Clear just the proposals (they're saved); keep the modal open if a prose
// description is still unsaved so the user can apply that too.
setProposed(null);
setSelectedFields(new Set());
setFieldEdits({});
if (!result.trim()) onClose();
};
// A vision model must be explicitly selected — otherwise the request sends
// `model: undefined` and the server resolves the provider's default, which on
// a local backend is often a text-only model that silently drops the images.
// VisionProviderPicker surfaces the "no vision model" case via `vision.model`
// being '' (it also shows the explanatory warning).
const canRun = images.length > 0 && !describing && !building && !uploading
&& vision.hasProviders && !!vision.model;
const multi = images.length > 1;
return (
<Modal open={open} onClose={onClose} size="lg" closeOnBackdrop={false} usePortal ariaLabel={`Describe ${noun} from images`}>
<div className="bg-port-card border border-port-border rounded-lg shadow-xl flex flex-col max-h-[85vh]">
<div className="flex items-center justify-between px-4 py-3 border-b border-port-border">
<h3 className="text-sm font-semibold text-white flex items-center gap-2">
<Sparkles size={14} className="text-port-accent" />
{entryName ? `"${entryName}"` : `This ${noun}`} from image{multi ? 's' : ''}
</h3>
<button type="button" onClick={onClose} className="text-gray-500 hover:text-white min-h-[44px] min-w-[44px] flex items-center justify-center" aria-label="Close">
<X size={16} />
</button>
</div>
<div className="p-4 space-y-4 overflow-y-auto">
<p className="text-xs text-gray-500">
Add one image, or several of the same {noun}, from your device or your gallery.
{isCharacter
? ' Describe them into the descriptor field, or build their structured details (palette, visual notes, expressions, …) from the reference.'
: ' A vision model turns them into image-gen-ready prose.'}
</p>
{/* Image picker + thumbnails */}
<div>
<div className="flex flex-wrap gap-2">
{images.map((img) => (
<div key={imageKey(img)} className="relative w-20 h-20">
<img
src={img.preview || (img.source === 'upload' ? `/api/screenshots/${encodeURIComponent(img.filename)}` : `/data/images/${encodeURIComponent(img.filename)}`)}
alt="reference"
className="w-full h-full object-cover rounded border border-port-border"
/>
<button
type="button"
onClick={() => removeImage(imageKey(img))}
title="Remove image"
aria-label="Remove image"
className="min-h-[44px] min-w-[44px] inline-flex items-center justify-center absolute -top-1.5 -right-1.5 bg-port-bg border border-port-border rounded-full p-0.5 text-gray-400 hover:text-port-error"
>
<X size={11} />
</button>
</div>
))}
{images.length < MAX_IMAGES ? (
<>
<FilePickerButton
accept="image/*"
multiple
onChange={(e) => handleFiles(e.target.files)}
disabled={uploading}
ariaLabel="Upload reference images"
className="w-20 h-20 flex flex-col items-center justify-center gap-1 rounded border border-dashed border-port-border text-gray-500 hover:text-port-accent hover:border-port-accent/50"
>
{uploading ? <Loader2 size={16} className="animate-spin" /> : <ImagePlus size={16} />}
<span className="text-[10px]">Upload</span>
</FilePickerButton>
<button
type="button"
onClick={() => setGalleryOpen(true)}
className="w-20 h-20 flex flex-col items-center justify-center gap-1 rounded border border-dashed border-port-border text-gray-500 hover:text-port-accent hover:border-port-accent/50"
>
<Images size={16} />
<span className="text-[10px]">Gallery</span>
</button>
</>
) : null}
</div>
</div>
{/* Provider + model picker (API/vision-capable providers only) */}
<VisionProviderPicker onChange={setVision} />
{/* Optional context to disambiguate the subject for the model. */}
<div>
<label htmlFor="vision-describe-context" className="block text-xs text-gray-500 mb-1">
Known context (optional)
</label>
<input
id="vision-describe-context"
type="text"
value={context}
onChange={(e) => setContext(e.target.value)}
placeholder={`Anything the images don't make obvious about this ${noun}`}
maxLength={2000}
className="w-full px-2 py-1.5 text-xs bg-port-bg border border-port-border rounded text-gray-200"
/>
</div>
<div className="flex flex-wrap gap-2">
<button
type="button"
onClick={runDescribe}
disabled={!canRun}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded bg-port-accent text-white text-sm disabled:opacity-40 disabled:cursor-not-allowed hover:bg-port-accent/90"
>
{describing ? <Loader2 size={14} className="animate-spin" /> : <Sparkles size={14} />}
{describing ? 'Describing…' : `Describe from image${multi ? 's' : ''}`}
</button>
{isCharacter ? (
<button
type="button"
onClick={runBuildAttributes}
disabled={!canRun}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded border border-port-accent text-port-accent text-sm disabled:opacity-40 disabled:cursor-not-allowed hover:bg-port-accent/10"
>
{building ? <Loader2 size={14} className="animate-spin" /> : <Sparkles size={14} />}
{building ? 'Analyzing…' : 'Build character details'}
</button>
) : null}
</div>
{/* Prose result — editable before applying */}
{result ? (
<div className="space-y-2">
<label htmlFor="vision-describe-result" className="block text-xs text-gray-500">
Generated description (edit before applying if you like)
</label>
<textarea
id="vision-describe-result"
value={result}
onChange={(e) => setResult(e.target.value)}
rows={6}
className="w-full px-2 py-1.5 text-xs bg-port-bg border border-port-border rounded text-gray-200 whitespace-pre-wrap"
/>
<button
type="button"
onClick={applyDescription}
disabled={!result.trim()}
className="px-3 py-1.5 rounded bg-port-accent text-white text-sm disabled:opacity-40 disabled:cursor-not-allowed hover:bg-port-accent/90"
>
Apply description
</button>
</div>
) : null}
{/* Structured attribute review — characters only */}
{proposed && proposed.updatedFields.length > 0 ? (
<div className="space-y-2">
<p className="text-xs text-gray-500">
Proposed details (only blank fields). Uncheck any you don't want; edit text inline. Existing values are never overwritten.
</p>
<div className="space-y-2 max-h-72 overflow-y-auto pr-1">
{proposed.updatedFields.map((f) => {
const value = proposed.fields[f];
const isList = Array.isArray(value);
const checked = selectedFields.has(f);
return (
<div key={f} className="border border-port-border rounded p-2 bg-port-bg">
<label className="flex items-center gap-2 text-xs text-gray-300 mb-1">
<input type="checkbox" checked={checked} onChange={() => toggleField(f)} />
<span className="font-medium">{humanizeField(f)}</span>
{isList ? <span className="text-gray-500">({value.length} item{value.length > 1 ? 's' : ''})</span> : null}
</label>
{isList ? (
<p className="text-[11px] text-gray-400 pl-6">
{value.map((row) => row?.name || row?.label).filter(Boolean).join(', ') || '—'}
</p>
) : (
<textarea
value={fieldEdits[f] ?? ''}
onChange={(e) => setFieldEdits((prev) => ({ ...prev, [f]: e.target.value }))}
disabled={!checked}
rows={2}
aria-label={humanizeField(f)}
className="w-full px-2 py-1 text-xs bg-port-card border border-port-border rounded text-gray-200 disabled:opacity-50"
/>
)}
</div>
);
})}
</div>
<button
type="button"
onClick={applyAttributes}
disabled={selectedFields.size === 0}
className="px-3 py-1.5 rounded bg-port-accent text-white text-sm disabled:opacity-40 disabled:cursor-not-allowed hover:bg-port-accent/90"
>
Apply {selectedFields.size} detail{selectedFields.size > 1 ? 's' : ''}
</button>
</div>
) : null}
</div>
<div className="flex items-center justify-end gap-2 px-4 py-3 border-t border-port-border">
<button type="button" onClick={onClose} className="px-3 py-1.5 rounded border border-port-border text-gray-300 text-sm hover:text-white">
Close
</button>
</div>
</div>
<GalleryImagePicker
open={galleryOpen}
onClose={() => setGalleryOpen(false)}
onSelect={handleGalleryPick}
/>
</Modal>
);
}