Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Creative Director — server-side scene evaluator (local-vision path).
*
* The `evaluate` step judges a freshly-rendered scene against the style spec +
* scene intent. Historically this spawned a full CoS coding agent that landed
* on the active provider's HEAVY model (Opus for Claude) purely to Read a
* couple of thumbnails and PATCH a verdict — vastly overpowered for a bounded
* "look at 1-3 frames, score 3 dimensions, accept/retry/fail" decision.
*
* This module runs that same decision server-side through PortOS's existing
* vision-API path (`runPromptThroughProvider({ screenshots })`), preferring a
* LOCAL vision-capable model (Ollama/LM Studio VLM). A CoS *agent* task can't
* use a local vision model at all — `agentProviderResolution.js` rejects
* `type:'api'` providers (no file-writing harness) and a claude-on-Ollama CLI
* coding model isn't vision-capable — so the only way to evaluate on a local
* vision model is this direct API call.
*
* `dispatchSceneEvaluation` is the single entry point the orchestrator calls.
* When no vision-capable API provider is configured/available (or the vision
* call fails), it falls back to the original Opus-agent path
* (`enqueueEvaluateTask`), so installs without a local vision model keep
* working exactly as before.
*/
import { existsSync } from 'fs';
import { randomUUID } from 'crypto';
import { z } from 'zod';
import { PATHS } from '../../lib/fileUtils.js';
import { safeUnder } from '../../lib/ffmpeg.js';
import { extractJson } from '../../lib/jsonExtract.js';
import { runPromptThroughProvider, assertVisionRunUsedImages } from '../promptRunner.js';
import { buildEvaluateVisionPrompt, CD_MAX_SCENE_RETRIES } from '../creativeDirectorPrompts.js';
import { getSettings } from '../settings.js';
import { getProviderById } from '../providers.js';
import { listVisionModels } from '../localLlm.js';
import { addItem } from '../mediaCollections.js';
import { updateScene, recordRun, updateRun } from './local.js';
import { resolveStagePin } from './projectsLogic.js';
import { enqueueEvaluateTask } from './agentBridge.js';
import { logFailureWithStack } from '../../lib/failureLogging.js';
// Cap frames per call — the runner base64-inlines every frame into one request
// body, so a large batch balloons the prompt and a local VLM's context window.
// 5 samples across the timeline is plenty to judge a short clip.
const MAX_EVAL_FRAMES = 5;
// Local VLMs on modest hardware can be slow to first token; honor the
// configured provider timeout and use this default only when it is absent.
// This timeout is the only bound on generation length — the toolkit's api
// runner does not send `max_tokens`, so a verbose model is capped by time, not
// tokens (the structured-JSON prompt keeps a compliant model's reply short).
const DEFAULT_VISION_EVAL_TIMEOUT_MS = 180000;
// Local backends served by an aiToolkit `api`-type provider. Auto-resolution
// only picks from these — the whole point is a LOCAL vision model. An explicit
// assignment (below) may still point at any configured API vision provider.
const LOCAL_VISION_BACKENDS = new Set(['ollama', 'lmstudio']);
// Only `accepted` is load-bearing — everything else is advisory. Map an
// explicit JSON `null` to `undefined` (absent) BEFORE validation, then
// `.catch(undefined)` drops anything that doesn't fit rather than rejecting the
// whole verdict. Two footguns this guards against:
// - Numbers: `z.coerce.number` turns `null`/an object into `0`/`NaN`; the
// null→undefined map + min/max bound + `.catch` mean a model emitting
// `"score": 85` (percent) or `"imageStrength": null` drops the field instead
// of forcing 0 or nuking the verdict.
// - Strings: use strict `z.string()` (NOT `z.coerce.string()`) so a malformed
// nested shape like `"refinedPrompt": {"prompt":"add fog"}` doesn't coerce to
// the literal `"[object Object]"` and get written as the retry render prompt.
// A non-string just drops to undefined, leaving the prompt unchanged.
const nullToUndefined = (v) => (v === null ? undefined : v);
const optNumber = z.preprocess(nullToUndefined, z.coerce.number().min(0).max(1).optional().catch(undefined));
const optString = (max) => z.preprocess(nullToUndefined, z.string().max(max).optional().catch(undefined));
const verdictSchema = z.object({
accepted: z.boolean(),
score: optNumber,
notes: optString(2000),
refinedPrompt: optString(8000),
imageStrength: optNumber,
});
/**
* Parse a vision model's reply into a validated verdict. Uses the shared
* `extractJson` (code-fence stripping, balanced-block walking, trailing-comma
* repair) to tolerate chatter around the object, preferring the block that
* actually carries a boolean `accepted`. Throws when no usable verdict is
* present so the caller falls back to the agent path.
*
* Pure — exported for tests.
*/
export function parseVisionVerdict(text) {
const { value } = extractJson(String(text || ''), {
shapePredicate: (v) => v && typeof v === 'object' && typeof v.accepted === 'boolean',
});
if (value === undefined) throw new Error('no JSON verdict in vision reply');
return verdictSchema.parse(value);
}
/**
* Resolve which API provider + model should run the vision evaluation.
*
* Order:
* 1. Explicit assignment — the project's own `modelOverrides.evaluation` pin
* (per-project CD provider/model pins) when set, else the global
* `settings.creativeDirector.evaluation` pin (set via AI Assignments).
* Honored as long as it's an enabled API provider; the assigned `model`
* (if any) is used verbatim.
* 2. Auto — the first installed local (Ollama/LM Studio) vision-capable model.
*
* @param {object} [project] the CD project, for its per-project override.
* @returns {Promise<{ provider: object, model: string|undefined }|null>} null
* when nothing suitable is configured (caller falls back to the agent).
*/
export async function resolveVisionEvalTarget(project) {
if (project?.workspace === 'video' && project.videoExecution?.choices) {
const choice = project.videoExecution.choices.evaluation;
if (choice.type !== 'api') return null;
const provider = await getProviderById(choice.providerId);
if (!provider || provider.type !== 'api' || provider.enabled === false) throw new Error('The reviewed evaluation provider is unavailable. Change Models or Settings and Resume.');
return { provider, model: choice.model };
}
// An API provider is usable only if it exists and is enabled — vision runs
// through the api-only chat path, so a CLI/TUI provider can't serve it.
const usableApiProvider = async (id) => {
const p = await getProviderById(id).catch(() => null);
return p && p.type === 'api' && p.enabled !== false ? p : null;
};
const settings = await getSettings().catch(() => ({}));
const assigned = resolveStagePin('evaluation', project, settings);
if (assigned.providerId) {
const provider = await usableApiProvider(assigned.providerId);
if (provider) return { provider, model: assigned.model || undefined };
// A stale/invalid pin shouldn't silently downgrade to Opus — fall through
// to auto-resolution so a healthy local VLM is still preferred.
}
// Scan ALL local vision candidates, not just the first: the first entry may
// belong to a disabled/missing provider (e.g. Ollama off) while a later one
// (e.g. LM Studio) is usable. Stopping at the first would wrongly fall back
// to the Opus agent.
const visionModels = await listVisionModels().catch(() => []);
for (const m of visionModels) {
if (!LOCAL_VISION_BACKENDS.has(m.backend)) continue;
const provider = await usableApiProvider(m.providerId);
if (provider) return { provider, model: m.id };
}
return null;
}
/**
* Collect the on-disk frame paths for a scene's evaluation. Prefers the
* multi-frame timeline samples; falls back to the single `{jobId}.jpg`
* thumbnail. Only returns files that actually exist (frames can be deleted with
* the underlying video while a project is paused).
*/
function collectFramePaths(scene) {
// safeUnder basenames + rejects `..`/slashes so a tampered scene.renderedJobId
// or evaluationFrames entry can't traverse outside the thumbnails dir.
const frames = Array.isArray(scene.evaluationFrames) ? scene.evaluationFrames : [];
const paths = frames
.map((f) => safeUnder(PATHS.videoThumbnails, f))
.filter((p) => p && existsSync(p))
.slice(0, MAX_EVAL_FRAMES);
if (paths.length) return paths;
if (scene.renderedJobId) {
const single = safeUnder(PATHS.videoThumbnails, `${scene.renderedJobId}.jpg`);
if (single && existsSync(single)) return [single];
}
return [];
}
/**
* Evaluate a scene with a vision model. Resolves the provider, attaches the
* sampled frames, and parses a structured verdict.
*
* @returns {Promise<
* | { ok: true, verdict: object, llm: { provider: string, model: string|null }, runId: string }
* | { ok: false, fallbackToAgent: true, reason: string }
* >}
*/
export async function evaluateSceneWithVision(project, scene) {
const target = await resolveVisionEvalTarget(project);
if (!target) {
return { ok: false, fallbackToAgent: true, reason: 'no vision-capable API provider configured' };
}
const frames = collectFramePaths(scene);
if (frames.length === 0) {
// No image on disk — the vision API path has nothing to send. The agent's
// template has the same prerequisite, but let it handle the edge case.
return { ok: false, fallbackToAgent: true, reason: 'no evaluation frames on disk' };
}
// Record a RUNNING evaluate run BEFORE the bounded vision call so a
// concurrent advanceAfterSceneSettled (user clicks Start/Resume, or a stale
// completion fires) sees a live run via completionHook's `noLiveEvaluateRun`
// check and won't dispatch a second evaluation of the same render. This
// mirrors the agent path, which records a running run in agentBridge before
// enqueueing. applySceneVerdict flips this run to completed on success.
const runId = randomUUID();
await recordRun(project.id, {
runId,
kind: 'evaluate',
sceneId: scene.sceneId || null,
status: 'running',
via: 'vision',
provider: target.provider.id,
model: target.model || null,
}).catch((err) => console.log(`⚠️ CD vision recordRun(running) failed: ${err.message}`));
try {
// The commission style source's reference images (#8724) ride AFTER the
// frames so the prompt can name each group by position. A project without
// them sends exactly the frames, as before.
const references = project.styleReferenceImages?.length
? (await import('../creativeStyleSources.js')).styleReferenceImagePaths(project.styleReferenceImages)
: [];
const prompt = buildEvaluateVisionPrompt(project, scene, frames.length, { referenceCount: references.length });
const result = await runPromptThroughProvider({
provider: target.provider,
model: target.model,
prompt,
source: 'cd-scene-evaluate',
screenshots: [...frames, ...references],
timeout: target.provider.timeout ?? DEFAULT_VISION_EVAL_TIMEOUT_MS,
...(project.workspace === 'video' ? { allowFallback: false } : {}),
});
// Guard against a silent fallback to a non-API provider (which would drop the
// images and hallucinate a verdict from the prompt text alone).
const ran = assertVisionRunUsedImages(result, target.provider);
const verdict = parseVisionVerdict(result.text);
return {
ok: true,
verdict,
llm: { provider: ran.id || target.provider.id, model: result.model || target.model || null },
runId,
};
} catch (err) {
// The vision call / parse failed — close the running run so it doesn't
// linger as "live" and block the agent fallback's own orphan handling.
await updateRun(project.id, runId, { status: 'failed', completedAt: new Date().toISOString() })
.catch(() => {});
throw err;
}
}
// advanceAfterSceneSettled is dynamically imported to avoid a static import
// cycle (completionHook → sceneEvaluator → completionHook).
const loadAdvance = () => import('./completionHook.js').then((m) => m.advanceAfterSceneSettled);
/**
* Apply a parsed verdict to a scene, driving the exact same transitions the
* agent's PATCH + orchestrator would:
* - accepted → status 'accepted' + add the rendered video to the
* project collection, then advance.
* - miss, retries left → status 'pending' + refined prompt + retryCount++
* (+ optional imageStrength), then advance (re-render).
* - miss, exhausted → status 'failed', then advance.
*
* `score`/`imageStrength` are already validated to [0,1]-or-undefined by
* `verdictSchema`, so no re-clamping is needed here.
*
* `runId` closes the RUNNING evaluate run opened by evaluateSceneWithVision.
* When absent (a direct caller), a completed run is recorded instead so the
* Runs tab still shows a local-model evaluation.
*/
export async function applySceneVerdict(project, scene, verdict, llm = null, runId = null) {
const evaluation = {
accepted: !!verdict.accepted,
score: verdict.score,
notes: verdict.notes || '',
sampledAt: new Date().toISOString(),
};
const retryCount = scene.retryCount || 0;
// Close the in-flight evaluate run (or record a fresh completed one) so the
// Runs tab shows a local-model evaluation instead of a silent gap.
const runPatch = {
status: 'completed',
completedAt: evaluation.sampledAt,
via: 'vision',
provider: llm?.provider || null,
model: llm?.model || null,
};
await (runId
? updateRun(project.id, runId, runPatch)
: recordRun(project.id, { kind: 'evaluate', sceneId: scene.sceneId || null, ...runPatch })
).catch((err) => console.log(`⚠️ CD vision run record failed: ${err.message}`));
// Advancing to the next scene / stitch is a downstream orchestration step —
// fire-and-forget and self-healing (idempotent, re-triggered on the next
// Start/Resume/completion), exactly how the PATCH route calls it. Its failure
// must NOT propagate out of applySceneVerdict: dispatchSceneEvaluation's catch
// would otherwise overwrite the scene we just correctly settled
// (accepted/pending) with 'failed' for a purely downstream error (e.g. an
// ffmpeg stitch failure on the final accepted scene).
const advance = async () => {
const advanceAfterSceneSettled = await loadAdvance();
return advanceAfterSceneSettled(project.id).catch((err) =>
logFailureWithStack(`❌ CD advance after scene ${scene.sceneId} of ${project.id} settled failed`, err));
};
if (verdict.accepted) {
await updateScene(project.id, scene.sceneId, { ...(project.workspace === 'video' ? { expectedWorkRevision: scene.workRevision || 0 } : {}), status: 'accepted', evaluation });
if (project.collectionId && scene.renderedJobId) {
await addItem(project.collectionId, { kind: 'video', ref: scene.renderedJobId })
.catch((err) => {
// Idempotent — a re-evaluation of an already-collected scene is fine.
if (!/already in collection/i.test(err.message)) {
console.log(`⚠️ CD collection add failed for ${scene.renderedJobId}: ${err.message}`);
}
});
}
console.log(`✅ CD scene ${scene.sceneId} accepted by vision (${llm?.provider || 'local'}/${llm?.model || '?'})`);
return advance();
}
if (retryCount < (project.workspace === 'video' ? project.videoExecution?.limits?.maxRetries ?? 0 : CD_MAX_SCENE_RETRIES)) {
const patch = { status: 'pending', retryCount: retryCount + 1, evaluation };
if (verdict.refinedPrompt && verdict.refinedPrompt.trim()) patch.prompt = verdict.refinedPrompt.trim();
if (verdict.imageStrength !== undefined) patch.imageStrength = verdict.imageStrength;
await updateScene(project.id, scene.sceneId, { ...patch, ...(project.workspace === 'video' ? { expectedWorkRevision: scene.workRevision || 0 } : {}) });
console.log(`🔁 CD scene ${scene.sceneId} rejected by vision — retry ${patch.retryCount}/${CD_MAX_SCENE_RETRIES}`);
return advance();
}
await updateScene(project.id, scene.sceneId, { ...(project.workspace === 'video' ? { expectedWorkRevision: scene.workRevision || 0 } : {}), status: 'failed', evaluation });
console.error(`⛔ CD scene ${scene.sceneId} failed by vision — retries exhausted`);
return advance();
}
// In-process re-entrancy guard keyed on project:sceneId, acquired SYNCHRONOUSLY
// at the top of dispatch — before any await — so it closes the whole window,
// including slow (cache-cold) provider discovery, not just the vision call. Both
// dispatch call sites (sceneRunner render-completion + completionHook orphan
// resume) go through here, so a concurrent advanceAfterSceneSettled can't start
// a second evaluation of the same render. The persisted running-run marker below
// is the cross-path/observability backstop; this Set is the primary guard.
const inflightSceneEval = new Set();
/**
* Single entry point the orchestrator calls to settle a rendered scene. Tries
* the local-vision path; on no-provider or ANY failure falls back to the
* Opus-agent path. Never throws — it runs outside the Express request lifecycle
* (job-completion hook / orchestrator), where an uncaught throw would crash the
* process.
*/
export async function dispatchSceneEvaluation(project, scene) {
const lockKey = `${project.id}:${scene.sceneId}`;
if (inflightSceneEval.has(lockKey)) {
console.log(`⏳ CD scene ${scene.sceneId} evaluation already in flight — skipping duplicate dispatch`);
return null;
}
inflightSceneEval.add(lockKey);
try {
if (project.workspace === 'video') {
const { videoReviewAllowsDispatch } = await import('./videoReview.js');
if (!await videoReviewAllowsDispatch(project.id, [])) return null;
const { getProject } = await import('./local.js');
const { effectiveVideoProject, reserveVideoAttempt, settleVideoAttempt, assertVideoAttemptDispatch, pauseVideoExecution } = await import('./videoExecution.js');
project = effectiveVideoProject(await getProject(project.id));
const currentScene = project.treatment?.scenes?.find(value => value.sceneId === scene.sceneId);
if (!currentScene || (currentScene.workRevision || 0) !== (scene.workRevision || 0)) return null;
if (project.videoExecution.choices.evaluation.type === 'agent') return enqueueEvaluateTask(project, currentScene);
const attempt = await reserveVideoAttempt(project.id, { kind: 'evaluation', expectedProductionRevision: project.videoWorkRevision || 0, key: `evaluation:${scene.sceneId}`, sceneId: scene.sceneId, workRevision: scene.workRevision || 0 });
if (!attempt) return null;
try {
await assertVideoAttemptDispatch(project.id, attempt.id);
const result = await evaluateSceneWithVision(project, currentScene);
if (!result.ok) throw new Error(result.reason || 'The reviewed vision evaluator is unavailable.');
await settleVideoAttempt(project.id, attempt.id, { status: 'completed' });
return await applySceneVerdict(project, currentScene, result.verdict, result.llm, result.runId);
} catch (error) {
await settleVideoAttempt(project.id, attempt.id, { status: 'failed' });
if (error.code !== 'VIDEO_WORK_STALE') await pauseVideoExecution(project.id, `Evaluation stopped: ${error.message}. Review Models and Resume.`);
return null;
}
}
return await runSceneEvaluation(project, scene);
} finally {
inflightSceneEval.delete(lockKey);
}
}
async function runSceneEvaluation(project, scene) {
let result;
try {
result = await evaluateSceneWithVision(project, scene);
} catch (err) {
console.log(`⚠️ CD vision eval errored for scene ${scene.sceneId} — falling back to agent: ${err.message}`);
result = { ok: false, fallbackToAgent: true, reason: err.message };
}
if (result.ok) {
try {
await applySceneVerdict(project, scene, result.verdict, result.llm, result.runId);
return { via: 'vision', verdict: result.verdict, llm: result.llm };
} catch (err) {
if (err.code === 'VIDEO_WORK_STALE') return null;
// Verdict came back but persisting/advancing threw. Don't re-run on the
// agent — that could double-apply the verdict (and re-add to the
// collection). Best-effort mark the scene failed so the orchestrator can
// recover, and never throw (this runs outside the request lifecycle).
console.error(`❌ CD applySceneVerdict failed for scene ${scene.sceneId}: ${err.message}`);
await updateScene(project.id, scene.sceneId, { ...(project.workspace === 'video' ? { expectedWorkRevision: scene.workRevision || 0 } : {}),
status: 'failed',
evaluation: {
accepted: false,
notes: `Vision verdict could not be applied: ${err.message}`,
sampledAt: new Date().toISOString(),
},
}).catch((e) => console.log(`⚠️ CD verdict-failure mark for ${scene.sceneId} failed: ${e.message}`));
const advanceAfterSceneSettled = await loadAdvance();
return advanceAfterSceneSettled(project.id).catch(() => {});
}
}
console.log(`🎬 CD scene ${scene.sceneId} evaluation via agent (${result.reason})`);
// Honor the never-throws contract: enqueueEvaluateTask can reject on a disk /
// prompt-build failure, and this runs off a media-job event listener whose
// floating promise would surface as an unhandledRejection (fatal on modern
// Node). The scene stays 'evaluating' on failure, so a later resume retries.
return enqueueEvaluateTask(project, scene).catch((err) => {
console.error(`❌ CD agent-eval enqueue failed for scene ${scene.sceneId}: ${err.message}`);
return null;
});
}