Skip to content

Open source CapCut alternative.

TypeScript

feat(site): stock search ranked by fit, described voices resolved

David Lea82ed2849082
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 8cb0a0e4..1619f94a 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -183,7 +183,10 @@ import { queueWatchSweep, withSweepPaused } from "./watch/sweep";
import { synthesizeMusic } from "./audioGen";
import { composeMusicPrompt } from "./composeGen";
import { stockAssetInDoc } from "./genvideo/docWriter";
-import { resolveVoice, synthesizeSpeech, SPEECH_VOICES } from "./tts";
+import { resolveVoiceAsk, synthesizeSpeech, SPEECH_VOICES } from "./tts";
+import { hostedPost } from "./hosted";
+import { cutJudge } from "./chatRuntime";
+import { askJudgeChunked, noul } from "./judge";
import { clampVideoDuration, defaultVideoAspects, videoResolutionOf } from "./videoModels";
import { DUCK_DEFAULT, generateSubtitlesReadout } from "./voiceover";
import {
@@ -2903,8 +2906,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
return { note: res.message };
},
- stock_search: (s, input) => {
- const q = String(input.query ?? "").trim().toLowerCase();
+ stock_search: async (s, input) => {
+ const q = String(input.query ?? "").trim();
const kindIn =
input.kind === "video" ||
input.kind === "image" ||
@@ -2961,13 +2964,9 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
});
}
}
- const words = q.split(/\s+/).filter(Boolean);
- const matches = hits.filter((h) => {
- const hay = [h.id, h.category, h.prompt, h.persona ?? "", ...h.tags]
- .join(" ")
- .toLowerCase();
- return words.every((w) => hay.includes(w));
- });
+ // A query ranks every candidate by a judged fit; no query browses the
+ // catalog in manifest order.
+ const matches = q ? await rankStock(q, hits) : hits;
const trim = (t: string, n: number) => (t.length > n ? `${t.slice(0, n - 1)}…` : t);
return {
results: matches.slice(0, 12).map((h) => ({
@@ -3841,7 +3840,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
const lane = targetSubtitleTrack(input);
if (!laneCues(useEditor.getState().subtitles, lane).some((c) => c.text.trim()))
throw new ToolError("No subtitles on that track — generate subtitles first.");
- const voice = resolveVoice(typeof input.voice === "string" ? input.voice : undefined);
+ const voice = await resolveVoiceAsk(typeof input.voice === "string" ? input.voice : undefined);
// The shared readout also re-times the cues to the generated voice's
// pace, keeping the word highlighter in step.
const out = await generateSubtitlesReadout(voice, {
@@ -4234,6 +4233,56 @@ class ToolError extends Error {}
* decodes, the transcription mixdown. A page has those primitives and a
* headless process installs them; a process that installed neither refuses
* these with a typed error rather than a stray "document is not defined". */
+/** The stock candidates an editor searching for `query` would want, best
+ * first: one judged fit per candidate, in chunks, kept above the setting's
+ * floor. */
+async function rankStock<T extends { id: string; kind: string; category: string; prompt: string; persona?: string; tags: string[] }>(
+ query: string,
+ candidates: T[],
+): Promise<T[]> {
+ if (candidates.length === 0) return [];
+ const post = (payload: Record<string, unknown>, signal?: AbortSignal) =>
+ hostedPost("/api/inference/judge", payload, signal);
+ const byKey = new Map<string, T>(candidates.map((c, i) => [`fit::${i}`, c]));
+ const questions = Object.fromEntries(
+ candidates.map((_, i) => [
+ `fit::${i}`,
+ noul(`Would an editor searching the stock catalog for \`query\` want candidate ${i} in \`candidates\`?`, {
+ true: "The candidate is what the search describes, or serves its purpose well.",
+ false: "The candidate is unrelated to the search, or only shares a word with it.",
+ }),
+ ]),
+ );
+ const state = (keys: string[]) => ({
+ query,
+ candidates: keys.map((k) => {
+ const c = byKey.get(k)!;
+ return {
+ i: Number(k.slice("fit::".length)),
+ kind: c.kind,
+ category: c.category,
+ prompt: c.prompt.slice(0, 240),
+ ...(c.persona ? { persona: c.persona.slice(0, 160) } : {}),
+ tags: c.tags.slice(0, 8),
+ };
+ }),
+ });
+ let answers: Record<string, { type: string; noul?: number }>;
+ try {
+ answers = (await askJudgeChunked(post, state, questions, STOCK_JUDGE_CHUNK)) as Record<string, { type: string; noul?: number }>;
+ } catch (e) {
+ throw new ToolError(`Stock search is unavailable: ${e instanceof Error ? e.message : String(e)}`);
+ }
+ const floor = cutJudge().stockFit;
+ return candidates
+ .map((c, i) => ({ c, p: answers[`fit::${i}`]?.noul ?? 0 }))
+ .filter((x) => x.p >= floor)
+ .sort((a, b) => b.p - a.p)
+ .map((x) => x.c);
+}
+
+const STOCK_JUDGE_CHUNK = 50;
+
export const MEDIA_RUNTIME_TOOLS: ReadonlySet<string> = new Set([
"convert_media",
"watch_video",
@@ -4298,7 +4347,7 @@ async function synthesizeVoiceover(
) {
const projectId = useEditor.getState().projectId;
if (!projectId) throw new ToolError("No project open.");
- const voice = resolveVoice(typeof input.voice === "string" ? input.voice : undefined);
+ const voice = await resolveVoiceAsk(typeof input.voice === "string" ? input.voice : undefined);
const direction =
typeof input.direction === "string" && input.direction.trim()
? input.direction.trim()
diff --git a/site/src/cut/lib/tts.ts b/site/src/cut/lib/tts.ts
index 6ff45623..ba82ec24 100644
--- a/site/src/cut/lib/tts.ts
+++ b/site/src/cut/lib/tts.ts
@@ -4,6 +4,8 @@ import { geminiModelRoleNames } from "@/lib/inference/gemini-models";
import { bytesFromBase64 } from "./bytes";
import { NO_CREDITS_MESSAGE } from "./credits";
import { hostedPost } from "./hosted";
+import { cutJudge } from "./chatRuntime";
+import { askJudge, choice } from "./judge";
import { importFileToProject } from "./media";
import { mediaSlug, type MediaAsset } from "./types";
import {
@@ -18,6 +20,37 @@ import {
// the build script); re-exported here so existing importers keep working.
export { DEFAULT_VOICE, resolveVoice, SPEECH_VOICES, type SpeechVoice };
+const VOICE_QUESTION = choice(
+ "`wanted` describes the voice the user asked for in a video editor's voiceover. Which of these voices matches the description best?",
+ Object.fromEntries(
+ SPEECH_VOICES.map((v) => [v.id, `${v.name}: ${v.style}, ${v.gender === "f" ? "a woman" : "a man"}`]),
+ ) as Record<string, string>,
+);
+
+/** A voice ask resolved to a model id: an id or display name matches as
+ * `resolveVoice` does; a description ("a warm older man") is judged against
+ * the catalog, and one the judge cannot commit to takes the default. */
+export async function resolveVoiceAsk(wanted?: string): Promise<string> {
+ const w = wanted?.trim();
+ if (!w) return DEFAULT_VOICE;
+ const named = resolveVoice(w);
+ if (named !== DEFAULT_VOICE || w.toLowerCase() === DEFAULT_VOICE.toLowerCase()) return named;
+ if (SPEECH_VOICES.some((v) => v.name.toLowerCase() === w.toLowerCase())) return named;
+ try {
+ const { answers } = await askJudge(
+ (payload, signal) => hostedPost("/api/inference/judge", payload, signal),
+ { wanted: w.slice(0, 500) },
+ { voice: VOICE_QUESTION },
+ );
+ const pick = answers.voice;
+ return pick.confidence >= cutJudge().voicePick && SPEECH_VOICES.some((v) => v.id === pick.choice)
+ ? pick.choice
+ : DEFAULT_VOICE;
+ } catch {
+ return DEFAULT_VOICE;
+ }
+}
+
// Client side of AI voiceovers: Gemini speech generation on Donkey's hosted
// inference routes, with the user's Donkey sign-in and credits (same-origin on
// the cut hosts, like image/video generation). Each segment comes back as raw
@@ -366,7 +399,7 @@ export async function renderSpeechClip(
const lines = raw.map((l, i) => ({ text: plan.texts[i] ?? l.text, at: l.at }));
const offset = Math.min(...lines.map((s) => s.at));
- const voice = resolveVoice(opts.voice);
+ const voice = await resolveVoiceAsk(opts.voice);
// A small pool: readouts can be dozens of lines, one hosted call each.
const clips: PcmClip[] = new Array(lines.length);