Open source CapCut alternative.
TypeScript
feat(site): Gemini 3.8 Flash for chat, review and captions [rebuild]
David Le4be37bb7803a
diff --git a/site/src/cut/server/cloud/clipTitle.ts b/site/src/cut/server/cloud/clipTitle.tsindex f6916e53..0ac8c6fe 100644--- a/site/src/cut/server/cloud/clipTitle.ts+++ b/site/src/cut/server/cloud/clipTitle.ts@@ -187,7 +187,7 @@ export const clipTitleCloud = {model: MODEL,contents: [{ role: "user", parts }],// JSON mode without a schema — constrained decoding degrades output.- config: { responseMimeType: "application/json", temperature: 0.4 },+ config: { responseMimeType: "application/json" },});} catch (error) {await recordFailedInferenceUsage({diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.tsindex 5f96b719..2d3dc2c5 100644--- a/site/src/lib/credits/provider-pricing.ts+++ b/site/src/lib/credits/provider-pricing.ts@@ -10,10 +10,12 @@ import {geminiModels,geminiMusicModels,geminiOmniModels,+ geminiTranscribeModels,geminiTtsModels,type GeminiModel,type GeminiMusicModel,type GeminiOmniModel,+ type GeminiTranscribeModel,type GeminiTtsModel,} from "@/lib/inference/gemini-models";import { falMatteModels, type FalMatteModel } from "@/lib/inference/matte-models";@@ -239,7 +241,7 @@ const openAITextCreditRates: {{ model: "o4-mini", input: "1.1", cachedInput: "0.275", output: "4.4" },];-// gemini-3.7-flash launch pricing runs through 2026-12-31; standard pricing+// gemini-3.8-flash launch pricing runs through 2026-12-31; standard pricing// starts 2027-01-01 UTC. Audio input bills at the text rate. The record entry// is a getter, so every lookup reads the clock and the switch happens on// schedule without a deploy.@@ -271,9 +273,9 @@ const geminiModelPricing: Record<GeminiModel, ProviderCreditPricing> = {output: "1.5",inputAudio: "0.5",}),- // Generative image editing/generation ("nano banana") bills per output image:- // ~1290 output tokens at $30/1M ≈ $0.039 each.- [geminiModels.flashImage]: { generationCostMicros: usdWithMargin("0.039") },+ // Generative image editing/generation ("nano banana 2") bills per output+ // image: $0.045 at 1K, the size the scripts render.+ [geminiModels.flashImage]: { generationCostMicros: usdWithMargin("0.045") },// "Nano banana pro" bills per output image by resolution: $0.134 at 1K/2K, $0.24 at// 4K. Our per-image price can't vary by resolution, so this charges the 1K/2K rate// (the default is 2K); 4K currently under-bills. Make billing resolution-aware before@@ -292,15 +294,31 @@ function geminiTtsCreditPricing(model: string): ProviderCreditPricing | undefinereturn geminiTtsModelPricing[model as GeminiTtsModel];}+// Speech-to-text (Gemini Transcribe) bills by tokens: $2/1M audio in (about+// $0.003 a minute) and $12/1M text out. It runs on the plain "gemini" provider+// through generateContent, so the lookup rides geminiCreditPricing. The Record+// is keyed by GeminiTranscribeModel, so adding an id without a price fails the+// build.+const geminiTranscribeModelPricing: Record<GeminiTranscribeModel, ProviderCreditPricing> = {+ [geminiTranscribeModels.file]: textAudioTokenPricing({+ input: "2",+ output: "12",+ inputAudio: "2",+ }),+};+// Unified video generation (Gemini Omni Flash). The provider bills by tokens —-// $1.50/1M input, $17.50/1M output, and a rendered second of 720p video is-// 5,792 output tokens — but the render is async: the submit response carries no-// token counts, and charging on the completing poll instead would let one-// balance launch unbounded concurrent renders. So a clip charges FLAT at-// submit (usage {generationCount: 1}): ~10s × 5,792 tokens/s × $17.5/1M ≈-// $1.02, the model's fixed-length output. The token rates stay for the rare-// interaction that completes synchronously with real counts. The Record is-// keyed by GeminiOmniModel, so adding an Omni id without a price fails the build.+// $1.50/1M input, $17.50/1M output, and a rendered second of video costs a+// fixed number of output tokens per resolution (geminiOmniTokensPerSecond) —+// but the render is async: the submit response carries no token counts, and+// charging on the completing poll instead would let one balance launch+// unbounded concurrent renders. So a clip charges at submit in units of one+// second of 360p video (1,931 output tokens ≈ $0.034): the adapter counts the+// units the requested length and resolution add up to (a 10s 720p clip is 30,+// a 10s 4k clip 90), the preflight holds the balance to that count, and the+// same count bills. The token rates stay for the rare interaction that+// completes synchronously with real counts. The Record is keyed by+// GeminiOmniModel, so adding an Omni id without a price fails the build.const geminiOmniModelPricing: Record<GeminiOmniModel, ProviderCreditPricing> = {[geminiOmniModels.flashVideo]: {...textTokenPricing({@@ -308,7 +326,8 @@ const geminiOmniModelPricing: Record<GeminiOmniModel, ProviderCreditPricing> = {input: "1.5",output: "17.5",}),- generationCostMicros: usdWithMargin("1.02"),+ // One unit is a second of 360p: 1,931 output tokens at $17.50/1M.+ generationCostMicros: usdWithMargin("0.033793"),},};@@ -358,14 +377,17 @@ function geminiCreditPricing(model: string): ProviderCreditPricing | undefined {// Exact match wins, so a specific id is never shadowed by a broader prefix entry (e.g. an image// model "…-flash-image" under the "…-flash" text prefix). For dated/snapshot ids that only match// by prefix, try the longest (most specific) key first for the same reason.- const exact = geminiModelPricing[model as GeminiModel];+ const exact =+ geminiModelPricing[model as GeminiModel] ??+ geminiTranscribeModelPricing[model as GeminiTranscribeModel];if (exact) {return exact;}- const byLongestPrefix = Object.entries(geminiModelPricing).sort(- ([a], [b]) => b.length - a.length,- );+ const byLongestPrefix = [+ ...Object.entries(geminiModelPricing),+ ...Object.entries(geminiTranscribeModelPricing),+ ].sort(([a], [b]) => b.length - a.length);for (const [id, pricing] of byLongestPrefix) {if (modelMatches(model, id)) {return pricing;diff --git a/site/src/lib/inference/adapters/gemini-responses.ts b/site/src/lib/inference/adapters/gemini-responses.tsindex 3e1e231f..a2a99d38 100644--- a/site/src/lib/inference/adapters/gemini-responses.ts+++ b/site/src/lib/inference/adapters/gemini-responses.ts@@ -671,25 +671,17 @@ function thinkingLevelFromBody(body: JsonObject): ThinkingLevel | undefined {}function generationConfigFromBody(body: JsonObject): Partial<GenerateContentConfig> {+ // Sampling knobs (temperature, top_p, top_k) are not forwarded: the Gemini+ // 3.x line ignores them, and the models are tuned for their defaults.const config: Partial<GenerateContentConfig> = {};- const temperature = numberValue(body.temperature);- if (temperature !== undefined) {- config.temperature = temperature;- }-- const topP = numberValue(body.top_p) ?? numberValue(body.topP);- if (topP !== undefined) {- config.topP = topP;- }-const maxOutputTokens = numberValue(body.max_output_tokens) ?? numberValue(body.maxOutputTokens);if (maxOutputTokens !== undefined) {config.maxOutputTokens = maxOutputTokens;}// Bound reasoning so thinking tokens (which count against maxOutputTokens) can't starve the// structured output. Callers driving tight per-turn loops pass a small budget; 0 disables thinking.- // Gemini 3.x models (e.g. gemini-3.7-flash) take thinking_level (minimal|low|medium|high), NOT the- // integer thinking_budget — passing the budget to them is silently ignored. Prefer the level when the+ // Gemini 3.x models (e.g. gemini-3.8-flash) take thinking_level (low|medium|high; minimal on the+ // models that still accept it); the integer thinking_budget is silently ignored there. Prefer the level when the// caller sets it; the two are mutually exclusive. Older 2.x models still use thinking_budget. In both// cases request the thought summary so callers can persist the reasoning (the normalized response// separates it from output_text).diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.tsindex 3a33e636..238d513d 100644--- a/site/src/lib/inference/gemini-models.ts+++ b/site/src/lib/inference/gemini-models.ts@@ -8,11 +8,11 @@// Canonical, dated-or-versioned model IDs. Add a new constant when adopting a// new model; do not inline raw version strings at call sites.export const geminiModels = {- flash: "gemini-3.7-flash",+ flash: "gemini-3.8-flash",flashLite: "gemini-3.1-flash-lite",- // Generative image editing/generation ("nano banana"). Bump here when adopting a- // newer image model.- flashImage: "gemini-2.5-flash-image",+ // Generative image editing/generation ("nano banana 2"). Bump here when+ // adopting a newer image model.+ flashImage: "gemini-3.1-flash-image",// "Nano banana pro": higher-fidelity image editing/generation that takes a real// aspectRatio + imageSize (1K/2K/4K) via imageConfig. GA since 2026-05-28 (the// -preview id was shut down 2026-06-25); served only on Vertex's global@@ -22,13 +22,13 @@ export const geminiModels = {export type GeminiModel = (typeof geminiModels)[keyof typeof geminiModels];-// Unified video generation (Gemini Omni Flash): one Interactions API call takes-// text plus optional seed/reference images and renders up to ~10s of 720p video-// with audio. The clip length is the model's call — there is no duration knob —-// and image output is not enabled for this model. Bump here when adopting a-// newer Omni.+// Unified video generation (Gemini Omni 1.1 Flash): one Interactions API call+// takes text plus optional seed/reference images and renders a clip with audio.+// The request names the resolution and may name the length; with no length the+// model picks (up to ~10s). Image output is not enabled for this model. Bump+// here when adopting a newer Omni.export const geminiOmniModels = {- flashVideo: "gemini-omni-flash-preview",+ flashVideo: "gemini-omni-1.1-flash-preview",} as const;export type GeminiOmniModel = (typeof geminiOmniModels)[keyof typeof geminiOmniModels];@@ -37,6 +37,35 @@ export type GeminiOmniModel = (typeof geminiOmniModels)[keyof typeof geminiOmniM// client registry (videoModels.ts) reads the same number.export const geminiOmniMaxReferenceImages = 3;+// Output sizes an Omni render can be asked for. 720p is native; 1080p and 4k+// are upscaled by the model. The panel, the chat tool schema, and the adapter+// all read this list.+export const geminiOmniResolutions = ["360p", "720p", "1080p", "4k"] as const;++export type GeminiOmniResolution = (typeof geminiOmniResolutions)[number];++export const geminiOmniDefaultResolution: GeminiOmniResolution = "720p";++// Clip lengths an Omni render can be asked for, in whole seconds. A request+// with no length lets the model pick, which lands at the top of this range.+export const geminiOmniDurationSeconds = { min: 4, max: 10 } as const;++// Output tokens per rendered second of video with audio, by resolution — the+// unit Omni bills in. A render is charged at submit from the length and+// resolution it asked for (provider-pricing.ts), since the async submit+// returns no counts of its own.+export const geminiOmniTokensPerSecond: Record<GeminiOmniResolution, number> = {+ "360p": 1_931,+ "720p": 5_792,+ "1080p": 8_688,+ "4k": 17_376,+};++/** The billing unit for a video render: one second of 360p output. Every other+ * size bills as its multiple of this, so the credit preflight and the charge+ * count the same thing. */+export const geminiOmniUnitTokens = geminiOmniTokensPerSecond["360p"];+// Generative speech (Gemini TTS): text in, spoken audio out, with prompt-driven// style direction and inline audio tags. Bump here when adopting a newer TTS model.export const geminiTtsModels = {@@ -45,6 +74,17 @@ export const geminiTtsModels = {export type GeminiTtsModel = (typeof geminiTtsModels)[keyof typeof geminiTtsModels];+// Speech-to-text (Gemini Transcribe): a recorded audio file in, the transcript+// with per-word timings out, the language detected by the model. Runs through+// generateContent with an audio transcription config; it takes no prompt,+// system instruction, or output schema. Bump here when adopting a newer one.+export const geminiTranscribeModels = {+ file: "gemini-3.5-transcribe-preview",+} as const;++export type GeminiTranscribeModel =+ (typeof geminiTranscribeModels)[keyof typeof geminiTranscribeModels];+// Generative music (Gemini/Lyria) over the Interactions API: a text prompt in,// an instrumental clip out — the background bed for a cut. `clip` renders a// fixed ~30s clip; `pro` a full-length (~2min) track. These are the legacy@@ -73,4 +113,6 @@ export const geminiModelRoles = {// against the plan, sheets, and benchmarks. Runs the strongest multimodal// judge we serve — bump here to upgrade every review gate at once.review: geminiModels.flash,+ // Hosted speech-to-text: the recorded mix in, word-timed cues out.+ transcription: geminiTranscribeModels.file,} as const;