Skip to content

Open source CapCut alternative.

TypeScript

feat(site): Gemini 3.8 Flash for chat, review and captions [rebuild]

David Le4be37bb7803a
diff --git a/site/src/cut/server/cloud/clipTitle.ts b/site/src/cut/server/cloud/clipTitle.ts
index f6916e53..0ac8c6fe 100644
--- a/site/src/cut/server/cloud/clipTitle.ts
+++ b/site/src/cut/server/cloud/clipTitle.ts
@@ -187,7 +187,7 @@ export const clipTitleCloud = {
model: MODEL,
contents: [{ role: "user", parts }],
// JSON mode without a schema — constrained decoding degrades output.
- config: { responseMimeType: "application/json", temperature: 0.4 },
+ config: { responseMimeType: "application/json" },
});
} catch (error) {
await recordFailedInferenceUsage({
diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.ts
index 5f96b719..2d3dc2c5 100644
--- a/site/src/lib/credits/provider-pricing.ts
+++ b/site/src/lib/credits/provider-pricing.ts
@@ -10,10 +10,12 @@ import {
geminiModels,
geminiMusicModels,
geminiOmniModels,
+ geminiTranscribeModels,
geminiTtsModels,
type GeminiModel,
type GeminiMusicModel,
type GeminiOmniModel,
+ type GeminiTranscribeModel,
type GeminiTtsModel,
} from "@/lib/inference/gemini-models";
import { falMatteModels, type FalMatteModel } from "@/lib/inference/matte-models";
@@ -239,7 +241,7 @@ const openAITextCreditRates: {
{ model: "o4-mini", input: "1.1", cachedInput: "0.275", output: "4.4" },
];
-// gemini-3.7-flash launch pricing runs through 2026-12-31; standard pricing
+// gemini-3.8-flash launch pricing runs through 2026-12-31; standard pricing
// starts 2027-01-01 UTC. Audio input bills at the text rate. The record entry
// is a getter, so every lookup reads the clock and the switch happens on
// schedule without a deploy.
@@ -271,9 +273,9 @@ const geminiModelPricing: Record<GeminiModel, ProviderCreditPricing> = {
output: "1.5",
inputAudio: "0.5",
}),
- // Generative image editing/generation ("nano banana") bills per output image:
- // ~1290 output tokens at $30/1M ≈ $0.039 each.
- [geminiModels.flashImage]: { generationCostMicros: usdWithMargin("0.039") },
+ // Generative image editing/generation ("nano banana 2") bills per output
+ // image: $0.045 at 1K, the size the scripts render.
+ [geminiModels.flashImage]: { generationCostMicros: usdWithMargin("0.045") },
// "Nano banana pro" bills per output image by resolution: $0.134 at 1K/2K, $0.24 at
// 4K. Our per-image price can't vary by resolution, so this charges the 1K/2K rate
// (the default is 2K); 4K currently under-bills. Make billing resolution-aware before
@@ -292,15 +294,31 @@ function geminiTtsCreditPricing(model: string): ProviderCreditPricing | undefine
return geminiTtsModelPricing[model as GeminiTtsModel];
}
+// Speech-to-text (Gemini Transcribe) bills by tokens: $2/1M audio in (about
+// $0.003 a minute) and $12/1M text out. It runs on the plain "gemini" provider
+// through generateContent, so the lookup rides geminiCreditPricing. The Record
+// is keyed by GeminiTranscribeModel, so adding an id without a price fails the
+// build.
+const geminiTranscribeModelPricing: Record<GeminiTranscribeModel, ProviderCreditPricing> = {
+ [geminiTranscribeModels.file]: textAudioTokenPricing({
+ input: "2",
+ output: "12",
+ inputAudio: "2",
+ }),
+};
+
// Unified video generation (Gemini Omni Flash). The provider bills by tokens —
-// $1.50/1M input, $17.50/1M output, and a rendered second of 720p video is
-// 5,792 output tokens — but the render is async: the submit response carries no
-// token counts, and charging on the completing poll instead would let one
-// balance launch unbounded concurrent renders. So a clip charges FLAT at
-// submit (usage {generationCount: 1}): ~10s × 5,792 tokens/s × $17.5/1M ≈
-// $1.02, the model's fixed-length output. The token rates stay for the rare
-// interaction that completes synchronously with real counts. The Record is
-// keyed by GeminiOmniModel, so adding an Omni id without a price fails the build.
+// $1.50/1M input, $17.50/1M output, and a rendered second of video costs a
+// fixed number of output tokens per resolution (geminiOmniTokensPerSecond) —
+// but the render is async: the submit response carries no token counts, and
+// charging on the completing poll instead would let one balance launch
+// unbounded concurrent renders. So a clip charges at submit in units of one
+// second of 360p video (1,931 output tokens ≈ $0.034): the adapter counts the
+// units the requested length and resolution add up to (a 10s 720p clip is 30,
+// a 10s 4k clip 90), the preflight holds the balance to that count, and the
+// same count bills. The token rates stay for the rare interaction that
+// completes synchronously with real counts. The Record is keyed by
+// GeminiOmniModel, so adding an Omni id without a price fails the build.
const geminiOmniModelPricing: Record<GeminiOmniModel, ProviderCreditPricing> = {
[geminiOmniModels.flashVideo]: {
...textTokenPricing({
@@ -308,7 +326,8 @@ const geminiOmniModelPricing: Record<GeminiOmniModel, ProviderCreditPricing> = {
input: "1.5",
output: "17.5",
}),
- generationCostMicros: usdWithMargin("1.02"),
+ // One unit is a second of 360p: 1,931 output tokens at $17.50/1M.
+ generationCostMicros: usdWithMargin("0.033793"),
},
};
@@ -358,14 +377,17 @@ function geminiCreditPricing(model: string): ProviderCreditPricing | undefined {
// Exact match wins, so a specific id is never shadowed by a broader prefix entry (e.g. an image
// model "…-flash-image" under the "…-flash" text prefix). For dated/snapshot ids that only match
// by prefix, try the longest (most specific) key first for the same reason.
- const exact = geminiModelPricing[model as GeminiModel];
+ const exact =
+ geminiModelPricing[model as GeminiModel] ??
+ geminiTranscribeModelPricing[model as GeminiTranscribeModel];
if (exact) {
return exact;
}
- const byLongestPrefix = Object.entries(geminiModelPricing).sort(
- ([a], [b]) => b.length - a.length,
- );
+ const byLongestPrefix = [
+ ...Object.entries(geminiModelPricing),
+ ...Object.entries(geminiTranscribeModelPricing),
+ ].sort(([a], [b]) => b.length - a.length);
for (const [id, pricing] of byLongestPrefix) {
if (modelMatches(model, id)) {
return pricing;
diff --git a/site/src/lib/inference/adapters/gemini-responses.ts b/site/src/lib/inference/adapters/gemini-responses.ts
index 3e1e231f..a2a99d38 100644
--- a/site/src/lib/inference/adapters/gemini-responses.ts
+++ b/site/src/lib/inference/adapters/gemini-responses.ts
@@ -671,25 +671,17 @@ function thinkingLevelFromBody(body: JsonObject): ThinkingLevel | undefined {
}
function generationConfigFromBody(body: JsonObject): Partial<GenerateContentConfig> {
+ // Sampling knobs (temperature, top_p, top_k) are not forwarded: the Gemini
+ // 3.x line ignores them, and the models are tuned for their defaults.
const config: Partial<GenerateContentConfig> = {};
- const temperature = numberValue(body.temperature);
- if (temperature !== undefined) {
- config.temperature = temperature;
- }
-
- const topP = numberValue(body.top_p) ?? numberValue(body.topP);
- if (topP !== undefined) {
- config.topP = topP;
- }
-
const maxOutputTokens = numberValue(body.max_output_tokens) ?? numberValue(body.maxOutputTokens);
if (maxOutputTokens !== undefined) {
config.maxOutputTokens = maxOutputTokens;
}
// Bound reasoning so thinking tokens (which count against maxOutputTokens) can't starve the
// structured output. Callers driving tight per-turn loops pass a small budget; 0 disables thinking.
- // Gemini 3.x models (e.g. gemini-3.7-flash) take thinking_level (minimal|low|medium|high), NOT the
- // integer thinking_budget — passing the budget to them is silently ignored. Prefer the level when the
+ // Gemini 3.x models (e.g. gemini-3.8-flash) take thinking_level (low|medium|high; minimal on the
+ // models that still accept it); the integer thinking_budget is silently ignored there. Prefer the level when the
// caller sets it; the two are mutually exclusive. Older 2.x models still use thinking_budget. In both
// cases request the thought summary so callers can persist the reasoning (the normalized response
// separates it from output_text).
diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.ts
index 3a33e636..238d513d 100644
--- a/site/src/lib/inference/gemini-models.ts
+++ b/site/src/lib/inference/gemini-models.ts
@@ -8,11 +8,11 @@
// Canonical, dated-or-versioned model IDs. Add a new constant when adopting a
// new model; do not inline raw version strings at call sites.
export const geminiModels = {
- flash: "gemini-3.7-flash",
+ flash: "gemini-3.8-flash",
flashLite: "gemini-3.1-flash-lite",
- // Generative image editing/generation ("nano banana"). Bump here when adopting a
- // newer image model.
- flashImage: "gemini-2.5-flash-image",
+ // Generative image editing/generation ("nano banana 2"). Bump here when
+ // adopting a newer image model.
+ flashImage: "gemini-3.1-flash-image",
// "Nano banana pro": higher-fidelity image editing/generation that takes a real
// aspectRatio + imageSize (1K/2K/4K) via imageConfig. GA since 2026-05-28 (the
// -preview id was shut down 2026-06-25); served only on Vertex's global
@@ -22,13 +22,13 @@ export const geminiModels = {
export type GeminiModel = (typeof geminiModels)[keyof typeof geminiModels];
-// Unified video generation (Gemini Omni Flash): one Interactions API call takes
-// text plus optional seed/reference images and renders up to ~10s of 720p video
-// with audio. The clip length is the model's call — there is no duration knob —
-// and image output is not enabled for this model. Bump here when adopting a
-// newer Omni.
+// Unified video generation (Gemini Omni 1.1 Flash): one Interactions API call
+// takes text plus optional seed/reference images and renders a clip with audio.
+// The request names the resolution and may name the length; with no length the
+// model picks (up to ~10s). Image output is not enabled for this model. Bump
+// here when adopting a newer Omni.
export const geminiOmniModels = {
- flashVideo: "gemini-omni-flash-preview",
+ flashVideo: "gemini-omni-1.1-flash-preview",
} as const;
export type GeminiOmniModel = (typeof geminiOmniModels)[keyof typeof geminiOmniModels];
@@ -37,6 +37,35 @@ export type GeminiOmniModel = (typeof geminiOmniModels)[keyof typeof geminiOmniM
// client registry (videoModels.ts) reads the same number.
export const geminiOmniMaxReferenceImages = 3;
+// Output sizes an Omni render can be asked for. 720p is native; 1080p and 4k
+// are upscaled by the model. The panel, the chat tool schema, and the adapter
+// all read this list.
+export const geminiOmniResolutions = ["360p", "720p", "1080p", "4k"] as const;
+
+export type GeminiOmniResolution = (typeof geminiOmniResolutions)[number];
+
+export const geminiOmniDefaultResolution: GeminiOmniResolution = "720p";
+
+// Clip lengths an Omni render can be asked for, in whole seconds. A request
+// with no length lets the model pick, which lands at the top of this range.
+export const geminiOmniDurationSeconds = { min: 4, max: 10 } as const;
+
+// Output tokens per rendered second of video with audio, by resolution — the
+// unit Omni bills in. A render is charged at submit from the length and
+// resolution it asked for (provider-pricing.ts), since the async submit
+// returns no counts of its own.
+export const geminiOmniTokensPerSecond: Record<GeminiOmniResolution, number> = {
+ "360p": 1_931,
+ "720p": 5_792,
+ "1080p": 8_688,
+ "4k": 17_376,
+};
+
+/** The billing unit for a video render: one second of 360p output. Every other
+ * size bills as its multiple of this, so the credit preflight and the charge
+ * count the same thing. */
+export const geminiOmniUnitTokens = geminiOmniTokensPerSecond["360p"];
+
// Generative speech (Gemini TTS): text in, spoken audio out, with prompt-driven
// style direction and inline audio tags. Bump here when adopting a newer TTS model.
export const geminiTtsModels = {
@@ -45,6 +74,17 @@ export const geminiTtsModels = {
export type GeminiTtsModel = (typeof geminiTtsModels)[keyof typeof geminiTtsModels];
+// Speech-to-text (Gemini Transcribe): a recorded audio file in, the transcript
+// with per-word timings out, the language detected by the model. Runs through
+// generateContent with an audio transcription config; it takes no prompt,
+// system instruction, or output schema. Bump here when adopting a newer one.
+export const geminiTranscribeModels = {
+ file: "gemini-3.5-transcribe-preview",
+} as const;
+
+export type GeminiTranscribeModel =
+ (typeof geminiTranscribeModels)[keyof typeof geminiTranscribeModels];
+
// Generative music (Gemini/Lyria) over the Interactions API: a text prompt in,
// an instrumental clip out — the background bed for a cut. `clip` renders a
// fixed ~30s clip; `pro` a full-length (~2min) track. These are the legacy
@@ -73,4 +113,6 @@ export const geminiModelRoles = {
// against the plan, sheets, and benchmarks. Runs the strongest multimodal
// judge we serve — bump here to upgrade every review gate at once.
review: geminiModels.flash,
+ // Hosted speech-to-text: the recorded mix in, word-timed cues out.
+ transcription: geminiTranscribeModels.file,
} as const;