Skip to content

Open source CapCut alternative.

TypeScript

fix(site): judged choices commit on the winner's own probability [rebuild]

David Le1793fee6ed29
diff --git a/site/src/cut/lib/providerFailure.test.ts b/site/src/cut/lib/providerFailure.test.ts
new file mode 100644
index 00000000..ce1d55cc
--- /dev/null
+++ b/site/src/cut/lib/providerFailure.test.ts
@@ -0,0 +1,34 @@
+import { describe, expect, test } from "bun:test";
+import { providerFailureClass, type ProviderFailureClass } from "./providerFailure";
+
+const answer = (choice: ProviderFailureClass, p: number) => ({
+ choice,
+ probabilities: {
+ content_refused: choice === "content_refused" ? p : (1 - p) / 2,
+ transient_failure: choice === "transient_failure" ? p : (1 - p) / 2,
+ other: choice === "other" ? p : (1 - p) / 2,
+ },
+});
+
+describe("providerFailureClass", () => {
+ test("a class that carries the probability is acted on", () => {
+ expect(providerFailureClass(answer("content_refused", 0.82))).toBe("content_refused");
+ expect(providerFailureClass(answer("transient_failure", 0.61))).toBe("transient_failure");
+ });
+
+ test("a winner the judgment spreads thin reads as other", () => {
+ expect(providerFailureClass(answer("content_refused", 0.41))).toBe("other");
+ });
+
+ test("the floor reads the winner's own probability, never the spread", () => {
+ // A clear pick over three options concentrates the distribution loosely:
+ // the same answer that reads 0.34 confidence carries 0.8 on its class,
+ // and the refusal has to survive so the text-only rung still runs.
+ const clear = { choice: "content_refused" as const, probabilities: { content_refused: 0.8, transient_failure: 0.12, other: 0.08 }, confidence: 0.34 };
+ expect(providerFailureClass(clear)).toBe("content_refused");
+ });
+
+ test("an answer naming a class the distribution does not carry reads as other", () => {
+ expect(providerFailureClass({ choice: "content_refused", probabilities: {} })).toBe("other");
+ });
+});
diff --git a/site/src/cut/lib/providerFailure.ts b/site/src/cut/lib/providerFailure.ts
index 3fa708c7..7b3d313f 100644
--- a/site/src/cut/lib/providerFailure.ts
+++ b/site/src/cut/lib/providerFailure.ts
@@ -19,16 +19,16 @@ export const PROVIDER_FAILURE_QUESTION = choice(
},
);
-/** The class a judgment cannot commit to. */
-const CONFIDENCE_FLOOR = 0.5;
+/** The probability the named class has to carry to be acted on. */
+const COMMIT_FLOOR = 0.5;
-/** Classify a failure's cause from the judge's answer; below the confidence
- * floor it reads as "other". */
+/** Classify a failure's cause from the judge's answer. The class commits on
+ * its own probability; a winner the judgment spreads thin reads as "other". */
export function providerFailureClass(answer: {
choice: ProviderFailureClass;
- confidence: number;
+ probabilities: Record<string, number>;
}): ProviderFailureClass {
- return answer.confidence >= CONFIDENCE_FLOOR ? answer.choice : "other";
+ return (answer.probabilities[answer.choice] ?? 0) >= COMMIT_FLOOR ? answer.choice : "other";
}
/** The failure class of a provider error, judged through the hosted judge.
diff --git a/site/src/cut/lib/tts.ts b/site/src/cut/lib/tts.ts
index ba82ec24..44c2e534 100644
--- a/site/src/cut/lib/tts.ts
+++ b/site/src/cut/lib/tts.ts
@@ -43,7 +43,11 @@ export async function resolveVoiceAsk(wanted?: string): Promise<string> {
{ voice: VOICE_QUESTION },
);
const pick = answers.voice;
- return pick.confidence >= cutJudge().voicePick && SPEECH_VOICES.some((v) => v.id === pick.choice)
+ // The winner's own probability, not the spread of the distribution: over
+ // twenty voices every plausible match takes a share, so a correct pick
+ // reads as low confidence.
+ const p = pick.probabilities[pick.choice] ?? 0;
+ return p >= cutJudge().voicePick && SPEECH_VOICES.some((v) => v.id === pick.choice)
? pick.choice
: DEFAULT_VOICE;
} catch {