Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: head a decision score with its most likely level (#13111)

Leo Borcherding8e3fd25af028
diff --git a/studio/frontend/src/features/settings/components/decision-try-dialog.tsx b/studio/frontend/src/features/settings/components/decision-try-dialog.tsx
index 9929edff5c9..9128d554b7b 100644
--- a/studio/frontend/src/features/settings/components/decision-try-dialog.tsx
+++ b/studio/frontend/src/features/settings/components/decision-try-dialog.tsx
@@ -601,11 +601,11 @@ function AnswerPreview({
</div>
);
}
- const { keys, nearest, max } = scoreLevels(answer);
+ const { keys, top, max } = scoreLevels(answer);
return (
<div className="flex flex-col gap-6">
<AnswerHeading
- answer={criterionText(answer.legend[nearest]) ?? nearest}
+ answer={criterionText(answer.legend[top]) ?? top}
detail={t("decisions.scoreDetail", {
score: answer.score.toFixed(2),
max,
@@ -618,7 +618,7 @@ function AnswerPreview({
index={key}
label={criterionText(answer.legend[key]) ?? key}
value={answer.probabilities[key] ?? 0}
- top={key === nearest}
+ top={key === top}
percent={percent}
/>
))}
diff --git a/studio/frontend/src/features/settings/lib/decision-request.ts b/studio/frontend/src/features/settings/lib/decision-request.ts
index 2951621989f..e59cb18b1ef 100644
--- a/studio/frontend/src/features/settings/lib/decision-request.ts
+++ b/studio/frontend/src/features/settings/lib/decision-request.ts
@@ -375,12 +375,18 @@ export function choiceCriterion(
export function scoreLevels(
answer: Extract<DecisionAnswer, { type: "score" }>,
-): { keys: string[]; nearest: string; max: number } {
+): { keys: string[]; top: string; max: number } {
const keys = Object.keys(answer.legend).sort((a, b) => Number(a) - Number(b));
const distance = (key: string) => Math.abs(Number(key) - answer.score);
const nearest = keys.reduce(
(best, key) => (distance(key) < distance(best) ? key : best),
keys[0] ?? "0",
);
- return { keys, nearest, max: Number(keys.at(-1) ?? 0) };
+ // The score is an expectation, so a split answer's nearest level can be one the model barely picked.
+ const p = (key: string) => answer.probabilities[key] ?? -1;
+ const top = keys.reduce(
+ (best, key) => (p(key) > p(best) ? key : best),
+ nearest,
+ );
+ return { keys, top, max: Number(keys.at(-1) ?? 0) };
}
diff --git a/studio/frontend/tests/decisions-request.test.ts b/studio/frontend/tests/decisions-request.test.ts
index 4ae06e37727..82098e0edd2 100644
--- a/studio/frontend/tests/decisions-request.test.ts
+++ b/studio/frontend/tests/decisions-request.test.ts
@@ -165,7 +165,7 @@ test("every type starts from its first preset and every preset is a valid reques
}
});
-test("a score lands on the nearest level of its legend", () => {
+test("a score with no probabilities lands on the nearest level of its legend", () => {
const answer = {
type: "score" as const,
score: 2.6,
@@ -175,10 +175,32 @@ test("a score lands on the nearest level of its legend", () => {
};
assert.deepEqual(scoreLevels(answer), {
keys: ["0", "1", "2", "10"],
- nearest: "2",
+ top: "2",
max: 10,
});
- assert.equal(scoreLevels({ ...answer, score: -1 }).nearest, "0");
+ assert.equal(scoreLevels({ ...answer, score: -1 }).top, "0");
+});
+
+test("a split score heads with its most likely level, not the one nearest the expectation", () => {
+ // Measured in Unsloth Desktop 0.1.905: the 429 grading example came back like this.
+ const answer = {
+ type: "score" as const,
+ score: 0.5032,
+ confidence: 0.6559,
+ legend: { "0": "wrong", "1": "touches", "2": "partial", "3": "correct" },
+ probabilities: { "0": 0.8303, "1": 0.0021, "2": 0.0018, "3": 0.1658 },
+ };
+ assert.equal(scoreLevels(answer).top, "0");
+ const mirrored = { "0": 0.1658, "1": 0.0018, "2": 0.0021, "3": 0.8303 };
+ assert.equal(
+ scoreLevels({ ...answer, score: 2.4968, probabilities: mirrored }).top,
+ "3",
+ );
+ const flat = { "0": 0.25, "1": 0.25, "2": 0.25, "3": 0.25 };
+ assert.equal(
+ scoreLevels({ ...answer, score: 1.5, probabilities: flat }).top,
+ "1",
+ );
});
test("Try it in Settings > API opens the decision playground", () => {