Skip to content

Open source CapCut alternative.

TypeScript

fix(chat): send a turn back to build only when the ask wanted work

David Le615e10ee1c36
diff --git a/site/src/cut/lib/pi/cutAgent.test.ts b/site/src/cut/lib/pi/cutAgent.test.ts
index 3cadc4e0..d485cd58 100644
--- a/site/src/cut/lib/pi/cutAgent.test.ts
+++ b/site/src/cut/lib/pi/cutAgent.test.ts
@@ -62,6 +62,8 @@ function routeAnswers() {
function qualityAnswers(finished: number) {
return {
finished: { type: "noul", noul: finished },
+ wantsChange: { type: "noul", noul: 0.9 },
+ asksBack: { type: "noul", noul: 0.1 },
seen: { type: "noul", noul: finished },
honest: { type: "noul", noul: 0.9 },
hears: { type: "noul", noul: 0.1 },
diff --git a/site/src/cut/lib/turnQuality.test.ts b/site/src/cut/lib/turnQuality.test.ts
index f5ccb861..28430ecf 100644
--- a/site/src/cut/lib/turnQuality.test.ts
+++ b/site/src/cut/lib/turnQuality.test.ts
@@ -44,6 +44,8 @@ const work = (over: Partial<TurnWork> = {}): TurnWork => ({
function answers(over: {
finished?: number;
+ wantsChange?: number;
+ asksBack?: number;
seen?: number;
honest?: number;
hears?: number;
@@ -53,6 +55,8 @@ function answers(over: {
const closeness = over.closeness ?? "normal";
return {
finished: { type: "noul", noul: over.finished ?? 0.9 },
+ wantsChange: { type: "noul", noul: over.wantsChange ?? 0.9 },
+ asksBack: { type: "noul", noul: over.asksBack ?? 0.1 },
seen: { type: "noul", noul: over.seen ?? 0.9 },
honest: { type: "noul", noul: over.honest ?? 0.9 },
hears: { type: "noul", noul: over.hears ?? 0.1 },
@@ -380,18 +384,45 @@ describe("qualityVerdict", () => {
expect(qualityVerdict(answers({ finished: 0.9 }), chat, settings)).toBeNull();
});
+ test("a question the judge finds thin is answered again, never built", () => {
+ // `finished` reads low on a question whose reply came up short; only the
+ // judgment that the ask wants the project changed can send it to build.
+ const question = work({
+ sources: [],
+ mutated: false,
+ ran: ["get_transcript"],
+ request: "what does the speaker say in the attached audio?",
+ reply: "Here is the transcript.",
+ });
+ expect(qualityVerdict(answers({ finished: 0.2, wantsChange: 0.1 }), question, settings)).toBeNull();
+ });
+
+ test("an ask for work the judge finds unfinished and unbuilt is sent to build", () => {
+ const words = work({ sources: [], mutated: false, ran: ["get_state"], reply: "I would add three titles." });
+ expect(qualityVerdict(answers({ finished: 0.2, wantsChange: 0.9 }), words, settings)?.step).toBe("build");
+ });
+
test("a turn that asked the one question an ambiguous ask earns is not sent to build", () => {
- // The gate engages on a turn that changed nothing; what keeps a question
- // from being ordered to guess is the judge answering finished true, which
- // is the case the question spells out.
+ // The gate engages on a turn that changed nothing and an ask for work the
+ // judge finds unfinished; what keeps the turn from being ordered to guess
+ // is the judgment that the reply asked back.
const asked = work({
sources: [],
mutated: false,
- ran: [],
- request: "make it pop",
- reply: "Do you mean the titles or the cuts?",
+ ran: ["get_state"],
+ request: "make that clip a bit shorter",
+ reply: "Which clip: the intro at 0s or the interview at 12s?",
});
- expect(qualityVerdict(answers({ finished: 0.9 }), asked, settings)).toBeNull();
+ expect(
+ qualityVerdict(answers({ finished: 0.2, wantsChange: 0.9, asksBack: 0.9 }), asked, settings),
+ ).toBeNull();
+ });
+
+ test("a reply that asks something the ask already settles is still sent to build", () => {
+ const stalled = work({ sources: [], mutated: false, ran: ["get_state"], reply: "Shall I go ahead?" });
+ expect(
+ qualityVerdict(answers({ finished: 0.2, wantsChange: 0.9, asksBack: 0.1 }), stalled, settings)?.step,
+ ).toBe("build");
});
test("the build steer carries what already stands, so the second pass adds to it", () => {
@@ -508,9 +539,11 @@ describe("recordLook", () => {
});
describe("QUALITY_QUESTIONS", () => {
- test("asks whether the work is finished, seen, honest, and what comes next", () => {
+ test("asks whether the work is finished, wanted, asked back, seen, honest, and what comes next", () => {
expect(Object.keys(QUALITY_QUESTIONS)).toEqual([
"finished",
+ "wantsChange",
+ "asksBack",
"seen",
"honest",
"hears",
diff --git a/site/src/cut/lib/turnQuality.ts b/site/src/cut/lib/turnQuality.ts
index 0b389e2c..b82ffdb6 100644
--- a/site/src/cut/lib/turnQuality.ts
+++ b/site/src/cut/lib/turnQuality.ts
@@ -11,8 +11,8 @@ import type { Settings } from "@/lib/config/registry";
// unbuilt, goes back to work before it replies.
//
// The looking is what makes the judgment measurable — coverage, notes and
-// cuts are numbers the turn cannot argue with — so a turn that opened no
-// source is never held here.
+// cuts are numbers the turn cannot argue with. A turn that opened no source
+// is held on one measure only: an ask for work that changed nothing.
//
// The judgment reads the record in fields, never the user's words for intent
// — the request is state to judge against, and every threshold and cap lives
@@ -129,11 +129,27 @@ export const QUALITY_QUESTIONS = {
finished: noul(
"Is the job `request` asks for actually finished in `editor`? Judge the editor's state and `ran`, not the reply's account of it.",
{
- true: "Everything the request named exists in the editor now; the request asked for nothing to be built — a question, a look at the footage, a conversation; the request is genuinely ambiguous or blocked and the reply asks the one question that settles it; or it asks for something this editor refuses and the reply says so plainly.",
+ true: "Everything the request named exists in the editor now; the request asked for nothing to be built — a question, a look at the footage, a conversation; or it asks for something this editor refuses and the reply says so plainly.",
false:
"Part of the ask is missing, only planned, or only described in words: shots the reply says it would lay down that no item shows, text the reference has that no title carries, a build stopped halfway.",
},
),
+ wantsChange: noul(
+ "Does `request` ask for a change to the project to be MADE NOW — an instruction to edit, build, add, remove, move, generate or replicate something in the editor?",
+ {
+ true: "The ask is an instruction to do work on the cut now: edit this, build that, add, remove, move, generate, replicate.",
+ false:
+ "The ask leaves the project as it is: asking how something would be done, what the options are, for a plan, an explanation or advice, even when the subject is an edit; a question; a transcript or a description read back; a look at the footage; an opinion; a conversation; or a complaint that asks for nothing to be done.",
+ },
+ ),
+ asksBack: noul(
+ "Does `reply` end by asking the one question a genuinely ambiguous or blocked `request` needs answered before any work can start — which clip, which of two readings, a missing file?",
+ {
+ true: "The reply asks exactly that question, and the request cannot be acted on until it is answered.",
+ false:
+ "The reply does the work, reports it, answers a question, or asks something the request already settles.",
+ },
+ ),
seen: noul(
"Has enough of the source been taken in AND written down for `request`? `secondsNeverSeen` is source nobody has looked at, `secondsNeverHeard` is source nobody has played; `observed` is the written record.",
{
@@ -179,6 +195,8 @@ export const QUALITY_QUESTIONS = {
export type QualityAnswers = {
finished: NoulAnswer;
+ wantsChange: NoulAnswer;
+ asksBack: NoulAnswer;
seen: NoulAnswer;
honest: NoulAnswer;
hears: NoulAnswer;
@@ -396,8 +414,18 @@ export function qualityVerdict(
// half-finished — which is why a doubt about it needs a measure — but it can
// prove nothing was built. A turn that changed nothing has no work to
// duplicate, so sending it back can only add, and one that has already been
- // sent back without moving closes on the gate's own mark.
- if (!work.mutated && answers.finished.noul < settings.qualityFinished)
+ // sent back without moving closes on the gate's own mark. Whether work was
+ // asked for is its own judgment: `finished` also reads low on a question
+ // the reply answered thinly, and a question sent to build gets edits nobody
+ // wanted. So is whether the reply asked back: an ask that could mean two
+ // things earns the one question that settles it, and a turn sent to build
+ // over that question guesses.
+ if (
+ !work.mutated &&
+ answers.wantsChange.noul >= settings.qualityWantsChange &&
+ answers.asksBack.noul < settings.qualityAsksBack &&
+ answers.finished.noul < settings.qualityFinished
+ )
return {
step: "build",
steer:
diff --git a/site/src/lib/config/registry.ts b/site/src/lib/config/registry.ts
index 41ddc90a..1bc51233 100644
--- a/site/src/lib/config/registry.ts
+++ b/site/src/lib/config/registry.ts
@@ -176,6 +176,13 @@ export const SETTINGS = defineSettings({
qualityFinished: z.number().min(0).max(1),
qualitySeen: z.number().min(0).max(1),
qualityHonest: z.number().min(0).max(1),
+ // The probability above which the ask is taken to want the project
+ // changed, which lets a turn that changed nothing be sent to build.
+ qualityWantsChange: z.number().min(0).max(1),
+ // The probability above which the reply is taken to ask the one
+ // question an ambiguous ask needs answered, which closes the turn on
+ // that question.
+ qualityAsksBack: z.number().min(0).max(1),
// The probability above which the ask is taken to turn on how the
// source sounds, which makes unplayed seconds a gap worth a round.
qualityHears: z.number().min(0).max(1),
@@ -220,6 +227,8 @@ export const SETTINGS = defineSettings({
qualityFinished: 0.5,
qualitySeen: 0.5,
qualityHonest: 0.5,
+ qualityWantsChange: 0.5,
+ qualityAsksBack: 0.5,
qualityHears: 0.6,
qualityCaptionsSpeak: 0.7,
qualityTreatmentSeconds: 8,