Skip to content

Open source CapCut alternative.

TypeScript

test(site): the chat eval pins which turns go instant

David Lef80e0c1573d3
diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.json
index 23970b4a..2b9b3ecb 100644
--- a/evals/cut-chat.latest-report.json
+++ b/evals/cut-chat.latest-report.json
@@ -1,8 +1,8 @@
{
"schema": "cut-chat-eval/v1",
- "generatedAt": "2026-09-17T12:17:25.851Z",
+ "generatedAt": "2026-09-18T03:44:43.038Z",
"base": "http://localhost:3000",
- "runsPerCase": 2,
+ "runsPerCase": 1,
"configs": [
{
"label": "config",
@@ -12,70 +12,70 @@
"cases": 9,
"passRate": 1,
"gateMs": {
- "p50": 251,
- "p95": 304,
- "mean": 249,
+ "p50": 529,
+ "p95": 742,
+ "mean": 504,
"n": 9
},
"ttftMs": {
- "p50": 2728,
- "p95": 6411,
- "mean": 3041,
+ "p50": 2656,
+ "p95": 5759,
+ "mean": 3124,
"n": 9
},
"totalMs": {
- "p50": 3280,
- "p95": 6511,
- "mean": 3372,
+ "p50": 3768,
+ "p95": 5943,
+ "mean": 3533,
"n": 9
},
- "meanRounds": 1.5
+ "meanRounds": 1
},
"multi-tool": {
"cases": 9,
- "passRate": 1,
+ "passRate": 0.89,
"gateMs": {
- "p50": 245,
- "p95": 301,
- "mean": 252,
- "n": 9
+ "p50": 476,
+ "p95": 580,
+ "mean": 461,
+ "n": 8
},
"ttftMs": {
- "p50": 5792,
- "p95": 32285,
- "mean": 10035,
- "n": 9
+ "p50": 5673,
+ "p95": 28945,
+ "mean": 9066,
+ "n": 8
},
"totalMs": {
- "p50": 6118,
- "p95": 32550,
- "mean": 10335,
- "n": 9
+ "p50": 5895,
+ "p95": 29504,
+ "mean": 9412,
+ "n": 8
},
- "meanRounds": 4.8
+ "meanRounds": 3
},
"single-tool": {
"cases": 24,
- "passRate": 0.94,
+ "passRate": 0.96,
"gateMs": {
- "p50": 255,
- "p95": 319,
- "mean": 289,
- "n": 24
+ "p50": 452,
+ "p95": 620,
+ "mean": 444,
+ "n": 23
},
"ttftMs": {
- "p50": 3166,
- "p95": 11762,
- "mean": 5085,
- "n": 24
+ "p50": 4551,
+ "p95": 19592,
+ "mean": 5992,
+ "n": 23
},
"totalMs": {
- "p50": 3433,
- "p95": 12041,
- "mean": 5332,
- "n": 24
+ "p50": 4756,
+ "p95": 19752,
+ "mean": 6183,
+ "n": 23
},
- "meanRounds": 2.6
+ "meanRounds": 2.3
}
},
"cases": [
@@ -86,34 +86,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 304,
- "p95": 812,
- "mean": 558,
- "n": 2
+ "p50": 742,
+ "p95": 742,
+ "mean": 742,
+ "n": 1
},
"ttftMs": {
- "p50": 2728,
- "p95": 3610,
- "mean": 3169,
- "n": 2
+ "p50": 3664,
+ "p95": 3664,
+ "mean": 3664,
+ "n": 1
},
"totalMs": {
- "p50": 2928,
- "p95": 3713,
- "mean": 3320,
- "n": 2
+ "p50": 3768,
+ "p95": 3768,
+ "mean": 3768,
+ "n": 1
},
"modelMs": {
- "p50": 2622,
- "p95": 2891,
- "mean": 2756,
- "n": 2
+ "p50": 3017,
+ "p95": 3017,
+ "mean": 3017,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 1,
@@ -121,44 +121,6 @@
}
},
"runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": null,
- "areas": [
- "timeline",
- "inspector",
- "subtitles",
- "audio",
- "library",
- "side_panel",
- "editor"
- ],
- "declaredTools": 91,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 811.7028329999999,
- "gateSkipped": true,
- "ttftMs": 3609.8061660000003,
- "modelMs": 2891.1585,
- "toolMs": 0,
- "totalMs": 3712.7595410000004,
- "judgeMs": 637.5777499999995,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2891.1585,
- "firstDeltaMs": 2789.6068339999997,
- "calls": []
- }
- ]
- }
- },
{
"pass": true,
"intent": "complex",
@@ -175,22 +137,23 @@
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 304.1766669999997,
+ "gateMs": 742.2675000000002,
"gateSkipped": true,
- "ttftMs": 2727.847667,
- "modelMs": 2621.7646250000007,
+ "ttftMs": 3663.9621669999997,
+ "modelMs": 3016.7649169999995,
"toolMs": 0,
- "totalMs": 2927.532292000001,
- "judgeMs": 206.19129099999918,
+ "totalMs": 3768.323375,
+ "judgeMs": 228.9335000000001,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2621.7646250000007,
- "firstDeltaMs": 2422.3260420000006,
+ "roundMs": 3016.7649169999995,
+ "firstDeltaMs": 2913.461125,
"calls": []
}
]
@@ -205,38 +168,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 263,
- "p95": 316,
- "mean": 290,
- "n": 2
+ "p50": 529,
+ "p95": 529,
+ "mean": 529,
+ "n": 1
},
"ttftMs": {
- "p50": 4319,
- "p95": 30883,
- "mean": 17601,
- "n": 2
+ "p50": 2484,
+ "p95": 2484,
+ "mean": 2484,
+ "n": 1
},
"totalMs": {
- "p50": 4319,
- "p95": 31109,
- "mean": 17714,
- "n": 2
+ "p50": 2533,
+ "p95": 2533,
+ "mean": 2533,
+ "n": 1
},
"modelMs": {
- "p50": 4053,
- "p95": 30789,
- "mean": 17421,
- "n": 2
+ "p50": 2002,
+ "p95": 2002,
+ "mean": 2002,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
@@ -249,62 +212,23 @@
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 316.32133299999987,
- "gateSkipped": false,
- "ttftMs": 30882.879916,
- "modelMs": 30789.048667000003,
- "toolMs": 0,
- "totalMs": 31108.895458000003,
- "judgeMs": 538.7172090000022,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3538.2138749999995,
- "firstDeltaMs": 3307.114125,
- "calls": []
- },
- {
- "roundMs": 27250.834792,
- "firstDeltaMs": 27026.010374999998,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "skill": null,
- "areas": [],
- "declaredTools": 0,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 263.0132920000033,
+ "gateMs": 528.8679589999992,
"gateSkipped": false,
- "ttftMs": 4318.823958000001,
- "modelMs": 4053.2067920000045,
+ "ttftMs": 2483.8594999999996,
+ "modelMs": 2002.343833,
"toolMs": 0,
- "totalMs": 4319.161749999999,
- "judgeMs": 606.3485000000001,
+ "totalMs": 2533.1364169999997,
+ "judgeMs": 346.0425000000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1134.1152920000022,
- "firstDeltaMs": 1008.7466249999998,
- "calls": []
- },
- {
- "roundMs": 2919.0915000000023,
- "firstDeltaMs": 2918.957416999998,
+ "roundMs": 2002.343833,
+ "firstDeltaMs": 1953.405417,
"calls": []
}
]
@@ -319,38 +243,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 251,
- "p95": 282,
- "mean": 266,
- "n": 2
+ "p50": 376,
+ "p95": 376,
+ "mean": 376,
+ "n": 1
},
"ttftMs": {
- "p50": 4282,
- "p95": 4923,
- "mean": 4603,
- "n": 2
+ "p50": 4685,
+ "p95": 4685,
+ "mean": 4685,
+ "n": 1
},
"totalMs": {
- "p50": 4458,
- "p95": 4972,
- "mean": 4715,
- "n": 2
+ "p50": 4741,
+ "p95": 4741,
+ "mean": 4741,
+ "n": 1
},
"modelMs": {
- "p50": 4206,
- "p95": 4677,
- "mean": 4441,
- "n": 2
+ "p50": 4363,
+ "p95": 4363,
+ "mean": 4363,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
@@ -363,62 +287,23 @@
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 281.59308299999975,
- "gateSkipped": false,
- "ttftMs": 4922.769625000001,
- "modelMs": 4676.804083000003,
- "toolMs": 0,
- "totalMs": 4972.416833000003,
- "judgeMs": 489.70624999999563,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1204.658457999998,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3472.1456250000047,
- "firstDeltaMs": 3422.9637500000026,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "skill": null,
- "areas": [],
- "declaredTools": 0,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 251.05025000000023,
+ "gateMs": 375.9370420000005,
"gateSkipped": false,
- "ttftMs": 4282.4438329999975,
- "modelMs": 4205.652916999999,
+ "ttftMs": 4684.84525,
+ "modelMs": 4363.426416,
"toolMs": 0,
- "totalMs": 4458.468416999996,
- "judgeMs": 1035.0701249999984,
+ "totalMs": 4740.879457999999,
+ "judgeMs": 600.5082500000008,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1077.3654590000006,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3128.287457999999,
- "firstDeltaMs": 2952.6718330000003,
+ "roundMs": 4363.426416,
+ "firstDeltaMs": 4307.648416,
"calls": []
}
]
@@ -433,34 +318,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 291,
- "p95": 332,
- "mean": 311,
- "n": 2
+ "p50": 447,
+ "p95": 447,
+ "mean": 447,
+ "n": 1
},
"ttftMs": {
- "p50": 2914,
- "p95": 3019,
- "mean": 2967,
- "n": 2
+ "p50": 3329,
+ "p95": 3329,
+ "mean": 3329,
+ "n": 1
},
"totalMs": {
- "p50": 3090,
- "p95": 3278,
- "mean": 3184,
- "n": 2
+ "p50": 3512,
+ "p95": 3512,
+ "mean": 3512,
+ "n": 1
},
"modelMs": {
- "p50": 2795,
- "p95": 2941,
- "mean": 2868,
- "n": 2
+ "p50": 3050,
+ "p95": 3050,
+ "mean": 3050,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -480,136 +365,38 @@
"library",
"editor"
],
- "declaredTools": 85,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1818.1259169999976,
- "execMs": 0.28608299999905284,
- "args": {
- "text": "so today we're",
- "id": "cue1"
- }
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1818.9567919999972,
- "execMs": 0.004207999998470768,
- "args": {
- "id": "cue3",
- "text": "and they absolutely love the water"
- }
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 1819.2722079999949,
- "execMs": 0.02195899999787798,
- "args": {
- "id": "cue4"
- }
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1819.3593330000003,
- "execMs": 0.002542000001994893,
- "args": {
- "text": "so let's watch them play",
- "id": "cue5"
- }
- }
- ],
- "timings": {
- "gateMs": 331.9095410000009,
- "gateSkipped": false,
- "ttftMs": 3018.7580420000013,
- "modelMs": 2940.6652909999975,
- "toolMs": 0.3147919999973965,
- "totalMs": 3277.793249999995,
- "judgeMs": 304.1790420000034,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1483.0979999999981,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.28608299999905284
- },
- {
- "name": "update_cue",
- "execMs": 0.004207999998470768
- },
- {
- "name": "delete_cue",
- "execMs": 0.02195899999787798
- },
- {
- "name": "update_cue",
- "execMs": 0.002542000001994893
- }
- ]
- },
- {
- "roundMs": 1457.5672909999994,
- "firstDeltaMs": 1198.645958000001,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "subtitles",
- "library",
- "editor"
- ],
- "declaredTools": 85,
+ "declaredTools": 86,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "update_cue",
"round": 0,
- "atMs": 1847.401791999997,
- "execMs": 0.007083999997121282,
+ "atMs": 2172.778875,
+ "execMs": 0.29262500000004366,
"args": {
- "text": "So today we're",
- "id": "cue1"
+ "id": "cue1",
+ "text": "So today we're"
}
},
{
"name": "update_cue",
"round": 0,
- "atMs": 1847.5505419999972,
- "execMs": 0.00391700000182027,
+ "atMs": 2173.8610420000005,
+ "execMs": 0.005166999999346444,
"args": {
"id": "cue3",
- "text": "and it's great."
+ "text": "And..."
}
},
{
"name": "align_to_audio",
"round": 0,
- "atMs": 1848.802791999995,
- "execMs": 0.005500000006577466,
+ "atMs": 2174.714249999999,
+ "execMs": 0.003582999999707681,
"args": {
"target": "captions",
"track": 0
@@ -617,37 +404,37 @@
}
],
"timings": {
- "gateMs": 291.03254200000083,
+ "gateMs": 447.4327080000003,
"gateSkipped": false,
- "ttftMs": 2914.2705419999984,
- "modelMs": 2794.726251,
- "toolMs": 0.016501000005519018,
- "totalMs": 3090.3525420000005,
- "judgeMs": 351.53241700000217,
+ "ttftMs": 3328.927875000001,
+ "modelMs": 3049.699666999999,
+ "toolMs": 0.3013749999990978,
+ "totalMs": 3512.293834,
+ "judgeMs": 268.95491700000275,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1554.4014590000006,
+ "roundMs": 1714.9200419999997,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.007083999997121282
+ "execMs": 0.29262500000004366
},
{
"name": "update_cue",
- "execMs": 0.00391700000182027
+ "execMs": 0.005166999999346444
},
{
"name": "align_to_audio",
- "execMs": 0.005500000006577466
+ "execMs": 0.003582999999707681
}
]
},
{
- "roundMs": 1240.3247919999994,
- "firstDeltaMs": 1064.6275000000023,
+ "roundMs": 1334.7796249999992,
+ "firstDeltaMs": 1153.1853330000013,
"calls": []
}
]
@@ -662,34 +449,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 239,
- "p95": 253,
- "mean": 246,
- "n": 2
+ "p50": 697,
+ "p95": 697,
+ "mean": 697,
+ "n": 1
},
"ttftMs": {
- "p50": 1176,
- "p95": 1269,
- "mean": 1222,
- "n": 2
+ "p50": 1735,
+ "p95": 1735,
+ "mean": 1735,
+ "n": 1
},
"totalMs": {
- "p50": 1276,
- "p95": 1444,
- "mean": 1360,
- "n": 2
+ "p50": 1905,
+ "p95": 1905,
+ "mean": 1905,
+ "n": 1
},
"modelMs": {
- "p50": 1035,
- "p95": 1187,
- "mean": 1111,
- "n": 2
+ "p50": 1201,
+ "p95": 1201,
+ "mean": 1201,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 1,
@@ -702,60 +489,30 @@
"intent": "simple",
"skill": null,
"areas": [
- "timeline"
- ],
- "declaredTools": 44,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 253.24716699999408,
- "gateSkipped": false,
- "ttftMs": 1269.1092500000013,
- "modelMs": 1187.2047910000038,
- "toolMs": 0,
- "totalMs": 1444.1679999999978,
- "judgeMs": 244.8036659999998,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1187.2047910000038,
- "firstDeltaMs": 1012.2492910000001,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "timeline"
+ "timeline",
+ "top_bar"
],
- "declaredTools": 44,
+ "declaredTools": 49,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 238.75079099999857,
+ "gateMs": 697.3682919999992,
"gateSkipped": false,
- "ttftMs": 1175.5901660000018,
- "modelMs": 1035.3704170000055,
+ "ttftMs": 1735.3238330000022,
+ "modelMs": 1201.3779170000016,
"toolMs": 0,
- "totalMs": 1276.0852499999964,
- "judgeMs": 271.54412500000035,
+ "totalMs": 1905.153666000002,
+ "judgeMs": 242.05804200000057,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1035.3704170000055,
- "firstDeltaMs": 935.2327500000029,
+ "roundMs": 1201.3779170000016,
+ "firstDeltaMs": 1032.3630840000005,
"calls": []
}
]
@@ -770,34 +527,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 250,
- "p95": 274,
- "mean": 262,
- "n": 2
+ "p50": 324,
+ "p95": 324,
+ "mean": 324,
+ "n": 1
},
"ttftMs": {
- "p50": 5792,
- "p95": 7322,
- "mean": 6557,
- "n": 2
+ "p50": 28945,
+ "p95": 28945,
+ "mean": 28945,
+ "n": 1
},
"totalMs": {
- "p50": 6118,
- "p95": 7661,
- "mean": 6889,
- "n": 2
+ "p50": 29504,
+ "p95": 29504,
+ "mean": 29504,
+ "n": 1
},
"modelMs": {
- "p50": 5840,
- "p95": 7407,
- "mean": 6624,
- "n": 2
+ "p50": 29170,
+ "p95": 29170,
+ "mean": 29170,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -819,23 +576,23 @@
"audio",
"video_gen",
"scene",
- "stock",
"library",
"side_panel",
"top_bar",
"editor"
],
- "declaredTools": 112,
+ "declaredTools": 111,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "add_clip",
"round": 0,
- "atMs": 3654.9874589999963,
- "execMs": 0.09112500000628643,
+ "atMs": 23231.843917,
+ "execMs": 0.12508299999899464,
"args": {
"asset_id": "a-i1"
}
@@ -843,163 +600,83 @@
{
"name": "add_clip",
"round": 0,
- "atMs": 3655.246874999997,
- "execMs": 0.003874999994877726,
+ "atMs": 23232.451457999996,
+ "execMs": 0.00466700000106357,
"args": {
"asset_id": "a-i2"
}
}
],
"timings": {
- "gateMs": 274.00900000000547,
+ "gateMs": 323.57904100000087,
"gateSkipped": true,
- "ttftMs": 5791.829333999995,
- "modelMs": 5840.383915999992,
- "toolMs": 0.09500000000116415,
- "totalMs": 6118.179999999993,
- "judgeMs": 772.9698329999956,
+ "ttftMs": 28944.707083,
+ "modelMs": 29170.149917000006,
+ "toolMs": 0.1297500000000582,
+ "totalMs": 29504.305291999997,
+ "judgeMs": 501.3070000000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3378.5229580000014,
+ "roundMs": 22899.947292,
"firstDeltaMs": null,
"calls": [
{
"name": "add_clip",
- "execMs": 0.09112500000628643
+ "execMs": 0.12508299999899464
},
{
"name": "add_clip",
- "execMs": 0.003874999994877726
+ "execMs": 0.00466700000106357
}
]
},
{
- "roundMs": 2461.86095799999,
- "firstDeltaMs": 2135.6040829999984,
+ "roundMs": 6270.202625000005,
+ "firstDeltaMs": 5710.906500000005,
"calls": []
}
]
}
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "timeline-editing",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 112,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 5664.788457999995,
- "execMs": 0.009791000004042871,
- "args": {
- "asset_id": "a-i1",
- "index": 1
- }
- },
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 5665.031625000003,
- "execMs": 0.0037089999968884513,
- "args": {
- "index": 2,
- "asset_id": "a-i2"
- }
- }
- ],
- "timings": {
- "gateMs": 249.76866600000358,
- "gateSkipped": true,
- "ttftMs": 7321.556125000003,
- "modelMs": 7407.458249999996,
- "toolMs": 0.013500000000931323,
- "totalMs": 7660.595541000002,
- "judgeMs": 519.989167000007,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5413.399124999996,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.009791000004042871
- },
- {
- "name": "add_clip",
- "execMs": 0.0037089999968884513
- }
- ]
- },
- {
- "roundMs": 1994.0591249999998,
- "firstDeltaMs": 1655.4207499999902,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "photos-question-stays-in-chat",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 230,
- "p95": 254,
- "mean": 242,
- "n": 2
+ }
+ ]
+ },
+ {
+ "name": "photos-question-stays-in-chat",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 273,
+ "p95": 273,
+ "mean": 273,
+ "n": 1
},
"ttftMs": {
- "p50": 2151,
- "p95": 2347,
- "mean": 2249,
- "n": 2
+ "p50": 2656,
+ "p95": 2656,
+ "mean": 2656,
+ "n": 1
},
"totalMs": {
- "p50": 3280,
- "p95": 3711,
- "mean": 3495,
- "n": 2
+ "p50": 3898,
+ "p95": 3898,
+ "mean": 3898,
+ "n": 1
},
"modelMs": {
- "p50": 3046,
- "p95": 3453,
- "mean": 3250,
- "n": 2
+ "p50": 3621,
+ "p95": 3621,
+ "mean": 3621,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 1,
@@ -1028,71 +705,27 @@
"top_bar",
"editor"
],
- "declaredTools": 112,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "concise factual 0.21; upsell 0.84; long 1.9",
- "trace": [],
- "timings": {
- "gateMs": 230.47950000000128,
- "gateSkipped": true,
- "ttftMs": 2150.78966699999,
- "modelMs": 3045.852583,
- "toolMs": 0,
- "totalMs": 3279.6020419999986,
- "judgeMs": 226.97287500000675,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3045.852583,
- "firstDeltaMs": 1917.5350000000035,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "elements",
- "effects",
- "transitions",
- "subtitles",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 110,
+ "declaredTools": 113,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "concise factual 0.20; upsell 0.55; long 2.0",
+ "instant": null,
+ "judgeNote": "concise factual 0.17; upsell 0.89; long 1.9",
"trace": [],
"timings": {
- "gateMs": 254.1837910000031,
+ "gateMs": 272.50187499999447,
"gateSkipped": true,
- "ttftMs": 2346.775207999992,
- "modelMs": 3453.3652920000022,
+ "ttftMs": 2656.004375000004,
+ "modelMs": 3621.348416999994,
"toolMs": 0,
- "totalMs": 3711.0452919999952,
- "judgeMs": 332.4827920000098,
+ "totalMs": 3898.3247919999994,
+ "judgeMs": 286.1975420000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 3453.3652920000022,
- "firstDeltaMs": 2090.231500000009,
+ "roundMs": 3621.348416999994,
+ "firstDeltaMs": 2379.258916999999,
"calls": []
}
]
@@ -1107,38 +740,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 259,
- "p95": 263,
- "mean": 261,
- "n": 2
+ "p50": 375,
+ "p95": 375,
+ "mean": 375,
+ "n": 1
},
"ttftMs": {
- "p50": 1434,
- "p95": 8678,
- "mean": 5056,
- "n": 2
+ "p50": 5759,
+ "p95": 5759,
+ "mean": 5759,
+ "n": 1
},
"totalMs": {
- "p50": 1696,
- "p95": 8806,
- "mean": 5251,
- "n": 2
+ "p50": 5943,
+ "p95": 5943,
+ "mean": 5943,
+ "n": 1
},
"modelMs": {
- "p50": 1432,
- "p95": 8539,
- "mean": 4985,
- "n": 2
+ "p50": 5565,
+ "p95": 5565,
+ "mean": 5565,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 1.5,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
@@ -1156,62 +789,23 @@
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 259.3814999999886,
- "gateSkipped": false,
- "ttftMs": 8677.94183299999,
- "modelMs": 8538.731751,
- "toolMs": 0,
- "totalMs": 8805.912207999994,
- "judgeMs": 548.3632499999949,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3911.315333999999,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 4627.416417,
- "firstDeltaMs": 4499.9619590000075,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 38,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 262.62966599999345,
+ "gateMs": 375.1201250000013,
"gateSkipped": false,
- "ttftMs": 1433.572291000004,
- "modelMs": 1431.7364170000073,
+ "ttftMs": 5759.179917000001,
+ "modelMs": 5564.7642080000005,
"toolMs": 0,
- "totalMs": 1696.436333000005,
- "judgeMs": 273.283208000008,
+ "totalMs": 5943.383834,
+ "judgeMs": 589.8631250000035,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1431.7364170000073,
- "firstDeltaMs": 1169.2181249999994,
+ "roundMs": 5564.7642080000005,
+ "firstDeltaMs": 5380.985207999998,
"calls": []
}
]
@@ -1226,34 +820,34 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 263,
- "p95": 266,
- "mean": 265,
- "n": 2
+ "p50": 506,
+ "p95": 506,
+ "mean": 506,
+ "n": 1
},
"ttftMs": {
- "p50": 3166,
- "p95": 3209,
- "mean": 3188,
- "n": 2
+ "p50": 6569,
+ "p95": 6569,
+ "mean": 6569,
+ "n": 1
},
"totalMs": {
- "p50": 3433,
- "p95": 3436,
- "mean": 3434,
- "n": 2
+ "p50": 6693,
+ "p95": 6693,
+ "mean": 6693,
+ "n": 1
},
"modelMs": {
- "p50": 3158,
- "p95": 3166,
- "mean": 3162,
- "n": 2
+ "p50": 6178,
+ "p95": 6178,
+ "mean": 6178,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -1269,7 +863,6 @@
"preview",
"timeline",
"inspector",
- "animation",
"transitions",
"subtitles",
"audio",
@@ -1279,54 +872,98 @@
"stock",
"library",
"side_panel",
- "top_bar",
- "editor"
+ "top_bar"
],
- "declaredTools": 113,
+ "declaredTools": 108,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 1752.7311659999978,
- "execMs": 0.01779099999112077,
+ "atMs": 5054.001750000003,
+ "execMs": 0.008249999998952262,
"args": {
- "prompt": "A joyful dog running and playing in a sunny park, cinematic lighting."
+ "prompt": "A playful dog running happily through a green park, sunny day, cinematic style."
}
}
],
"timings": {
- "gateMs": 266.34341600000334,
+ "gateMs": 505.6267499999958,
"gateSkipped": false,
- "ttftMs": 3209.0444579999894,
- "modelMs": 3158.371082999991,
- "toolMs": 0.01779099999112077,
- "totalMs": 3432.8083329999936,
- "judgeMs": 260.37766700000793,
+ "ttftMs": 6569.255584000006,
+ "modelMs": 6178.490166999996,
+ "toolMs": 0.008249999998952262,
+ "totalMs": 6692.863583999999,
+ "judgeMs": 530.565208,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1480.4649579999968,
+ "roundMs": 4541.619249999996,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.01779099999112077
+ "execMs": 0.008249999998952262
}
]
},
{
- "roundMs": 1677.906124999994,
- "firstDeltaMs": 1454.6185830000031,
+ "roundMs": 1636.8709170000002,
+ "firstDeltaMs": 1513.5462920000064,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "video-ask-with-size-and-length",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 452,
+ "p95": 452,
+ "mean": 452,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 3412,
+ "p95": 3412,
+ "mean": 3412,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 3868,
+ "p95": 3868,
+ "mean": 3868,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 3405,
+ "p95": 3405,
+ "mean": 3405,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "simple",
@@ -1335,57 +972,60 @@
"preview",
"timeline",
"inspector",
- "transitions",
- "subtitles",
- "audio",
+ "effects",
+ "animation",
"image_gen",
"video_gen",
"scene",
"stock",
"library",
"side_panel",
- "top_bar"
+ "top_bar",
+ "editor"
],
- "declaredTools": 107,
+ "declaredTools": 98,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 1754.778250000003,
- "execMs": 0.009125000011408702,
+ "atMs": 2082.371459000002,
+ "execMs": 0.2822920000035083,
"args": {
- "prompt": "A cheerful dog running happily through a lush green park, sunny day, high definition, slow motion."
+ "prompt": "A paper boat drifting across a puddle, cinematic lighting, 1080p.",
+ "resolution": "1080p",
+ "duration_seconds": 6
}
}
],
"timings": {
- "gateMs": 263.310083999997,
+ "gateMs": 451.9445839999971,
"gateSkipped": false,
- "ttftMs": 3166.4713750000083,
- "modelMs": 3166.453999999998,
- "toolMs": 0.009125000011408702,
- "totalMs": 3435.7610419999983,
- "judgeMs": 277.9326669999864,
+ "ttftMs": 3412.0437919999968,
+ "modelMs": 3405.1449589999975,
+ "toolMs": 0.2822920000035083,
+ "totalMs": 3868.4129590000084,
+ "judgeMs": 276.29754200000025,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1488.7930000000051,
+ "roundMs": 1621.674708999999,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.009125000011408702
+ "execMs": 0.2822920000035083
}
]
},
{
- "roundMs": 1677.6609999999928,
- "firstDeltaMs": 1408.8366249999963,
+ "roundMs": 1783.4702499999985,
+ "firstDeltaMs": 1327.4159169999912,
"calls": []
}
]
@@ -1394,33 +1034,33 @@
]
},
{
- "name": "video-ask-with-size-and-length",
+ "name": "iterate-keeps-existing-clip",
"bucket": "single-tool",
- "passRate": 0.5,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1124,
- "p95": 1124,
- "mean": 1124,
+ "p50": 465,
+ "p95": 465,
+ "mean": 465,
"n": 1
},
"ttftMs": {
- "p50": 15281,
- "p95": 15281,
- "mean": 15281,
+ "p50": 5161,
+ "p95": 5161,
+ "mean": 5161,
"n": 1
},
"totalMs": {
- "p50": 16012,
- "p95": 16012,
- "mean": 16012,
+ "p50": 5478,
+ "p95": 5478,
+ "mean": 5478,
"n": 1
},
"modelMs": {
- "p50": 15599,
- "p95": 15599,
- "mean": 15599,
+ "p50": 5007,
+ "p95": 5007,
+ "mean": 5007,
"n": 1
},
"toolMs": {
@@ -1435,75 +1075,65 @@
}
},
"runs": [
- {
- "pass": false,
- "intent": "work",
- "notes": [],
- "trace": [],
- "timings": null,
- "error": "This operation was aborted"
- },
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "color-grading",
"areas": [
"preview",
"timeline",
"inspector",
+ "color",
"effects",
- "animation",
- "image_gen",
"video_gen",
"scene",
- "stock",
- "library",
"side_panel",
"top_bar",
"editor"
],
- "declaredTools": 97,
+ "declaredTools": 93,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "generate_video",
"round": 0,
- "atMs": 10523.85370800001,
- "execMs": 0.0022080000053392723,
+ "atMs": 3063.68908299999,
+ "execMs": 0.20266699999046978,
"args": {
- "prompt": "a paper boat drifting across a puddle, cinematic lighting, 1080p",
- "duration_seconds": 6,
- "resolution": "1080p"
+ "prompt": "Scenic beach at golden-hour sunset, warm glowing light, gentle ocean waves rolling onto the sand, sun dipping toward the horizon, cinematic camera movement",
+ "reference_asset_id": "a-v1",
+ "aspect": "9:16"
}
}
],
"timings": {
- "gateMs": 1124.1443750000035,
+ "gateMs": 464.5534590000025,
"gateSkipped": false,
- "ttftMs": 15281.440125000008,
- "modelMs": 15598.731249999997,
- "toolMs": 0.0022080000053392723,
- "totalMs": 16012.236250000002,
- "judgeMs": 711.551833999998,
+ "ttftMs": 5160.882582999999,
+ "modelMs": 5007.143416999999,
+ "toolMs": 0.20266699999046978,
+ "totalMs": 5478.474665999995,
+ "judgeMs": 531.4503750000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 10118.337124999991,
+ "roundMs": 2595.5232920000126,
"firstDeltaMs": null,
"calls": [
{
"name": "generate_video",
- "execMs": 0.0022080000053392723
+ "execMs": 0.20266699999046978
}
]
},
{
- "roundMs": 5480.394125000006,
- "firstDeltaMs": 4750.2749579999945,
+ "roundMs": 2411.6201249999867,
+ "firstDeltaMs": 2094.7140409999993,
"calls": []
}
]
@@ -1512,40 +1142,40 @@
]
},
{
- "name": "iterate-keeps-existing-clip",
+ "name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 303,
- "p95": 322,
- "mean": 313,
- "n": 2
+ "p50": 537,
+ "p95": 537,
+ "mean": 537,
+ "n": 1
},
"ttftMs": {
- "p50": 6209,
- "p95": 11595,
- "mean": 8902,
- "n": 2
+ "p50": 4551,
+ "p95": 4551,
+ "mean": 4551,
+ "n": 1
},
"totalMs": {
- "p50": 6431,
- "p95": 11925,
- "mean": 9178,
- "n": 2
+ "p50": 4697,
+ "p95": 4697,
+ "mean": 4697,
+ "n": 1
},
"modelMs": {
- "p50": 6118,
- "p95": 11573,
- "mean": 8846,
- "n": 2
+ "p50": 4146,
+ "p95": 4146,
+ "mean": 4146,
+ "n": 1
},
"toolMs": {
- "p50": 0,
- "p95": 3,
+ "p50": 1,
+ "p95": 1,
"mean": 1,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 3,
@@ -1555,134 +1185,194 @@
"runs": [
{
"pass": true,
- "intent": "complex",
- "skill": "color-grading",
+ "intent": "simple",
+ "skill": null,
"areas": [
- "preview",
"timeline",
"inspector",
- "color",
- "effects",
- "video_gen",
- "scene",
- "side_panel",
- "top_bar"
+ "audio",
+ "editor"
],
- "declaredTools": 88,
+ "declaredTools": 65,
"routingMisses": [],
- "roundModel": "gemini-3.8-flash",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "generate_video",
+ "name": "select_items",
+ "round": 0,
+ "atMs": 2080.143790999995,
+ "execMs": 0.6971659999981057,
+ "args": {
+ "describe": "all clips",
+ "kind": "clip"
+ }
+ },
+ {
+ "name": "set_clip_muted",
"round": 1,
- "atMs": 7802.556916000001,
- "execMs": 2.6554999999934807,
+ "atMs": 3460.424374999988,
+ "execMs": 0.011958000002778135,
"args": {
- "reference_asset_id": "a-v1",
- "prompt": "Scenic beach at golden-hour sunset, warm glowing sunlight reflecting across ocean waves and wet sand, gentle cinematic camera motion, peaceful evening light"
+ "muted": true,
+ "ids": [
+ "c1",
+ "c2",
+ "c3"
+ ]
}
}
],
"timings": {
- "gateMs": 321.89491600000474,
+ "gateMs": 536.8160420000058,
"gateSkipped": false,
- "ttftMs": 11595.309333000012,
- "modelMs": 11573.099707999994,
- "toolMs": 2.6554999999934807,
- "totalMs": 11924.54104099999,
- "judgeMs": 540.6497089999903,
+ "ttftMs": 4550.54912499999,
+ "modelMs": 4145.760459000012,
+ "toolMs": 0.7091240000008838,
+ "totalMs": 4696.913625000001,
+ "judgeMs": 582.5177919999987,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1767.5983750000014,
+ "roundMs": 1536.341292000012,
"firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 5688.663333000004,
+ "calls": [
+ {
+ "name": "select_items",
+ "execMs": 0.6971659999981057
+ }
+ ]
+ },
+ {
+ "roundMs": 1374.874125000002,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 2.6554999999934807
+ "name": "set_clip_muted",
+ "execMs": 0.011958000002778135
}
]
},
{
- "roundMs": 4116.837999999989,
- "firstDeltaMs": 3787.974208999978,
+ "roundMs": 1234.545041999998,
+ "firstDeltaMs": 1088.6765840000007,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "imported-photo-edit-references-photo",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 514,
+ "p95": 514,
+ "mean": 514,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 4378,
+ "p95": 4378,
+ "mean": 4378,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 4756,
+ "p95": 4756,
+ "mean": 4756,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 4237,
+ "p95": 4237,
+ "mean": 4237,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
- "intent": "complex",
- "skill": "color-grading",
+ "intent": "simple",
+ "skill": null,
"areas": [
"preview",
"timeline",
"inspector",
- "color",
+ "removal",
+ "elements",
"effects",
+ "animation",
+ "image_gen",
"video_gen",
"scene",
"stock",
"library",
"side_panel",
- "top_bar"
+ "top_bar",
+ "editor"
],
- "declaredTools": 97,
+ "declaredTools": 104,
"routingMisses": [],
- "roundModel": "gemini-3.8-flash",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": null,
+ "instant": null,
+ "judgeNote": "concise factual 0.46",
"trace": [
{
- "name": "generate_video",
- "round": 1,
- "atMs": 4222.986291999987,
- "execMs": 0.014374999998835847,
+ "name": "generate_image",
+ "round": 0,
+ "atMs": 2897.7629999999917,
+ "execMs": 0.08483300000079907,
"args": {
- "reference_asset_id": "a-v1",
- "prompt": "A scenic beach bathed in warm golden-hour sunset light, gentle ocean waves lapping at the shore, soft warm orange and pink horizon, cinematic camera motion",
- "aspect": "9:16"
+ "reference_asset_ids": [
+ "a-tw1"
+ ],
+ "prompt": "The girl in the photo from this post, wearing a Santa hat. The style, lighting, and composition are identical to the original image."
}
}
],
"timings": {
- "gateMs": 303.25975000002654,
+ "gateMs": 514.291750000004,
"gateSkipped": false,
- "ttftMs": 6209.459000000003,
- "modelMs": 6117.910833000002,
- "toolMs": 0.014374999998835847,
- "totalMs": 6431.130208999995,
- "judgeMs": 497.73662499999045,
+ "ttftMs": 4377.539250000002,
+ "modelMs": 4237.1919999999955,
+ "toolMs": 0.08483300000079907,
+ "totalMs": 4756.461207999993,
+ "judgeMs": 547.1037919999944,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1569.9868330000027,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2341.4284159999806,
+ "roundMs": 2380.1294999999955,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.014374999998835847
+ "name": "generate_image",
+ "execMs": 0.08483300000079907
}
]
},
{
- "roundMs": 2206.4955840000184,
- "firstDeltaMs": 1984.9706250000163,
+ "roundMs": 1857.0625,
+ "firstDeltaMs": 1478.8639589999948,
"calls": []
}
]
@@ -1691,40 +1381,40 @@
]
},
{
- "name": "mute-all-does-not-redo-prior-turn",
+ "name": "audio-ask-single-generate",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 217,
- "p95": 227,
- "mean": 222,
- "n": 2
+ "p50": 621,
+ "p95": 621,
+ "mean": 621,
+ "n": 1
},
"ttftMs": {
- "p50": 2833,
- "p95": 2954,
- "mean": 2894,
- "n": 2
+ "p50": 4195,
+ "p95": 4195,
+ "mean": 4195,
+ "n": 1
},
"totalMs": {
- "p50": 3014,
- "p95": 3059,
- "mean": 3037,
- "n": 2
+ "p50": 4339,
+ "p95": 4339,
+ "mean": 4339,
+ "n": 1
},
"modelMs": {
- "p50": 2793,
- "p95": 2823,
- "mean": 2808,
- "n": 2
+ "p50": 3711,
+ "p95": 3711,
+ "mean": 3711,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -1740,163 +1430,165 @@
"preview",
"timeline",
"inspector",
+ "subtitles",
"audio",
- "editor"
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
],
- "declaredTools": 69,
+ "declaredTools": 107,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1902.9193749999977,
- "execMs": 0.06995800000731833,
- "args": {
- "clipId": "c1",
- "muted": true
- }
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1903.2229999999981,
- "execMs": 0.0027500000142026693,
- "args": {
- "muted": true,
- "clipId": "c2"
- }
- },
- {
- "name": "set_clip_muted",
+ "name": "voiceover_generate",
"round": 0,
- "atMs": 1903.465750000003,
- "execMs": 0.003874999994877726,
+ "atMs": 2717.6584580000053,
+ "execMs": 0.01224999999976717,
"args": {
- "muted": true,
- "clipId": "c3"
+ "script": "Welcome to the dog show."
}
}
],
"timings": {
- "gateMs": 226.62166699999943,
+ "gateMs": 621.4086249999964,
"gateSkipped": false,
- "ttftMs": 2953.701915999991,
- "modelMs": 2823.3891669999866,
- "toolMs": 0.07658300001639873,
- "totalMs": 3058.836290999985,
- "judgeMs": 206.96162500002538,
+ "ttftMs": 4194.916041999997,
+ "modelMs": 3711.3718760000047,
+ "toolMs": 0.01224999999976717,
+ "totalMs": 4339.3164169999945,
+ "judgeMs": 711.4737920000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1669.7185839999875,
+ "roundMs": 2091.9540419999976,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.06995800000731833
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0027500000142026693
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.003874999994877726
+ "name": "voiceover_generate",
+ "execMs": 0.01224999999976717
}
]
},
{
- "roundMs": 1153.6705829999992,
- "firstDeltaMs": 1048.986957999994,
+ "roundMs": 1619.4178340000071,
+ "firstDeltaMs": 1475.3925420000014,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "music-ask-single-generate",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 311,
+ "p95": 311,
+ "mean": 311,
+ "n": 1
},
+ "ttftMs": {
+ "p50": 5346,
+ "p95": 5346,
+ "mean": 5346,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 5580,
+ "p95": 5580,
+ "mean": 5580,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 5262,
+ "p95": 5262,
+ "mean": 5262,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "audio-and-subtitles",
"areas": [
"preview",
"timeline",
"inspector",
+ "subtitles",
"audio",
- "editor"
+ "stock",
+ "library",
+ "side_panel"
],
- "declaredTools": 69,
+ "declaredTools": 95,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1580.9433750000026,
- "execMs": 0.01179200000478886,
- "args": {
- "clipId": "c1",
- "muted": true
- }
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 1581.1917919999978,
- "execMs": 0.0027920000138692558,
- "args": {
- "clipId": "c2",
- "muted": true
- }
- },
- {
- "name": "set_clip_muted",
+ "name": "generate_music",
"round": 0,
- "atMs": 1581.402833,
- "execMs": 0.002207999990787357,
+ "atMs": 2909.1016669999954,
+ "execMs": 0.07000000000698492,
"args": {
- "muted": true,
- "clipId": "c3"
+ "volume": 0.4,
+ "start": 0,
+ "instrumental": true,
+ "add_to_timeline": true,
+ "prompt": "Chill, mellow lofi hip hop background music with soft electric piano, smooth ambient textures, and gentle downtempo beat",
+ "length": "clip"
}
}
],
"timings": {
- "gateMs": 217.39104200000293,
+ "gateMs": 311.4786249999888,
"gateSkipped": false,
- "ttftMs": 2833.381625000009,
- "modelMs": 2792.722999999998,
- "toolMs": 0.016792000009445474,
- "totalMs": 3014.457208000007,
- "judgeMs": 218.88420800000313,
+ "ttftMs": 5345.563500000004,
+ "modelMs": 5262.396999999997,
+ "toolMs": 0.07000000000698492,
+ "totalMs": 5580.217499999999,
+ "judgeMs": 525.0467920000083,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1361.5271249999932,
+ "roundMs": 2593.245041999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.01179200000478886
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0027920000138692558
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.002207999990787357
+ "name": "generate_music",
+ "execMs": 0.07000000000698492
}
]
},
{
- "roundMs": 1431.195875000005,
- "firstDeltaMs": 1250.511415999994,
+ "roundMs": 2669.151958000002,
+ "firstDeltaMs": 2434.863249999995,
"calls": []
}
]
@@ -1905,40 +1597,40 @@
]
},
{
- "name": "imported-photo-edit-references-photo",
+ "name": "trim-ask-single-tool",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 220,
- "p95": 266,
- "mean": 243,
- "n": 2
+ "p50": 476,
+ "p95": 476,
+ "mean": 476,
+ "n": 1
},
"ttftMs": {
- "p50": 3314,
- "p95": 3858,
- "mean": 3586,
- "n": 2
+ "p50": 3297,
+ "p95": 3297,
+ "mean": 3297,
+ "n": 1
},
"totalMs": {
- "p50": 3667,
- "p95": 4203,
- "mean": 3935,
- "n": 2
+ "p50": 3723,
+ "p95": 3723,
+ "mean": 3723,
+ "n": 1
},
"modelMs": {
- "p50": 3438,
- "p95": 3931,
- "mean": 3685,
- "n": 2
+ "p50": 3240,
+ "p95": 3240,
+ "mean": 3240,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -1954,132 +1646,157 @@
"preview",
"timeline",
"inspector",
- "removal",
- "elements",
- "effects",
- "animation",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
"editor"
],
- "declaredTools": 103,
+ "declaredTools": 66,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "generate_image",
+ "name": "trim_clip",
"round": 0,
- "atMs": 1988.851915999985,
- "execMs": 0.2047080000047572,
+ "atMs": 1824.863083999997,
+ "execMs": 0.013250000003608875,
"args": {
- "reference_asset_ids": [
- "a-tw1"
- ],
- "prompt": "a photo of the person in the reference image wearing a santa hat, maintaining the style, lighting, and composition of the original photo"
+ "in": 0,
+ "out": 5,
+ "clipId": "c1"
}
}
],
"timings": {
- "gateMs": 219.92966699998942,
+ "gateMs": 475.8945420000091,
"gateSkipped": false,
- "ttftMs": 3313.9157499999856,
- "modelMs": 3438.0066660000302,
- "toolMs": 0.2047080000047572,
- "totalMs": 3667.044790999993,
- "judgeMs": 244.47749999997905,
+ "ttftMs": 3297.279917000007,
+ "modelMs": 3239.899292000002,
+ "toolMs": 0.013250000003608875,
+ "totalMs": 3722.5478339999972,
+ "judgeMs": 244.66629200000898,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1761.965166000009,
+ "roundMs": 1343.831416999994,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.2047080000047572
+ "name": "trim_clip",
+ "execMs": 0.013250000003608875
}
]
},
{
- "roundMs": 1676.0415000000212,
- "firstDeltaMs": 1323.256625000009,
+ "roundMs": 1896.067875000008,
+ "firstDeltaMs": 1471.1177080000052,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "transition-ask-styled",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 433,
+ "p95": 433,
+ "mean": 433,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 8104,
+ "p95": 8104,
+ "mean": 8104,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 8252,
+ "p95": 8252,
+ "mean": 8252,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 7814,
+ "p95": 7814,
+ "mean": 7814,
+ "n": 1
},
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "transitions-and-fades",
"areas": [
"preview",
"timeline",
"inspector",
- "removal",
- "elements",
"effects",
"animation",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
+ "transitions",
"top_bar",
"editor"
],
- "declaredTools": 103,
+ "declaredTools": 73,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "generate_image",
+ "name": "set_transition",
"round": 0,
- "atMs": 2701.744250000018,
- "execMs": 0.012959000014234334,
+ "atMs": 6044.441207999989,
+ "execMs": 0.06787500000791624,
"args": {
- "reference_asset_ids": [
- "a-tw1"
- ],
- "prompt": "A high-quality recreation of the girl in the \"first snow of the year\" photo, now wearing a festive red Santa hat, maintaining the original composition and aesthetic.",
- "aspect": "9:16"
+ "style": "pushdown",
+ "seconds": 0.5,
+ "clipId": "c1"
}
}
],
"timings": {
- "gateMs": 266.1987909999734,
+ "gateMs": 433.28800000000047,
"gateSkipped": false,
- "ttftMs": 3858.036208000005,
- "modelMs": 3931.0774989999773,
- "toolMs": 0.012959000014234334,
- "totalMs": 4203.301917000004,
- "judgeMs": 224.05791699999827,
+ "ttftMs": 8103.729582999993,
+ "modelMs": 7813.888875000019,
+ "toolMs": 0.06787500000791624,
+ "totalMs": 8251.897332999986,
+ "judgeMs": 508.81104099999357,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2431.2394159999967,
+ "roundMs": 5607.726459000012,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.012959000014234334
+ "name": "set_transition",
+ "execMs": 0.06787500000791624
}
]
},
{
- "roundMs": 1499.8380829999805,
- "firstDeltaMs": 1154.8791249999776,
+ "roundMs": 2206.1624160000065,
+ "firstDeltaMs": 2058.428791000013,
"calls": []
}
]
@@ -2088,44 +1805,44 @@
]
},
{
- "name": "audio-ask-single-generate",
+ "name": "slide-in-direction-correct",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 224,
- "p95": 248,
- "mean": 236,
- "n": 2
+ "p50": 532,
+ "p95": 532,
+ "mean": 532,
+ "n": 1
},
"ttftMs": {
- "p50": 2787,
- "p95": 3334,
- "mean": 3061,
- "n": 2
+ "p50": 534,
+ "p95": 534,
+ "mean": 534,
+ "n": 1
},
"totalMs": {
- "p50": 3004,
- "p95": 3585,
- "mean": 3295,
- "n": 2
+ "p50": 534,
+ "p95": 534,
+ "mean": 534,
+ "n": 1
},
"modelMs": {
- "p50": 2750,
- "p95": 3355,
- "mean": 3052,
- "n": 2
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 0,
+ "max": 0
}
},
"runs": [
@@ -2137,60 +1854,181 @@
"preview",
"timeline",
"inspector",
- "subtitles",
- "audio",
+ "effects",
+ "animation",
+ "transitions",
+ "editor"
+ ],
+ "declaredTools": 69,
+ "routingMisses": [],
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "instant": "set_animation",
+ "judgeNote": "failures owned 0.44",
+ "trace": [
+ {
+ "name": "set_animation",
+ "round": 0,
+ "atMs": 533.5733749999927,
+ "execMs": 0.06333299999823794,
+ "args": {
+ "clipId": "c2",
+ "which": "in",
+ "style": "slideright",
+ "seconds": 0.47
+ }
+ }
+ ],
+ "timings": {
+ "gateMs": 532.2724170000001,
+ "gateSkipped": false,
+ "ttftMs": 534.0615419999958,
+ "modelMs": 0,
+ "toolMs": 0.06333299999823794,
+ "totalMs": 534.0719169999938,
+ "judgeMs": 198.63141700001142,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": []
+ }
+ }
+ ]
+ },
+ {
+ "name": "look-ask-single-tool",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 366,
+ "p95": 366,
+ "mean": 366,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 367,
+ "p95": 367,
+ "mean": 367,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 367,
+ "p95": 367,
+ "mean": 367,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 0,
+ "max": 0
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "simple",
+ "skill": null,
+ "areas": [
+ "preview",
+ "timeline",
+ "inspector",
+ "color",
+ "effects",
"video_gen",
"scene",
- "stock",
- "library",
"side_panel",
- "top_bar"
+ "top_bar",
+ "editor"
],
- "declaredTools": 106,
+ "declaredTools": 93,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": "add_effect",
"judgeNote": null,
"trace": [
{
- "name": "voiceover_generate",
+ "name": "add_effect",
"round": 0,
- "atMs": 1881.8186250000144,
- "execMs": 0.007541999977547675,
+ "atMs": 366.4008329999924,
+ "execMs": 0.18120900000212714,
"args": {
- "script": "Welcome to the dog show."
+ "effect": "vhs",
+ "amount": 0.67,
+ "start": 6,
+ "end": 12.5
}
}
],
"timings": {
- "gateMs": 223.6105830000015,
+ "gateMs": 365.6525419999962,
"gateSkipped": false,
- "ttftMs": 3334.4220000000205,
- "modelMs": 3355.0260420000122,
- "toolMs": 0.007541999977547675,
- "totalMs": 3584.7433330000204,
- "judgeMs": 254.3882910000102,
+ "ttftMs": 366.85558299999684,
+ "modelMs": 0,
+ "toolMs": 0.18120900000212714,
+ "totalMs": 366.87566700000025,
+ "judgeMs": 271.4133749999892,
"retryMs": 0,
"retries": 0,
- "rounds": [
- {
- "roundMs": 1653.8843749999942,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.007541999977547675
- }
- ]
- },
- {
- "roundMs": 1701.141667000018,
- "firstDeltaMs": 1451.2494170000136,
- "calls": []
- }
- ]
+ "rounds": []
}
+ }
+ ]
+ },
+ {
+ "name": "audio-effect-single-tool",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 302,
+ "p95": 302,
+ "mean": 302,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 303,
+ "p95": 303,
+ "mean": 303,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 303,
+ "p95": 303,
+ "mean": 303,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 0,
+ "max": 0
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "simple",
@@ -2199,90 +2037,73 @@
"preview",
"timeline",
"inspector",
- "subtitles",
+ "effects",
"audio",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar"
+ "side_panel"
],
- "declaredTools": 106,
+ "declaredTools": 74,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": "add_effect",
"judgeNote": null,
"trace": [
{
- "name": "voiceover_generate",
+ "name": "add_effect",
"round": 0,
- "atMs": 1573.5448749999923,
- "execMs": 0.00987500001792796,
+ "atMs": 302.74691699999676,
+ "execMs": 0.3786249999975553,
"args": {
- "script": "Welcome to the dog show."
+ "effect": "telephone",
+ "amount": 0.68,
+ "start": 6,
+ "end": 12.5
}
}
],
"timings": {
- "gateMs": 248.26612499999464,
+ "gateMs": 302.0796249999985,
"gateSkipped": false,
- "ttftMs": 2787.0850840000203,
- "modelMs": 2749.9022919999843,
- "toolMs": 0.00987500001792796,
- "totalMs": 3004.278250000003,
- "judgeMs": 221.06387500002165,
+ "ttftMs": 303.43887499999255,
+ "modelMs": 0,
+ "toolMs": 0.3786249999975553,
+ "totalMs": 303.4515839999949,
+ "judgeMs": 281.96937499998603,
"retryMs": 0,
"retries": 0,
- "rounds": [
- {
- "roundMs": 1322.1817919999885,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.00987500001792796
- }
- ]
- },
- {
- "roundMs": 1427.7204999999958,
- "firstDeltaMs": 1212.0876250000147,
- "calls": []
- }
- ]
+ "rounds": []
}
}
]
},
{
- "name": "music-ask-single-generate",
+ "name": "sound-dissolve-single-tool",
"bucket": "single-tool",
- "passRate": 0.5,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 268,
- "p95": 268,
- "mean": 268,
+ "p50": 340,
+ "p95": 340,
+ "mean": 340,
"n": 1
},
"ttftMs": {
- "p50": 9835,
- "p95": 9835,
- "mean": 9835,
+ "p50": 4533,
+ "p95": 4533,
+ "mean": 4533,
"n": 1
},
"totalMs": {
- "p50": 10184,
- "p95": 10184,
- "mean": 10184,
+ "p50": 4686,
+ "p95": 4686,
+ "mean": 4686,
"n": 1
},
"modelMs": {
- "p50": 9912,
- "p95": 9912,
- "mean": 9912,
+ "p50": 4336,
+ "p95": 4336,
+ "mean": 4336,
"n": 1
},
"toolMs": {
@@ -2298,179 +2119,66 @@
},
"runs": [
{
- "pass": false,
- "intent": "complex",
- "skill": "audio-and-subtitles",
+ "pass": true,
+ "intent": "simple",
+ "skill": null,
"areas": [
"preview",
"timeline",
"inspector",
- "subtitles",
+ "effects",
+ "transitions",
"audio",
- "stock",
- "library",
- "side_panel"
+ "editor"
],
- "declaredTools": 94,
+ "declaredTools": 71,
"routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [
- "mutating tools called: add_clip"
- ],
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "generate_music",
+ "name": "set_transition",
"round": 1,
- "atMs": 5008.586624999996,
- "execMs": 0.17216600000392646,
- "args": {
- "instrumental": true,
- "length": "clip",
- "prompt": "Chill, relaxed lofi ambient acoustic background music, warm gentle guitar and soft synth pads, mellow and calming vibe",
- "start": 0,
- "add_to_timeline": true,
- "volume": 0.35
- }
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 19322.093041999993,
- "execMs": 0.1537909999897238,
- "args": {}
- },
- {
- "name": "add_clip",
- "round": 3,
- "atMs": 28052.555291999975,
- "execMs": 0.008332999976119027,
+ "atMs": 3238.128500000006,
+ "execMs": 0.3135829999955604,
"args": {
- "lane": 1,
- "asset_id": "a-music1",
- "start": 0
+ "seconds": 0.3,
+ "style": "audiocross",
+ "clipId": "c1"
}
}
],
"timings": {
- "gateMs": 246.73874999998952,
+ "gateMs": 340.3528329999972,
"gateSkipped": false,
- "ttftMs": 41920.403416999994,
- "modelMs": 41960.425292,
- "toolMs": 0.33428999996976927,
- "totalMs": 42219.28408299998,
- "judgeMs": 554.3412499999977,
+ "ttftMs": 4532.745125000001,
+ "modelMs": 4335.873957000003,
+ "toolMs": 0.3135829999955604,
+ "totalMs": 4685.774916000009,
+ "judgeMs": 657.7384170000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1830.7174170000071,
+ "roundMs": 1343.5783749999973,
"firstDeltaMs": null,
"calls": []
},
{
- "roundMs": 2924.800416000013,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.17216600000392646
- }
- ]
- },
- {
- "roundMs": 14310.745166999986,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.1537909999897238
- }
- ]
- },
- {
- "roundMs": 8728.772792000003,
+ "roundMs": 1546.9686660000007,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.008332999976119027
+ "name": "set_transition",
+ "execMs": 0.3135829999955604
}
]
},
{
- "roundMs": 14165.38949999999,
- "firstDeltaMs": 13866.961291999993,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "audio-and-subtitles",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "subtitles",
- "audio",
- "stock",
- "library",
- "side_panel",
- "top_bar"
- ],
- "declaredTools": 98,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_music",
- "round": 1,
- "atMs": 7789.254709000001,
- "execMs": 0.013916999974753708,
- "args": {
- "length": "clip",
- "add_to_timeline": true,
- "volume": 0.4,
- "start": 0,
- "prompt": "Chill lofi ambient background music, relaxed acoustic beats, mellow guitar and soft synth, warm summer beach vibe",
- "instrumental": true
- }
- }
- ],
- "timings": {
- "gateMs": 268.3392500000191,
- "gateSkipped": false,
- "ttftMs": 9835.427792000002,
- "modelMs": 9911.988834999967,
- "toolMs": 0.013916999974753708,
- "totalMs": 10184.317209,
- "judgeMs": 459.09216700002435,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4441.42420899999,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3076.8457089999865,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.013916999974753708
- }
- ]
- },
- {
- "roundMs": 2393.718916999991,
- "firstDeltaMs": 2044.9508330000099,
+ "roundMs": 1445.3269160000054,
+ "firstDeltaMs": 1292.6651660000061,
"calls": []
}
]
@@ -2479,40 +2187,40 @@
]
},
{
- "name": "trim-ask-single-tool",
+ "name": "animation-overrides-transition",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 218,
- "p95": 287,
- "mean": 252,
- "n": 2
+ "p50": 506,
+ "p95": 506,
+ "mean": 506,
+ "n": 1
},
"ttftMs": {
- "p50": 2617,
- "p95": 2766,
- "mean": 2691,
- "n": 2
+ "p50": 5866,
+ "p95": 5866,
+ "mean": 5866,
+ "n": 1
},
"totalMs": {
- "p50": 2761,
- "p95": 2916,
- "mean": 2838,
- "n": 2
+ "p50": 6147,
+ "p95": 6147,
+ "mean": 6147,
+ "n": 1
},
"modelMs": {
- "p50": 2538,
- "p95": 2627,
- "mean": 2582,
- "n": 2
+ "p50": 5634,
+ "p95": 5634,
+ "mean": 5634,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -2528,108 +2236,67 @@
"preview",
"timeline",
"inspector",
- "library",
+ "animation",
+ "transitions",
"editor"
],
- "declaredTools": 72,
+ "declaredTools": 68,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "remove_transition",
"round": 0,
- "atMs": 1551.334583000018,
- "execMs": 0.005499999999301508,
+ "atMs": 2514.8120419999905,
+ "execMs": 0.1587089999957243,
"args": {
- "clipId": "c1",
- "in": 0,
- "out": 5
- }
- }
- ],
- "timings": {
- "gateMs": 217.80354200000875,
- "gateSkipped": false,
- "ttftMs": 2617.0373330000148,
- "modelMs": 2538.0882920000004,
- "toolMs": 0.005499999999301508,
- "totalMs": 2761.014583000011,
- "judgeMs": 221.55958299999475,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1329.2464169999876,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "trim_clip",
- "execMs": 0.005499999999301508
- }
- ]
- },
- {
- "roundMs": 1208.8418750000128,
- "firstDeltaMs": 1064.9870419999934,
- "calls": []
+ "transitionId": "T-c1-c2"
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "library",
- "editor"
- ],
- "declaredTools": 72,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
+ },
{
- "name": "trim_clip",
+ "name": "set_animation",
"round": 0,
- "atMs": 1514.858332999982,
- "execMs": 0.004165999998804182,
+ "atMs": 2516.7521669999987,
+ "execMs": 0.009333000009064563,
"args": {
- "out": 5,
- "clipId": "c1"
+ "seconds": 0.5,
+ "which": "in",
+ "style": "pop",
+ "clipId": "c2"
}
}
],
"timings": {
- "gateMs": 286.5764579999959,
+ "gateMs": 505.9929159999883,
"gateSkipped": false,
- "ttftMs": 2765.8231249999953,
- "modelMs": 2626.754917000013,
- "toolMs": 0.004165999998804182,
- "totalMs": 2915.7682909999858,
- "judgeMs": 177.7972909999953,
+ "ttftMs": 5865.710541999986,
+ "modelMs": 5633.522166999988,
+ "toolMs": 0.16804200000478886,
+ "totalMs": 6147.075707999989,
+ "judgeMs": 566.6127080000006,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1226.7500419999997,
+ "roundMs": 2004.978959,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.004165999998804182
+ "name": "remove_transition",
+ "execMs": 0.1587089999957243
+ },
+ {
+ "name": "set_animation",
+ "execMs": 0.009333000009064563
}
]
},
{
- "roundMs": 1400.0048750000133,
- "firstDeltaMs": 1250.1533750000235,
+ "roundMs": 3628.543207999988,
+ "firstDeltaMs": 3347.3651669999963,
"calls": []
}
]
@@ -2638,336 +2305,311 @@
]
},
{
- "name": "transition-ask-styled",
- "bucket": "single-tool",
- "passRate": 1,
+ "name": "filler-words-cut-with-editor",
+ "bucket": "multi-tool",
+ "passRate": 0,
"budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 215,
- "p95": 215,
- "mean": 215,
- "n": 2
- },
- "ttftMs": {
- "p50": 2857,
- "p95": 6292,
- "mean": 4574,
- "n": 2
- },
- "totalMs": {
- "p50": 2966,
- "p95": 6440,
- "mean": 4703,
- "n": 2
- },
- "modelMs": {
- "p50": 2748,
- "p95": 6221,
- "mean": 4484,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2.5,
- "max": 3
- }
- },
+ "latency": null,
"runs": [
{
- "pass": true,
- "intent": "simple",
- "skill": null,
+ "pass": false,
+ "intent": "complex",
+ "skill": "watching-and-cutting",
"areas": [
"preview",
"timeline",
"inspector",
- "effects",
- "animation",
"transitions",
+ "subtitles",
+ "audio",
+ "library",
+ "side_panel",
+ "top_bar",
"editor"
],
- "declaredTools": 68,
+ "declaredTools": 101,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
+ "roundModel": "gemini-3.8-flash",
+ "notes": [
+ "mutating tools called: remove_gap"
+ ],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "find_filler",
"round": 0,
- "atMs": 1766.6607920000097,
- "execMs": 0.08154099999228492,
+ "atMs": 3767.003042000011,
+ "execMs": 0.0957909999997355,
"args": {
- "style": "pushdown",
- "clipId": "c1",
- "seconds": 0.5
- }
- }
- ],
- "timings": {
- "gateMs": 215.1701250000042,
- "gateSkipped": false,
- "ttftMs": 2856.6242499999935,
- "modelMs": 2747.8471250000002,
- "toolMs": 0.08154099999228492,
- "totalMs": 2966.304708999989,
- "judgeMs": 262.8814999999886,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1549.3415410000016,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.08154099999228492
- }
- ]
- },
- {
- "roundMs": 1198.5055839999986,
- "firstDeltaMs": 1088.9722919999913,
- "calls": []
+ "track": 0
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "transitions-and-fades",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "animation",
- "transitions",
- "library",
- "editor"
- ],
- "declaredTools": 75,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
+ },
{
- "name": "set_transition",
+ "name": "detect_silence",
"round": 1,
- "atMs": 3791.700832999981,
- "execMs": 0.009999999980209395,
+ "atMs": 6030.388542000015,
+ "execMs": 0.024291999987326562,
"args": {
- "seconds": 0.5,
- "clipId": "c1",
- "style": "pushdown"
+ "clip_id": "c1"
}
- }
- ],
- "timings": {
- "gateMs": 215.35533300001407,
- "gateSkipped": false,
- "ttftMs": 6291.746541999979,
- "modelMs": 6221.118248999963,
- "toolMs": 0.009999999980209395,
- "totalMs": 6440.362166999985,
- "judgeMs": 518.4899999999907,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1501.8468329999887,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2071.82699999999,
- "firstDeltaMs": null,
- "calls": [
+ },
+ {
+ "name": "watch_video",
+ "round": 2,
+ "atMs": 7921.313875000007,
+ "execMs": 0.02404200000455603,
+ "args": {
+ "to": 6,
+ "clip_id": "c1",
+ "interval_seconds": 1,
+ "from": 0
+ }
+ },
+ {
+ "name": "note_source",
+ "round": 3,
+ "atMs": 11197.174667,
+ "execMs": 0.030583000014303252,
+ "args": {
+ "clip_id": "c1",
+ "notes": [
{
- "name": "set_transition",
- "execMs": 0.009999999980209395
+ "from": 0,
+ "text": "Beach footage with dogs playing by the water",
+ "to": 6
}
]
- },
- {
- "roundMs": 2647.444415999984,
- "firstDeltaMs": 2499.012290999992,
- "calls": []
}
- ]
- }
- }
- ]
- },
- {
- "name": "slide-in-direction-correct",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 260,
- "p95": 263,
- "mean": 261,
- "n": 2
- },
- "ttftMs": {
- "p50": 2700,
- "p95": 4221,
- "mean": 3460,
- "n": 2
- },
- "totalMs": {
- "p50": 2788,
- "p95": 4408,
- "mean": 3598,
- "n": 2
- },
- "modelMs": {
- "p50": 2526,
- "p95": 4140,
- "mean": 3333,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2.5,
- "max": 3
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "animation",
- "transitions"
- ],
- "declaredTools": 63,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
+ },
{
- "name": "set_animation",
- "round": 1,
- "atMs": 3143.269540999987,
- "execMs": 0.07474999999976717,
+ "name": "trim_clip",
+ "round": 4,
+ "atMs": 13853.136291999996,
+ "execMs": 1.1199999999953434,
"args": {
- "which": "in",
- "style": "slideright",
- "clipId": "c2",
- "seconds": 0.5
+ "clipId": "c1",
+ "in": 0.45
+ }
+ },
+ {
+ "name": "split_at",
+ "round": 4,
+ "atMs": 13855.264875000008,
+ "execMs": 0.17829199999687262,
+ "args": {
+ "t": 4.1
+ }
+ },
+ {
+ "name": "split_at",
+ "round": 4,
+ "atMs": 13856.234000000011,
+ "execMs": 0.02383300001383759,
+ "args": {
+ "t": 5.15
+ }
+ },
+ {
+ "name": "delete_item",
+ "round": 5,
+ "atMs": 17126.910042000018,
+ "execMs": 0.11554200001410209,
+ "args": {
+ "kind": "clip",
+ "id": "c1-2-3"
+ }
+ },
+ {
+ "name": "remove_gap",
+ "round": 5,
+ "atMs": 17128.52962500001,
+ "execMs": 0.03500000000349246,
+ "args": {
+ "at": 4.5
+ }
+ },
+ {
+ "name": "get_state",
+ "round": 6,
+ "atMs": 18953.044083,
+ "execMs": 0.1336670000164304,
+ "args": {}
+ },
+ {
+ "name": "refine_speech_cuts",
+ "round": 7,
+ "atMs": 20914.840666999997,
+ "execMs": 0.06129099999088794,
+ "args": {
+ "pace": "natural",
+ "clip_ids": [
+ "c1-1",
+ "c1-2-4"
+ ]
+ }
+ },
+ {
+ "name": "align_to_audio",
+ "round": 8,
+ "atMs": 22686.507750000004,
+ "execMs": 0.024167000025045127,
+ "args": {
+ "track": 0,
+ "target": "captions"
+ }
+ },
+ {
+ "name": "update_cue",
+ "round": 9,
+ "atMs": 24849.964374999996,
+ "execMs": 0.03858300001593307,
+ "args": {
+ "text": "So today we're",
+ "id": "cue1"
+ }
+ },
+ {
+ "name": "update_cue",
+ "round": 9,
+ "atMs": 24850.235375000004,
+ "execMs": 0.013750000012805685,
+ "args": {
+ "id": "cue3",
+ "text": "And"
}
}
],
"timings": {
- "gateMs": 263.201040999993,
+ "gateMs": 481.8056660000002,
"gateSkipped": false,
- "ttftMs": 4220.703832999978,
- "modelMs": 4139.535084000032,
- "toolMs": 0.07474999999976717,
- "totalMs": 4408.397790999996,
- "judgeMs": 542.0186660000181,
+ "ttftMs": 26443.069042000003,
+ "modelMs": 26186.005707999953,
+ "toolMs": 1.9188330000906717,
+ "totalMs": 26703.704042000012,
+ "judgeMs": 511.13716700000805,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1399.340125000017,
+ "roundMs": 3280.327833000003,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "find_filler",
+ "execMs": 0.0957909999997355
+ }
+ ]
},
{
- "roundMs": 1476.0649590000103,
+ "roundMs": 2260.191166999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.07474999999976717
+ "name": "detect_silence",
+ "execMs": 0.024291999987326562
}
]
},
{
- "roundMs": 1264.1300000000047,
- "firstDeltaMs": 1076.5062500000058,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "animation",
- "transitions"
- ],
- "declaredTools": 64,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_animation",
- "round": 0,
- "atMs": 1625.3120000000054,
- "execMs": 0.003915999986929819,
- "args": {
- "seconds": 0.5,
- "clipId": "c2",
- "style": "slideright",
- "which": "in"
- }
- }
- ],
- "timings": {
- "gateMs": 259.50512499999604,
- "gateSkipped": false,
- "ttftMs": 2700.167665999994,
- "modelMs": 2526.1960839999956,
- "toolMs": 0.003915999986929819,
- "totalMs": 2787.675125000009,
- "judgeMs": 325.18249999999534,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 1887.7098330000008,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "watch_video",
+ "execMs": 0.02404200000455603
+ }
+ ]
+ },
{
- "roundMs": 1364.479833999998,
+ "roundMs": 3272.8237089999893,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.003915999986929819
+ "name": "note_source",
+ "execMs": 0.030583000014303252
+ }
+ ]
+ },
+ {
+ "roundMs": 2653.259625000006,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "trim_clip",
+ "execMs": 1.1199999999953434
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.17829199999687262
+ },
+ {
+ "name": "split_at",
+ "execMs": 0.02383300001383759
+ }
+ ]
+ },
+ {
+ "roundMs": 3267.363291000016,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "delete_item",
+ "execMs": 0.11554200001410209
+ },
+ {
+ "name": "remove_gap",
+ "execMs": 0.03500000000349246
+ }
+ ]
+ },
+ {
+ "roundMs": 1822.3442089999735,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.1336670000164304
+ }
+ ]
+ },
+ {
+ "roundMs": 1959.774624999991,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "refine_speech_cuts",
+ "execMs": 0.06129099999088794
+ }
+ ]
+ },
+ {
+ "roundMs": 1769.8817500000005,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "align_to_audio",
+ "execMs": 0.024167000025045127
+ }
+ ]
+ },
+ {
+ "roundMs": 2160.8513329999987,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.03858300001593307
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.013750000012805685
}
]
},
{
- "roundMs": 1161.7162499999977,
- "firstDeltaMs": 1074.3126660000125,
+ "roundMs": 1851.4783329999773,
+ "firstDeltaMs": 1591.298415999976,
"calls": []
}
]
@@ -2976,40 +2618,40 @@
]
},
{
- "name": "look-ask-single-tool",
+ "name": "edit-request-still-uses-tools",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 240,
- "p95": 290,
- "mean": 265,
- "n": 2
+ "p50": 509,
+ "p95": 509,
+ "mean": 509,
+ "n": 1
},
"ttftMs": {
- "p50": 2938,
- "p95": 2994,
- "mean": 2966,
- "n": 2
+ "p50": 3190,
+ "p95": 3190,
+ "mean": 3190,
+ "n": 1
},
"totalMs": {
- "p50": 3166,
- "p95": 3201,
- "mean": 3184,
- "n": 2
+ "p50": 3277,
+ "p95": 3277,
+ "mean": 3277,
+ "n": 1
},
"modelMs": {
- "p50": 2910,
- "p95": 2926,
- "mean": 2918,
- "n": 2
+ "p50": 2762,
+ "p95": 2762,
+ "mean": 2762,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -3025,121 +2667,53 @@
"preview",
"timeline",
"inspector",
- "color",
- "effects",
- "video_gen",
- "scene",
+ "subtitles",
"library",
"side_panel",
- "top_bar"
+ "top_bar",
+ "editor"
],
- "declaredTools": 95,
+ "declaredTools": 97,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
+ "name": "subtitles_generate",
"round": 0,
- "atMs": 1861.2977920000267,
- "execMs": 0.037958000000799075,
+ "atMs": 1846.7781249999825,
+ "execMs": 0.01699999999254942,
"args": {
- "effect": "vhs",
- "start": 6,
- "end": 12.5,
- "amount": 0.5
+ "track": 0
}
}
],
"timings": {
- "gateMs": 289.69391599998926,
+ "gateMs": 508.5648339999898,
"gateSkipped": false,
- "ttftMs": 2994.052250000008,
- "modelMs": 2909.613624999998,
- "toolMs": 0.037958000000799075,
- "totalMs": 3200.9923750000016,
- "judgeMs": 234.41870800001197,
+ "ttftMs": 3189.7560829999857,
+ "modelMs": 2761.642583000008,
+ "toolMs": 0.01699999999254942,
+ "totalMs": 3277.0722079999978,
+ "judgeMs": 375.7734590000182,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1570.4394169999869,
+ "roundMs": 1334.210208000004,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.037958000000799075
+ "name": "subtitles_generate",
+ "execMs": 0.01699999999254942
}
]
},
{
- "roundMs": 1339.1742080000113,
- "firstDeltaMs": 1132.3141659999965,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "effects",
- "video_gen",
- "scene",
- "library",
- "side_panel",
- "top_bar"
- ],
- "declaredTools": 95,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "add_effect",
- "round": 0,
- "atMs": 1858.4160839999968,
- "execMs": 0.0029999999969732016,
- "args": {
- "end": 12.5,
- "start": 6,
- "amount": 0.5,
- "effect": "vhs"
- }
- }
- ],
- "timings": {
- "gateMs": 239.67337499998393,
- "gateSkipped": false,
- "ttftMs": 2937.9317089999968,
- "modelMs": 2925.514500999998,
- "toolMs": 0.0029999999969732016,
- "totalMs": 3166.4014170000155,
- "judgeMs": 265.204000000027,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1618.0305419999931,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_effect",
- "execMs": 0.0029999999969732016
- }
- ]
- },
- {
- "roundMs": 1307.4839590000047,
- "firstDeltaMs": 1079.0800839999865,
+ "roundMs": 1427.432375000004,
+ "firstDeltaMs": 1340.4036250000063,
"calls": []
}
]
@@ -3148,158 +2722,151 @@
]
},
{
- "name": "audio-effect-single-tool",
+ "name": "cartoon-from-audio-plans-scene",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 251,
- "p95": 356,
- "mean": 304,
- "n": 2
+ "p50": 437,
+ "p95": 437,
+ "mean": 437,
+ "n": 1
},
"ttftMs": {
- "p50": 2636,
- "p95": 2729,
- "mean": 2683,
- "n": 2
+ "p50": 27617,
+ "p95": 27617,
+ "mean": 27617,
+ "n": 1
},
"totalMs": {
- "p50": 2894,
- "p95": 2894,
- "mean": 2894,
- "n": 2
+ "p50": 28057,
+ "p95": 28057,
+ "mean": 28057,
+ "n": 1
},
"modelMs": {
- "p50": 2534,
- "p95": 2640,
- "mean": 2587,
- "n": 2
+ "p50": 27607,
+ "p95": 27607,
+ "mean": 27607,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 4,
+ "max": 4
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "scene-productions",
"areas": [
"preview",
"timeline",
"inspector",
+ "color",
+ "removal",
+ "elements",
"effects",
+ "animation",
+ "transitions",
+ "subtitles",
"audio",
- "side_panel"
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
+ "editor"
],
- "declaredTools": 73,
+ "declaredTools": 128,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_clip_sound",
+ "name": "listen_audio",
"round": 0,
- "atMs": 1633.934459000011,
- "execMs": 0.11125000001629815,
+ "atMs": 3699.636708999984,
+ "execMs": 0.06904200001736172,
"args": {
- "clipId": "c2",
- "preset": "phone"
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "get_state",
+ "round": 1,
+ "atMs": 6497.891041999974,
+ "execMs": 0.08341700001619756,
+ "args": {}
+ },
+ {
+ "name": "generate_scene",
+ "round": 2,
+ "atMs": 10014.750249999983,
+ "execMs": 0.09912499997881241,
+ "args": {
+ "style": "smooth 2D cartoon animation, clean line art, vibrant flat colors, fluid traditional animation movement",
+ "brief": "A story following the speaker's narration, rendered as a smooth 2D cartoon.",
+ "from_audio_asset_id": "a-au1"
}
}
],
"timings": {
- "gateMs": 250.75916599997436,
+ "gateMs": 436.6330420000013,
"gateSkipped": false,
- "ttftMs": 2729.4537919999857,
- "modelMs": 2640.348874999967,
- "toolMs": 0.11125000001629815,
- "totalMs": 2893.5675420000043,
- "judgeMs": 215.6016670000099,
+ "ttftMs": 27616.98225,
+ "modelMs": 27606.593416000018,
+ "toolMs": 0.2515840000123717,
+ "totalMs": 28056.71645899999,
+ "judgeMs": 782.0522090000159,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1381.5887089999742,
+ "roundMs": 3259.5898330000055,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_sound",
- "execMs": 0.11125000001629815
+ "name": "listen_audio",
+ "execMs": 0.06904200001736172
}
]
},
{
- "roundMs": 1258.7601659999928,
- "firstDeltaMs": 1094.755749999982,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "audio",
- "side_panel"
- ],
- "declaredTools": 73,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_clip_sound",
- "round": 0,
- "atMs": 1617.3443329999864,
- "execMs": 0.021084000007249415,
- "args": {
- "preset": "phone",
- "clipId": "c2"
- }
- }
- ],
- "timings": {
- "gateMs": 356.3399999999965,
- "gateSkipped": false,
- "ttftMs": 2636.099332999991,
- "modelMs": 2534.3065840000054,
- "toolMs": 0.021084000007249415,
- "totalMs": 2893.990832999989,
- "judgeMs": 229.26095799999894,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 2796.185375000001,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "get_state",
+ "execMs": 0.08341700001619756
+ }
+ ]
+ },
{
- "roundMs": 1259.4970420000027,
+ "roundMs": 3511.175749999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_sound",
- "execMs": 0.021084000007249415
+ "name": "generate_scene",
+ "execMs": 0.09912499997881241
}
]
},
{
- "roundMs": 1274.8095420000027,
- "firstDeltaMs": 1016.9855420000094,
+ "roundMs": 18039.642458000017,
+ "firstDeltaMs": 17600.159249999997,
"calls": []
}
]
@@ -3308,162 +2875,134 @@
]
},
{
- "name": "sound-dissolve-single-tool",
+ "name": "cartoon-korean-audio-passes-language",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 225,
- "p95": 296,
- "mean": 261,
- "n": 2
+ "p50": 524,
+ "p95": 524,
+ "mean": 524,
+ "n": 1
},
"ttftMs": {
- "p50": 2623,
- "p95": 2923,
- "mean": 2773,
- "n": 2
+ "p50": 10025,
+ "p95": 10025,
+ "mean": 10025,
+ "n": 1
},
"totalMs": {
- "p50": 2867,
- "p95": 3142,
- "mean": 3005,
- "n": 2
+ "p50": 10196,
+ "p95": 10196,
+ "mean": 10196,
+ "n": 1
},
"modelMs": {
- "p50": 2640,
- "p95": 2844,
- "mean": 2742,
- "n": 2
+ "p50": 9663,
+ "p95": 9663,
+ "mean": 9663,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "scene-productions",
"areas": [
"preview",
"timeline",
"inspector",
+ "removal",
+ "elements",
"effects",
+ "animation",
"transitions",
+ "subtitles",
"audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
"editor"
],
- "declaredTools": 70,
+ "declaredTools": 121,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "listen_audio",
"round": 0,
- "atMs": 1747.8170419999806,
- "execMs": 0.07725000000209548,
+ "atMs": 4475.160665999982,
+ "execMs": 0.034499999979743734,
"args": {
- "clipId": "c1",
- "style": "audiocross",
- "seconds": 0.3
+ "asset_id": "a-au1"
+ }
+ },
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 7409.686082999979,
+ "execMs": 0.46141599997645244,
+ "args": {
+ "from_audio_asset_id": "a-au1",
+ "audio_language": "ko-KR",
+ "brief": "A charming and smooth story animated to the Korean narration audio. Flat hand-drawn 2D cartoon animation, fluid motion, clean outlines, vibrant soft color palette.",
+ "style": "smooth 2D cartoon animation, flat clean outlines, fluid character motion, vibrant and soft color palette"
}
}
],
"timings": {
- "gateMs": 296.3960420000076,
+ "gateMs": 523.9296669999894,
"gateSkipped": false,
- "ttftMs": 2922.7595419999852,
- "modelMs": 2843.9709999999905,
- "toolMs": 0.07725000000209548,
- "totalMs": 3141.900667000009,
- "judgeMs": 302.0467920000083,
+ "ttftMs": 10025.095915999991,
+ "modelMs": 9662.929749000032,
+ "toolMs": 0.4959159999561962,
+ "totalMs": 10196.3125,
+ "judgeMs": 562.0726250000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1450.6633750000037,
+ "roundMs": 3947.449708,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.07725000000209548
+ "name": "listen_audio",
+ "execMs": 0.034499999979743734
}
]
},
{
- "roundMs": 1393.3076249999867,
- "firstDeltaMs": 1174.4249589999963,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "transitions",
- "audio",
- "editor"
- ],
- "declaredTools": 70,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 1571.1429999999818,
- "execMs": 0.006624999979976565,
- "args": {
- "clipId": "c1",
- "seconds": 0.3,
- "style": "audiocross"
- }
- }
- ],
- "timings": {
- "gateMs": 225.32741700002225,
- "gateSkipped": false,
- "ttftMs": 2623.195624999993,
- "modelMs": 2639.6257500000065,
- "toolMs": 0.006624999979976565,
- "totalMs": 2867.185999999987,
- "judgeMs": 279.27512499998556,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1344.689000000013,
+ "roundMs": 2932.496833000012,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.006624999979976565
+ "name": "generate_scene",
+ "execMs": 0.46141599997645244
}
]
},
{
- "roundMs": 1294.9367499999935,
- "firstDeltaMs": 1051.1058339999872,
+ "roundMs": 2782.9832080000197,
+ "firstDeltaMs": 2612.181292000023,
"calls": []
}
]
@@ -3472,194 +3011,230 @@
]
},
{
- "name": "animation-overrides-transition",
+ "name": "cartoon-style-reference-anchors-look",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 255,
- "p95": 301,
- "mean": 278,
- "n": 2
+ "p50": 306,
+ "p95": 306,
+ "mean": 306,
+ "n": 1
},
"ttftMs": {
- "p50": 4192,
- "p95": 6068,
- "mean": 5130,
- "n": 2
+ "p50": 8790,
+ "p95": 8790,
+ "mean": 8790,
+ "n": 1
},
"totalMs": {
- "p50": 4350,
- "p95": 6422,
- "mean": 5386,
- "n": 2
+ "p50": 9161,
+ "p95": 9161,
+ "mean": 9161,
+ "n": 1
},
"modelMs": {
- "p50": 4091,
- "p95": 6119,
- "mean": 5105,
- "n": 2
+ "p50": 8847,
+ "p95": 8847,
+ "mean": 8847,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2.5,
+ "mean": 3,
"max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "scene-productions",
"areas": [
"preview",
"timeline",
"inspector",
+ "color",
+ "removal",
+ "elements",
+ "effects",
"animation",
"transitions",
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar",
"editor"
],
- "declaredTools": 67,
+ "declaredTools": 128,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "remove_transition",
+ "name": "listen_audio",
"round": 0,
- "atMs": 1751.9912089999998,
- "execMs": 0.09441600000718608,
+ "atMs": 3228.2281660000153,
+ "execMs": 0.015541000000666827,
"args": {
- "transitionId": "tr-c1"
+ "asset_id": "a-au1"
}
},
{
- "name": "set_animation",
+ "name": "generate_scene",
"round": 1,
- "atMs": 3176.6528340000077,
- "execMs": 0.0044169999891892076,
+ "atMs": 6621.747415999998,
+ "execMs": 0.22937500002444722,
"args": {
- "clipId": "c2",
- "style": "pop",
- "seconds": 0.5,
- "which": "in"
- }
- }
- ],
+ "style": "Colorful 2D cartoon animation matching the illustration style of dog-park.jpg, clean linework, vibrant friendly colors, expressive characters",
+ "from_audio_asset_id": "a-au1",
+ "reference_asset_ids": [
+ "a-i1"
+ ],
+ "brief": "A charming cartoon story following the narration, featuring playful animated dogs in a colorful park and beach setting with clean lines, expressive character animation, and vibrant colors."
+ }
+ }
+ ],
"timings": {
- "gateMs": 254.50633299996844,
- "gateSkipped": false,
- "ttftMs": 4192.076667000016,
- "modelMs": 4090.9922099999967,
- "toolMs": 0.09883299999637529,
- "totalMs": 4349.64837499999,
- "judgeMs": 589.4487920000101,
+ "gateMs": 305.56641699999454,
+ "gateSkipped": true,
+ "ttftMs": 8790.16737499999,
+ "modelMs": 8846.805790999992,
+ "toolMs": 0.24491600002511404,
+ "totalMs": 9160.907750000013,
+ "judgeMs": 519.856375000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1495.0590840000077,
+ "roundMs": 2919.6338749999995,
"firstDeltaMs": null,
"calls": [
{
- "name": "remove_transition",
- "execMs": 0.09441600000718608
+ "name": "listen_audio",
+ "execMs": 0.015541000000666827
}
]
},
{
- "roundMs": 1423.5612919999985,
+ "roundMs": 3390.7651250000054,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.0044169999891892076
+ "name": "generate_scene",
+ "execMs": 0.22937500002444722
}
]
},
{
- "roundMs": 1172.3718339999905,
- "firstDeltaMs": 1014.9619999999995,
+ "roundMs": 2536.4067909999867,
+ "firstDeltaMs": 2165.980958,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "cartoon-question-stays-in-chat",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 531,
+ "p95": 531,
+ "mean": 531,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 2651,
+ "p95": 2651,
+ "mean": 2651,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 4200,
+ "p95": 4200,
+ "mean": 4200,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 3665,
+ "p95": 3665,
+ "mean": 3665,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 1,
+ "max": 1
+ }
+ },
+ "runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "scene-productions",
"areas": [
"preview",
"timeline",
"inspector",
+ "removal",
+ "elements",
+ "effects",
"animation",
"transitions",
- "editor"
+ "subtitles",
+ "audio",
+ "image_gen",
+ "video_gen",
+ "scene",
+ "stock",
+ "library",
+ "side_panel",
+ "top_bar"
],
- "declaredTools": 67,
+ "declaredTools": 117,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "remove_transition",
- "round": 0,
- "atMs": 2244.4309999999823,
- "execMs": 0.0015410000341944396,
- "args": {
- "transitionId": "c1-transition-0"
- }
- },
- {
- "name": "set_animation",
- "round": 0,
- "atMs": 2244.4865839999984,
- "execMs": 0.000792000035289675,
- "args": {
- "style": "pop",
- "clipId": "c2",
- "seconds": 0.2,
- "which": "in"
- }
- }
- ],
+ "instant": null,
+ "judgeNote": "concise factual 0.35; long 2.0",
+ "trace": [],
"timings": {
- "gateMs": 301.0832919999957,
+ "gateMs": 531.2116249999963,
"gateSkipped": false,
- "ttftMs": 6067.817124999943,
- "modelMs": 6119.131208999956,
- "toolMs": 0.0023330000694841146,
- "totalMs": 6421.992083999969,
- "judgeMs": 509.14308300003177,
+ "ttftMs": 2650.5627919999824,
+ "modelMs": 3664.653541000007,
+ "toolMs": 0,
+ "totalMs": 4199.933374999993,
+ "judgeMs": 263.8937080000178,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1942.0425839999807,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.0015410000341944396
- },
- {
- "name": "set_animation",
- "execMs": 0.000792000035289675
- }
- ]
- },
- {
- "roundMs": 4177.088624999975,
- "firstDeltaMs": 3822.990208000003,
+ "roundMs": 3664.653541000007,
+ "firstDeltaMs": 2116.5879159999895,
"calls": []
}
]
@@ -3668,51 +3243,51 @@
]
},
{
- "name": "filler-words-cut-with-editor",
+ "name": "clip-mentions-redo-their-shots",
"bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 218,
- "p95": 290,
- "mean": 254,
- "n": 2
+ "p50": 289,
+ "p95": 289,
+ "mean": 289,
+ "n": 1
},
"ttftMs": {
- "p50": 32285,
- "p95": 38799,
- "mean": 35542,
- "n": 2
+ "p50": 13125,
+ "p95": 13125,
+ "mean": 13125,
+ "n": 1
},
"totalMs": {
- "p50": 32550,
- "p95": 39155,
- "mean": 35852,
- "n": 2
+ "p50": 13580,
+ "p95": 13580,
+ "mean": 13580,
+ "n": 1
},
"modelMs": {
- "p50": 32237,
- "p95": 38907,
- "mean": 35572,
- "n": 2
+ "p50": 13266,
+ "p95": 13266,
+ "mean": 13266,
+ "n": 1
},
"toolMs": {
"p50": 1,
"p95": 1,
"mean": 1,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 16,
- "max": 17
+ "mean": 6,
+ "max": 6
}
},
"runs": [
{
"pass": true,
"intent": "complex",
- "skill": "watching-and-cutting",
+ "skill": "audio-and-subtitles",
"areas": [
"preview",
"timeline",
@@ -3720,2652 +3295,185 @@
"transitions",
"subtitles",
"audio",
+ "video_gen",
+ "scene",
+ "stock",
"library",
"side_panel",
- "top_bar",
"editor"
],
- "declaredTools": 100,
+ "declaredTools": 107,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "upsell 0.54",
+ "instant": null,
+ "judgeNote": null,
"trace": [
{
- "name": "find_filler",
- "round": 1,
- "atMs": 7303.833249999967,
- "execMs": 0.2982499999925494,
- "args": {
- "clip_id": "c1",
- "track": 0
- }
- },
- {
- "name": "detect_silence",
- "round": 2,
- "atMs": 10485.969624999969,
- "execMs": 0.019415999995544553,
+ "name": "listen_audio",
+ "round": 0,
+ "atMs": 2591.3057499999995,
+ "execMs": 0.5308329999970738,
"args": {
"to": 6,
- "clip_id": "c1",
+ "clip_id": "au-n",
"from": 0
}
},
{
"name": "watch_video",
- "round": 3,
- "atMs": 12500.72104199999,
- "execMs": 0.029291999991983175,
+ "round": 0,
+ "atMs": 2592.829792000004,
+ "execMs": 0.025999999983469024,
"args": {
- "clip_id": "c1",
- "from": 0,
- "interval_seconds": 1,
- "to": 6
+ "clip_id": "sc-1"
}
},
{
"name": "note_source",
- "round": 4,
- "atMs": 14568.631832999992,
- "execMs": 0.014416999998502433,
+ "round": 1,
+ "atMs": 5092.499958,
+ "execMs": 0.021791999984998256,
"args": {
- "clip_id": "c1",
+ "clip_id": "sc-1",
"notes": [
{
+ "to": 3,
"from": 0,
- "to": 6,
- "text": "Speaker by the beach talking with dogs in view"
+ "text": "Shot 1 take showing scene 1 visual"
}
]
}
},
{
- "name": "select",
- "round": 5,
- "atMs": 16454.25408299995,
- "execMs": 0.024625000020023435,
- "args": {
- "kind": "clip",
- "id": "c1"
- }
- },
- {
- "name": "trim_clip",
- "round": 6,
- "atMs": 18193.951166999992,
- "execMs": 0.5514580000308342,
- "args": {
- "clipId": "c1",
- "in": 0.45
- }
- },
- {
- "name": "split_at",
- "round": 7,
- "atMs": 19819.45170799998,
- "execMs": 0.08112499996786937,
- "args": {
- "t": 4.1
- }
- },
- {
- "name": "trim_clip",
- "round": 8,
- "atMs": 21819.94316699996,
- "execMs": 0.018207999994046986,
- "args": {
- "in": 5.6,
- "clipId": "c1-2"
- }
- },
- {
- "name": "get_state",
- "round": 9,
- "atMs": 24824.277000000002,
- "execMs": 0.14995900000212714,
- "args": {}
- },
- {
- "name": "refine_speech_cuts",
- "round": 10,
- "atMs": 26635.33041699999,
- "execMs": 0.03062500001396984,
- "args": {
- "clip_ids": [
- "c1-1",
- "c1-2"
- ],
- "pace": "natural"
- }
- },
- {
- "name": "update_cue",
- "round": 11,
- "atMs": 28624.792749999964,
- "execMs": 0.013500000000931323,
+ "name": "watch_video",
+ "round": 1,
+ "atMs": 5092.804499999998,
+ "execMs": 0.004249999998137355,
"args": {
- "end": 1.8,
- "text": "So today we're",
- "id": "cue1",
- "start": 0
+ "clip_id": "sc-2"
}
},
{
- "name": "align_to_audio",
- "round": 12,
- "atMs": 30336.36599999998,
- "execMs": 0.006457999988924712,
+ "name": "note_source",
+ "round": 2,
+ "atMs": 7075.75787500001,
+ "execMs": 0.020499999984167516,
"args": {
- "track": 0,
- "target": "captions"
+ "clip_id": "sc-2",
+ "notes": [
+ {
+ "text": "Shot 2 take showing scene 2 visual",
+ "from": 0,
+ "to": 3
+ }
+ ]
}
},
{
"name": "get_state",
- "round": 13,
- "atMs": 32870.604082999984,
- "execMs": 0.09083299996564165,
+ "round": 2,
+ "atMs": 7076.017167000013,
+ "execMs": 0.01737499999580905,
"args": {}
},
{
- "name": "delete_cue",
- "round": 14,
- "atMs": 34794.35333299998,
- "execMs": 0.061999999976251274,
+ "name": "regenerate_shot",
+ "round": 3,
+ "atMs": 9038.298708000017,
+ "execMs": 0.02412499999627471,
"args": {
- "id": "cue3"
+ "note": "Match narration for first segment with distinct visual composition",
+ "n": 1
}
},
{
- "name": "listen_audio",
- "round": 15,
- "atMs": 36722.856582999986,
- "execMs": 0.021999999997206032,
+ "name": "regenerate_shot",
+ "round": 4,
+ "atMs": 11026.740292000002,
+ "execMs": 0.03508400000282563,
"args": {
- "from": 3.5,
- "to": 4.55,
- "clip_id": "c1-1"
+ "note": "Match narration for second segment with distinct visual composition from shot 1",
+ "n": 2
}
}
],
"timings": {
- "gateMs": 217.8545419999864,
- "gateSkipped": false,
- "ttftMs": 38799.28816699999,
- "modelMs": 38907.04075100011,
- "toolMs": 1.4121659999364056,
- "totalMs": 39155.218957999954,
- "judgeMs": 582.8360840000096,
+ "gateMs": 289.43787500000326,
+ "gateSkipped": true,
+ "ttftMs": 13125.493792000023,
+ "modelMs": 13266.17620799999,
+ "toolMs": 0.6799589999427553,
+ "totalMs": 13580.435958000016,
+ "judgeMs": 560.3659170000174,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1595.4662499999977,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 5485.319834000024,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "find_filler",
- "execMs": 0.2982499999925494
- }
- ]
- },
- {
- "roundMs": 3179.2717909999774,
+ "roundMs": 2296.8954169999924,
"firstDeltaMs": null,
"calls": [
{
- "name": "detect_silence",
- "execMs": 0.019415999995544553
- }
- ]
- },
- {
- "roundMs": 2012.2938750000321,
- "firstDeltaMs": null,
- "calls": [
+ "name": "listen_audio",
+ "execMs": 0.5308329999970738
+ },
{
"name": "watch_video",
- "execMs": 0.029291999991983175
+ "execMs": 0.025999999983469024
}
]
},
{
- "roundMs": 2065.308707999997,
+ "roundMs": 2495.436708000023,
"firstDeltaMs": null,
"calls": [
{
"name": "note_source",
- "execMs": 0.014416999998502433
- }
- ]
- },
- {
- "roundMs": 1882.5462919999845,
- "firstDeltaMs": null,
- "calls": [
+ "execMs": 0.021791999984998256
+ },
{
- "name": "select",
- "execMs": 0.024625000020023435
+ "name": "watch_video",
+ "execMs": 0.004249999998137355
}
]
},
{
- "roundMs": 1738.1774159999914,
+ "roundMs": 1980.9661669999768,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.5514580000308342
- }
- ]
- },
- {
- "roundMs": 1623.5362920000334,
- "firstDeltaMs": null,
- "calls": [
+ "name": "note_source",
+ "execMs": 0.020499999984167516
+ },
{
- "name": "split_at",
- "execMs": 0.08112499996786937
+ "name": "get_state",
+ "execMs": 0.01737499999580905
}
]
},
{
- "roundMs": 1999.857958000037,
+ "roundMs": 1956.2055830000027,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.018207999994046986
+ "name": "regenerate_shot",
+ "execMs": 0.02412499999627471
}
]
},
{
- "roundMs": 3003.715124999988,
+ "roundMs": 1985.682124999992,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.14995900000212714
+ "name": "regenerate_shot",
+ "execMs": 0.03508400000282563
}
]
},
{
- "roundMs": 1809.7702920000302,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.03062500001396984
- }
- ]
- },
- {
- "roundMs": 1988.8142089999747,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.013500000000931323
- }
- ]
- },
- {
- "roundMs": 1710.788750000007,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "align_to_audio",
- "execMs": 0.006457999988924712
- }
- ]
- },
- {
- "roundMs": 2533.3838749999995,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.09083299996564165
- }
- ]
- },
- {
- "roundMs": 1922.700708999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_cue",
- "execMs": 0.061999999976251274
- }
- ]
- },
- {
- "roundMs": 1925.6313749999972,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.021999999997206032
- }
- ]
- },
- {
- "roundMs": 2430.4580000000424,
- "firstDeltaMs": 2075.1564160000416,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "watching-and-cutting",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "transitions",
- "subtitles",
- "audio",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 100,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "find_filler",
- "round": 1,
- "atMs": 4396.68987500004,
- "execMs": 0.059084000007715076,
- "args": {
- "track": 0,
- "clip_id": "c1"
- }
- },
- {
- "name": "detect_silence",
- "round": 2,
- "atMs": 6149.900499999989,
- "execMs": 0.040542000031564385,
- "args": {
- "clip_id": "c1"
- }
- },
- {
- "name": "watch_video",
- "round": 3,
- "atMs": 8156.194791000045,
- "execMs": 0.017125000013038516,
- "args": {
- "clip_id": "c1",
- "to": 6,
- "interval_seconds": 1,
- "from": 0
- }
- },
- {
- "name": "note_source",
- "round": 4,
- "atMs": 10120.79916600004,
- "execMs": 0.017749999999068677,
- "args": {
- "asset_id": "a-v1",
- "notes": [
- {
- "from": 0,
- "text": "Woman at beach talking to camera with dogs in background",
- "to": 6
- }
- ]
- }
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 11832.032666000014,
- "execMs": 0.05033300002105534,
- "args": {
- "t": 4.55
- }
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 13592.981583000044,
- "execMs": 0.08833399997092783,
- "args": {
- "t": 5.6
- }
- },
- {
- "name": "trim_clip",
- "round": 7,
- "atMs": 15339.422750000027,
- "execMs": 0.055333999975118786,
- "args": {
- "clipId": "c1-1",
- "in": 0.45
- }
- },
- {
- "name": "delete_item",
- "round": 8,
- "atMs": 18217.488500000036,
- "execMs": 0.022624999983236194,
- "args": {
- "kind": "clip",
- "id": "c1-2-3"
- }
- },
- {
- "name": "get_state",
- "round": 9,
- "atMs": 19943.600416,
- "execMs": 0.0782500000204891,
- "args": {}
- },
- {
- "name": "refine_speech_cuts",
- "round": 10,
- "atMs": 21950.37174999999,
- "execMs": 0.04920799995306879,
- "args": {
- "pace": "natural",
- "clip_ids": [
- "c1-1",
- "c1-2-4"
- ]
- }
- },
- {
- "name": "update_cue",
- "round": 11,
- "atMs": 23760.78470800002,
- "execMs": 0.024000000033993274,
- "args": {
- "end": 1.35,
- "start": 0,
- "id": "cue1",
- "text": "so today we're"
- }
- },
- {
- "name": "get_state",
- "round": 12,
- "atMs": 28713.041333,
- "execMs": 0.0811659999890253,
- "args": {}
- },
- {
- "name": "update_cue",
- "round": 13,
- "atMs": 30496.30125000002,
- "execMs": 0.017583000008016825,
- "args": {
- "end": 4.1,
- "start": 4.1,
- "text": "and",
- "id": "cue3"
- }
- }
- ],
- "timings": {
- "gateMs": 289.78583300003083,
- "gateSkipped": false,
- "ttftMs": 32285.455958000035,
- "modelMs": 32237.16462699999,
- "toolMs": 0.6013340000063181,
- "totalMs": 32549.717833000002,
- "judgeMs": 500.9048329999787,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1260.811875000014,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2842.517625000037,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "find_filler",
- "execMs": 0.059084000007715076
- }
- ]
- },
- {
- "roundMs": 1751.425709000032,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.040542000031564385
- }
- ]
- },
- {
- "roundMs": 2004.4235419999459,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.017125000013038516
- }
- ]
- },
- {
- "roundMs": 1963.215749999974,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "note_source",
- "execMs": 0.017749999999068677
- }
- ]
- },
- {
- "roundMs": 1710.0438339999528,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.05033300002105534
- }
- ]
- },
- {
- "roundMs": 1759.844959000009,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.08833399997092783
- }
- ]
- },
- {
- "roundMs": 1744.530040999991,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "trim_clip",
- "execMs": 0.055333999975118786
- }
- ]
- },
- {
- "roundMs": 2876.2964999999967,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.022624999983236194
- }
- ]
- },
- {
- "roundMs": 1725.439167000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.0782500000204891
- }
- ]
- },
- {
- "roundMs": 2004.9705840000533,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.04920799995306879
- }
- ]
- },
- {
- "roundMs": 1808.8036250000005,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.024000000033993274
- }
- ]
- },
- {
- "roundMs": 4950.779749999987,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.0811659999890253
- }
- ]
- },
- {
- "roundMs": 1781.9015409999993,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_cue",
- "execMs": 0.017583000008016825
- }
- ]
- },
- {
- "roundMs": 2052.160124999995,
- "firstDeltaMs": 1788.3024160000496,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "edit-request-still-uses-tools",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 269,
- "p95": 291,
- "mean": 280,
- "n": 2
- },
- "ttftMs": {
- "p50": 2915,
- "p95": 5602,
- "mean": 4258,
- "n": 2
- },
- "totalMs": {
- "p50": 2930,
- "p95": 5753,
- "mean": 4342,
- "n": 2
- },
- "modelMs": {
- "p50": 2637,
- "p95": 5480,
- "mean": 4059,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "subtitles",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 96,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "subtitles_generate",
- "round": 0,
- "atMs": 4294.591166999948,
- "execMs": 0.00483300001360476,
- "args": {
- "track": 0
- }
- }
- ],
- "timings": {
- "gateMs": 269.41512499999953,
- "gateSkipped": false,
- "ttftMs": 5601.570207999961,
- "modelMs": 5480.3979999999865,
- "toolMs": 0.00483300001360476,
- "totalMs": 5753.372499999998,
- "judgeMs": 520.054833000002,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4022.840124999988,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_generate",
- "execMs": 0.00483300001360476
- }
- ]
- },
- {
- "roundMs": 1457.5578749999986,
- "firstDeltaMs": 1306.034290999989,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "subtitles",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 96,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "subtitles_generate",
- "round": 0,
- "atMs": 1606.8932919999934,
- "execMs": 0.006709000037517399,
- "args": {
- "track": 0
- }
- }
- ],
- "timings": {
- "gateMs": 290.6009159999667,
- "gateSkipped": false,
- "ttftMs": 2914.5904579999624,
- "modelMs": 2636.953916999977,
- "toolMs": 0.006709000037517399,
- "totalMs": 2929.628708000004,
- "judgeMs": 228.50675000000047,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1314.8173749999842,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_generate",
- "execMs": 0.006709000037517399
- }
- ]
- },
- {
- "roundMs": 1322.136541999993,
- "firstDeltaMs": 1307.2522920000483,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "cartoon-from-audio-plans-scene",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 275,
- "p95": 470,
- "mean": 373,
- "n": 2
- },
- "ttftMs": {
- "p50": 7286,
- "p95": 23363,
- "mean": 15325,
- "n": 2
- },
- "totalMs": {
- "p50": 7461,
- "p95": 23869,
- "mean": 15665,
- "n": 2
- },
- "modelMs": {
- "p50": 7051,
- "p95": 23588,
- "mean": 15319,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 4,
- "max": 4
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 127,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 4857.906708000053,
- "execMs": 0.07583400001749396,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 2,
- "atMs": 7902.72316600004,
- "execMs": 0.08183300000382587,
- "args": {
- "style": "Smooth 2D cartoon animation, hand-drawn vector aesthetic, clean lines, vibrant palette, fluid character animation",
- "brief": "A smooth, fluid 2D hand-drawn animated cartoon illustrating the narration: clean bold outlines, vibrant flat colors, rounded friendly character shapes, continuous animated movement.",
- "from_audio_asset_id": "a-au1"
- }
- }
- ],
- "timings": {
- "gateMs": 274.9295830000192,
- "gateSkipped": false,
- "ttftMs": 23363.34275000001,
- "modelMs": 23587.79954300006,
- "toolMs": 0.15766700002131984,
- "totalMs": 23869.057291000034,
- "judgeMs": 579.0723330000183,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1727.5787920000148,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2850.7872920000227,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.07583400001749396
- }
- ]
- },
- {
- "roundMs": 3043.916917000024,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.08183300000382587
- }
- ]
- },
- {
- "roundMs": 15965.516541999998,
- "firstDeltaMs": 15459.97320800001,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 127,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 2976.7070419999654,
- "execMs": 0.0313750000204891,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 2,
- "atMs": 5711.230125000002,
- "execMs": 0.005917000002227724,
- "args": {
- "from_audio_asset_id": "a-au1",
- "brief": "A charming story following the narration, animated as smooth 2D cartoon animation with clean ink line art, soft vibrant colors, fluid hand-drawn motion, and consistent character design.",
- "style": "Smooth 2D cartoon animation, hand-drawn vector art style, clean linework, vibrant flat colors, fluid motion"
- }
- }
- ],
- "timings": {
- "gateMs": 470.4172920000274,
- "gateSkipped": false,
- "ttftMs": 7285.673084000009,
- "modelMs": 7050.987667999987,
- "toolMs": 0.03729200002271682,
- "totalMs": 7460.546999999962,
- "judgeMs": 555.2023749999935,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 66.9012090000324,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2504.117499999993,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.0313750000204891
- }
- ]
- },
- {
- "roundMs": 2732.1188749999856,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.005917000002227724
- }
- ]
- },
- {
- "roundMs": 1747.850083999976,
- "firstDeltaMs": 1573.3913339999854,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "cartoon-korean-audio-passes-language",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 319,
- "p95": 320,
- "mean": 319,
- "n": 2
- },
- "ttftMs": {
- "p50": 11762,
- "p95": 12573,
- "mean": 12168,
- "n": 2
- },
- "totalMs": {
- "p50": 12041,
- "p95": 12908,
- "mean": 12474,
- "n": 2
- },
- "modelMs": {
- "p50": 11714,
- "p95": 12580,
- "mean": 12147,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 4,
- "max": 4
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 120,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 4272.4063750000205,
- "execMs": 0.009041999990586191,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 2,
- "atMs": 7042.81270900002,
- "execMs": 0.3398750000051223,
- "args": {
- "audio_language": "ko-KR",
- "from_audio_asset_id": "a-au1",
- "style": "smooth 2D cartoon animation, clean line art, vibrant pastel colors, fluid motion",
- "brief": "A smooth 2D cartoon animation visualizing the Korean narration about a dog's adventure in a park and beach. Clean lines, charming rounded 2D animated characters, soft vibrant colors, fluid cartoon motion."
- }
- }
- ],
- "timings": {
- "gateMs": 319.008833000029,
- "gateSkipped": false,
- "ttftMs": 11761.898417000019,
- "modelMs": 11713.877665999986,
- "toolMs": 0.34891699999570847,
- "totalMs": 12041.062666999991,
- "judgeMs": 539.3162920000032,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1689.5204999999842,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2259.5719999999856,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.009041999990586191
- }
- ]
- },
- {
- "roundMs": 2769.332875000022,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.3398750000051223
- }
- ]
- },
- {
- "roundMs": 4995.452290999994,
- "firstDeltaMs": 4716.576457999996,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 120,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 1,
- "atMs": 6614.068084000028,
- "execMs": 0.019958999997470528,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 2,
- "atMs": 8982.310875000025,
- "execMs": 0.013042000005953014,
- "args": {
- "style": "smooth 2D cartoon animation, clean line art, vibrant colors, fluid character motion",
- "from_audio_asset_id": "a-au1",
- "brief": "A smooth 2D cartoon animation depicting the story narrated in the audio, featuring expressive characters, clean linework, vibrant colors, and fluid motion.",
- "audio_language": "ko-KR"
- }
- }
- ],
- "timings": {
- "gateMs": 319.92545899999095,
- "gateSkipped": false,
- "ttftMs": 12573.294542000047,
- "modelMs": 12580.106417999952,
- "toolMs": 0.03300100000342354,
- "totalMs": 12907.581750000012,
- "judgeMs": 569.1341249999823,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1661.9575839999598,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 4627.9814590000315,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.019958999997470528
- }
- ]
- },
- {
- "roundMs": 2366.469333999965,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.013042000005953014
- }
- ]
- },
- {
- "roundMs": 3923.698040999996,
- "firstDeltaMs": 3589.924874999968,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "cartoon-style-reference-anchors-look",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 252,
- "p95": 260,
- "mean": 256,
- "n": 2
- },
- "ttftMs": {
- "p50": 5897,
- "p95": 8931,
- "mean": 7414,
- "n": 2
- },
- "totalMs": {
- "p50": 6194,
- "p95": 9198,
- "mean": 7696,
- "n": 2
- },
- "modelMs": {
- "p50": 5928,
- "p95": 8937,
- "mean": 7432,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 3,
- "max": 3
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 127,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 3098.703125,
- "execMs": 0.04220800002804026,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 1,
- "atMs": 6774.4283339999965,
- "execMs": 0.1091249999590218,
- "args": {
- "style": "flat hand-drawn 2D cartoon animation, colorful storybook illustration style, clean vibrant outlines, lively fluid motion",
- "from_audio_asset_id": "a-au1",
- "reference_asset_ids": [
- "a-i1"
- ]
- }
- }
- ],
- "timings": {
- "gateMs": 251.5820839999942,
- "gateSkipped": true,
- "ttftMs": 8930.976375000027,
- "modelMs": 8936.575916999951,
- "toolMs": 0.15133299998706207,
- "totalMs": 9198.011667000013,
- "judgeMs": 587.8315410000505,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2842.6163329999545,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.04220800002804026
- }
- ]
- },
- {
- "roundMs": 3672.175458999991,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.1091249999590218
- }
- ]
- },
- {
- "roundMs": 2421.7841250000056,
- "firstDeltaMs": 2154.885249999992,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 127,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 1975.7431670000078,
- "execMs": 0.006290999997872859,
- "args": {
- "asset_id": "a-au1"
- }
- },
- {
- "name": "generate_scene",
- "round": 1,
- "atMs": 4181.843500000017,
- "execMs": 0.011375000001862645,
- "args": {
- "style": "vibrant 2D cartoon animation style matching the attached illustration",
- "reference_asset_ids": [
- "a-i1"
- ],
- "from_audio_asset_id": "a-au1"
- }
- }
- ],
- "timings": {
- "gateMs": 260.4637910000165,
- "gateSkipped": true,
- "ttftMs": 5897.35820800002,
- "modelMs": 5928.399665999983,
- "toolMs": 0.017665999999735504,
- "totalMs": 6194.27816700004,
- "judgeMs": 574.2261669999571,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1712.2283329999773,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.006290999997872859
- }
- ]
- },
- {
- "roundMs": 2205.2085829999996,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_scene",
- "execMs": 0.011375000001862645
- }
- ]
- },
- {
- "roundMs": 2010.962750000006,
- "firstDeltaMs": 1714.131457999989,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "cartoon-question-stays-in-chat",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 210,
- "p95": 246,
- "mean": 228,
- "n": 2
- },
- "ttftMs": {
- "p50": 3343,
- "p95": 6764,
- "mean": 5053,
- "n": 2
- },
- "totalMs": {
- "p50": 4215,
- "p95": 8147,
- "mean": 6181,
- "n": 2
- },
- "modelMs": {
- "p50": 4001,
- "p95": 7895,
- "mean": 5948,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 120,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "concise factual 0.46; long 2.0",
- "trace": [],
- "timings": {
- "gateMs": 246.42304099997273,
- "gateSkipped": false,
- "ttftMs": 6763.740624999977,
- "modelMs": 7895.323125000054,
- "toolMs": 0,
- "totalMs": 8147.037375000014,
- "judgeMs": 486.8101669999887,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4289.968167000043,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 3605.354958000011,
- "firstDeltaMs": 2222.187957999995,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "scene-productions",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "removal",
- "elements",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "image_gen",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 120,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 210.49154099996667,
- "gateSkipped": false,
- "ttftMs": 3342.778667000006,
- "modelMs": 4000.9623339999234,
- "toolMs": 0,
- "totalMs": 4215.245208000008,
- "judgeMs": 598.4915000000037,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1541.7603749999544,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2459.201958999969,
- "firstDeltaMs": 1587.3116670000018,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "clip-mentions-redo-their-shots",
- "bucket": "multi-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 245,
- "p95": 256,
- "mean": 251,
- "n": 2
- },
- "ttftMs": {
- "p50": 19034,
- "p95": 26135,
- "mean": 22584,
- "n": 2
- },
- "totalMs": {
- "p50": 19271,
- "p95": 26690,
- "mean": 22980,
- "n": 2
- },
- "modelMs": {
- "p50": 19020,
- "p95": 26415,
- "mean": 22717,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 7.5,
- "max": 10
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "audio-and-subtitles",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "transitions",
- "subtitles",
- "audio",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "editor"
- ],
- "declaredTools": 106,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 2614.5971659999923,
- "execMs": 0.10375000000931323,
- "args": {
- "to": 6,
- "clip_id": "au-n",
- "from": 0
- }
- },
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 2614.805083000043,
- "execMs": 0.002582999994046986,
- "args": {
- "clip_id": "sc-1"
- }
- },
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 2614.8421250000247,
- "execMs": 0.0005830000154674053,
- "args": {
- "clip_id": "sc-2"
- }
- },
- {
- "name": "note_source",
- "round": 1,
- "atMs": 5099.87737500004,
- "execMs": 0.0039160000160336494,
- "args": {
- "clip_id": "sc-1",
- "notes": [
- {
- "text": "Shot 1 of generated scene: dog running in park/field",
- "to": 3,
- "from": 0
- }
- ]
- }
- },
- {
- "name": "note_source",
- "round": 1,
- "atMs": 5099.935500000021,
- "execMs": 0.0006249999860301614,
- "args": {
- "clip_id": "sc-2",
- "notes": [
- {
- "to": 3,
- "text": "Shot 2 of generated scene: very similar angle of dog running in park/field",
- "from": 0
- }
- ]
- }
- },
- {
- "name": "get_state",
- "round": 2,
- "atMs": 7577.166541000013,
- "execMs": 0.008750000037252903,
- "args": {}
- },
- {
- "name": "regenerate_shot",
- "round": 3,
- "atMs": 10322.591208000027,
- "execMs": 0.009959000046364963,
- "args": {
- "note": "Close-up reaction shot that matches the narration",
- "n": 2
- }
- }
- ],
- "timings": {
- "gateMs": 245.41316700004973,
- "gateSkipped": true,
- "ttftMs": 19034.05487500003,
- "modelMs": 19020.037291000015,
- "toolMs": 0.1301660001045093,
- "totalMs": 19271.390000000014,
- "judgeMs": 577.3008750000154,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2368.2152079999796,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.10375000000931323
- },
- {
- "name": "watch_video",
- "execMs": 0.002582999994046986
- },
- {
- "name": "watch_video",
- "execMs": 0.0005830000154674053
- }
- ]
- },
- {
- "roundMs": 2484.524792000011,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "note_source",
- "execMs": 0.0039160000160336494
- },
- {
- "name": "note_source",
- "execMs": 0.0006249999860301614
- }
- ]
- },
- {
- "roundMs": 2476.674291000003,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.008750000037252903
- }
- ]
- },
- {
- "roundMs": 2743.2271670000046,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.009959000046364963
- }
- ]
- },
- {
- "roundMs": 8947.395833000017,
- "firstDeltaMs": 8710.480333000014,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "audio-and-subtitles",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "transitions",
- "subtitles",
- "audio",
- "video_gen",
- "scene",
- "stock",
- "library",
- "side_panel",
- "editor"
- ],
- "declaredTools": 107,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "listen_audio",
- "round": 0,
- "atMs": 7032.2287079999805,
- "execMs": 0.01087499997811392,
- "args": {
- "from": 0,
- "asset_id": "a-au1",
- "to": 6
- }
- },
- {
- "name": "watch_video",
- "round": 1,
- "atMs": 9419.263541999971,
- "execMs": 0.010249999992083758,
- "args": {
- "clip_id": "sc-1"
- }
- },
- {
- "name": "note_source",
- "round": 2,
- "atMs": 12498.851542000019,
- "execMs": 0.016582999960519373,
- "args": {
- "clip_id": "sc-1",
- "notes": [
- {
- "from": 0,
- "to": 3,
- "text": "Close-up shot of a golden retriever puppy looking at the camera on grass"
- }
- ]
- }
- },
- {
- "name": "watch_video",
- "round": 3,
- "atMs": 14469.211249999993,
- "execMs": 0.016583000018727034,
- "args": {
- "clip_id": "sc-2"
- }
- },
- {
- "name": "note_source",
- "round": 4,
- "atMs": 16545.08166700002,
- "execMs": 0.01554200000828132,
- "args": {
- "notes": [
- {
- "to": 3,
- "text": "Close-up shot of the same golden retriever puppy sitting in the grass looking at the camera",
- "from": 0
- }
- ],
- "clip_id": "sc-2"
- }
- },
- {
- "name": "get_state",
- "round": 5,
- "atMs": 18451.339666999993,
- "execMs": 0.0050830000545829535,
- "args": {}
- },
- {
- "name": "detect_silence",
- "round": 6,
- "atMs": 20426.178917000012,
- "execMs": 0.02245799999218434,
- "args": {
- "asset_id": "a-au1",
- "from": 0,
- "to": 9
- }
- },
- {
- "name": "regenerate_shot",
- "round": 7,
- "atMs": 22340.80920800002,
- "execMs": 0.028541000036057085,
- "args": {
- "note": "Golden retriever puppy running through the park playing fetch",
- "n": 1
- }
- },
- {
- "name": "regenerate_shot",
- "round": 8,
- "atMs": 24551.809917000006,
- "execMs": 0.025666999979875982,
- "args": {
- "note": "Golden retriever puppy running along the beach splashing in the shallow water",
- "n": 2
- }
- }
- ],
- "timings": {
- "gateMs": 255.9873329999973,
- "gateSkipped": true,
- "ttftMs": 26134.942541999975,
- "modelMs": 26414.743376999977,
- "toolMs": 0.15158200002042577,
- "totalMs": 26689.50004199997,
- "judgeMs": 539.7487500000279,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 6773.223625000042,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.01087499997811392
- }
- ]
- },
- {
- "roundMs": 2385.6892089999747,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.010249999992083758
- }
- ]
- },
- {
- "roundMs": 3077.8324169999687,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "note_source",
- "execMs": 0.016582999960519373
- }
- ]
- },
- {
- "roundMs": 1968.5407499999856,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.016583000018727034
- }
- ]
- },
- {
- "roundMs": 2073.7079579999554,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "note_source",
- "execMs": 0.01554200000828132
- }
- ]
- },
- {
- "roundMs": 1904.663125000021,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.0050830000545829535
- }
- ]
- },
- {
- "roundMs": 1973.7992920000106,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.02245799999218434
- }
- ]
- },
- {
- "roundMs": 1912.4533330000122,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.028541000036057085
- }
- ]
- },
- {
- "roundMs": 2208.867084000027,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.025666999979875982
- }
- ]
- },
- {
- "roundMs": 2135.96658399998,
- "firstDeltaMs": 1581.7539169999654,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "parked-debris-cleanup-after-retime",
- "bucket": "multi-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 228,
- "p95": 264,
- "mean": 246,
- "n": 2
- },
- "ttftMs": {
- "p50": 7665,
- "p95": 7873,
- "mean": 7769,
- "n": 2
- },
- "totalMs": {
- "p50": 8215,
- "p95": 8422,
- "mean": 8318,
- "n": 2
- },
- "modelMs": {
- "p50": 7948,
- "p95": 8190,
- "mean": 8069,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 3,
- "max": 3
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "library",
- "top_bar",
- "editor"
- ],
- "declaredTools": 77,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed",
- "round": 0,
- "atMs": 4707.735999999975,
- "execMs": 0.041333000001031905,
- "args": {
- "clipId": "c2",
- "speed": 2
- }
- },
- {
- "name": "remove_transition",
- "round": 1,
- "atMs": 6412.277999999991,
- "execMs": 0.0020840000361204147,
- "args": {
- "transitionId": "tr-2"
- }
- }
- ],
- "timings": {
- "gateMs": 264.01820799999405,
- "gateSkipped": false,
- "ttftMs": 7664.744999999995,
- "modelMs": 7948.246749999991,
- "toolMs": 0.04341700003715232,
- "totalMs": 8215.141708999989,
- "judgeMs": 641.9884580000071,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4441.949957999983,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed",
- "execMs": 0.041333000001031905
- }
- ]
- },
- {
- "roundMs": 1703.8798750000424,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.0020840000361204147
- }
- ]
- },
- {
- "roundMs": 1802.416916999966,
- "firstDeltaMs": 1252.0935839999584,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "top_bar",
- "editor"
- ],
- "declaredTools": 70,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed",
- "round": 0,
- "atMs": 1598.6035830000183,
- "execMs": 0.0044169999891892076,
- "args": {
- "speed": 2,
- "clipId": "c2"
- }
- },
- {
- "name": "remove_transition",
- "round": 1,
- "atMs": 3753.244958000025,
- "execMs": 0.00266700005158782,
- "args": {
- "transitionId": "tr-2"
- }
- }
- ],
- "timings": {
- "gateMs": 227.9594999999972,
- "gateSkipped": false,
- "ttftMs": 7873.432917000027,
- "modelMs": 8190.208416000009,
- "toolMs": 0.007084000040777028,
- "totalMs": 8421.521833000006,
- "judgeMs": 540.487958999991,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1369.1699999999837,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed",
- "execMs": 0.0044169999891892076
- }
- ]
- },
- {
- "roundMs": 2153.7447500000126,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.00266700005158782
- }
- ]
- },
- {
- "roundMs": 4667.293666000012,
- "firstDeltaMs": 4119.487374999968,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "speed-ramp-from-description",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 259,
- "p95": 283,
- "mean": 271,
- "n": 2
- },
- "ttftMs": {
- "p50": 4348,
- "p95": 9251,
- "mean": 6800,
- "n": 2
- },
- "totalMs": {
- "p50": 4502,
- "p95": 9478,
- "mean": 6990,
- "n": 2
- },
- "modelMs": {
- "p50": 4238,
- "p95": 9186,
- "mean": 6712,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 4.5,
- "max": 6
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "transitions",
- "library"
- ],
- "declaredTools": 69,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed_curve",
- "round": 0,
- "atMs": 1928.2015830000164,
- "execMs": 0.22504200000548735,
- "args": {
- "clipId": "c2",
- "nodes": [
- {
- "speed": 1,
- "at": 0
- },
- {
- "speed": 1,
- "at": 3
- },
- {
- "speed": 0.2,
- "at": 4.7
- }
- ],
- "smooth": true
- }
- },
- {
- "name": "set_speed_curve",
- "round": 1,
- "atMs": 3617.7700000000186,
- "execMs": 0.0035829999833367765,
- "args": {
- "nodes": [
- {
- "at": 0,
- "speed": 1
- },
- {
- "speed": 1,
- "at": 3
- },
- {
- "speed": 0.2,
- "at": 4.7
- }
- ],
- "clipId": "c2",
- "smooth": true
- }
- },
- {
- "name": "set_speed_curve",
- "round": 2,
- "atMs": 4972.047542000015,
- "execMs": 0.001959000015631318,
- "args": {
- "smooth": true,
- "nodes": [
- {
- "at": 0,
- "speed": 1
- },
- {
- "speed": 1,
- "at": 3
- },
- {
- "at": 4.7,
- "speed": 0.2
- }
- ],
- "clipId": "c2"
- }
- },
- {
- "name": "set_speed_curve",
- "round": 3,
- "atMs": 6293.7252920000465,
- "execMs": 0.002249999961350113,
- "args": {
- "smooth": true,
- "clipId": "c2",
- "nodes": [
- {
- "at": 0,
- "speed": 1
- },
- {
- "at": 3.5,
- "speed": 1
- },
- {
- "speed": 0.2,
- "at": 4.7
- }
- ]
- }
- },
- {
- "name": "set_speed_curve",
- "round": 4,
- "atMs": 8030.042667000031,
- "execMs": 0.003916999965440482,
- "args": {
- "nodes": [
- {
- "at": 0,
- "speed": 1
- },
- {
- "speed": 1,
- "at": 3.5
- },
- {
- "speed": 0.2,
- "at": 4.7
- }
- ],
- "smooth": true,
- "clipId": "c2"
- }
- }
- ],
- "timings": {
- "gateMs": 282.7062079999596,
- "gateSkipped": false,
- "ttftMs": 9251.170000000042,
- "modelMs": 9186.10312600003,
- "toolMs": 0.23675099993124604,
- "totalMs": 9477.76929200004,
- "judgeMs": 456.96745799999917,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1641.5506250000326,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.22504200000548735
- }
- ]
- },
- {
- "roundMs": 1687.6142080000136,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.0035829999833367765
- }
- ]
- },
- {
- "roundMs": 1353.634625000006,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.001959000015631318
- }
- ]
- },
- {
- "roundMs": 1321.1547089999658,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.002249999961350113
- }
- ]
- },
- {
- "roundMs": 1735.5676669999957,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.003916999965440482
- }
- ]
- },
- {
- "roundMs": 1446.5812920000171,
- "firstDeltaMs": 1220.2594169999938,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "transitions",
- "top_bar"
- ],
- "declaredTools": 66,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_speed_curve",
- "round": 0,
- "atMs": 1823.3552090000012,
- "execMs": 0.005791999981738627,
- "args": {
- "smooth": true,
- "clipId": "c2",
- "preset": "rampOut"
- }
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 3159.632209000003,
- "execMs": 0.026083000004291534,
- "args": {}
- }
- ],
- "timings": {
- "gateMs": 258.54383300000336,
- "gateSkipped": false,
- "ttftMs": 4348.372625000018,
- "modelMs": 4238.458873999887,
- "toolMs": 0.03187499998603016,
- "totalMs": 4501.532874999975,
- "judgeMs": 609.0904170001158,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1562.485415999894,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_speed_curve",
- "execMs": 0.005791999981738627
- }
- ]
- },
- {
- "roundMs": 1335.548375000013,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.026083000004291534
- }
- ]
- },
- {
- "roundMs": 1340.42508299998,
- "firstDeltaMs": 1187.7447500000708,
+ "roundMs": 2550.990208000003,
+ "firstDeltaMs": 2096.4137080000073,
"calls": []
}
]
@@ -6374,219 +3482,33 @@
]
},
{
- "name": "remove-grey-ones-scope",
+ "name": "parked-debris-cleanup-after-retime",
"bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 236,
- "p95": 251,
- "mean": 243,
- "n": 2
- },
- "ttftMs": {
- "p50": 2585,
- "p95": 2889,
- "mean": 2737,
- "n": 2
- },
- "totalMs": {
- "p50": 2739,
- "p95": 3056,
- "mean": 2897,
- "n": 2
- },
- "modelMs": {
- "p50": 2501,
- "p95": 2802,
- "mean": 2651,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "animation",
- "transitions",
- "editor"
- ],
- "declaredTools": 68,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "remove_transition",
- "round": 0,
- "atMs": 1734.201457999996,
- "execMs": 0.10962499992456287,
- "args": {
- "transitionId": "tr-8"
- }
- },
- {
- "name": "remove_transition",
- "round": 0,
- "atMs": 1734.420832999982,
- "execMs": 0.0014170000795274973,
- "args": {
- "transitionId": "tr-9"
- }
- }
- ],
- "timings": {
- "gateMs": 250.64108299999498,
- "gateSkipped": false,
- "ttftMs": 2888.526375000016,
- "modelMs": 2802.1011250000447,
- "toolMs": 0.11104200000409037,
- "totalMs": 3055.6086249999935,
- "judgeMs": 231.79179200006183,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1481.803791000042,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.10962499992456287
- },
- {
- "name": "remove_transition",
- "execMs": 0.0014170000795274973
- }
- ]
- },
- {
- "roundMs": 1320.2973340000026,
- "firstDeltaMs": 1153.4305419999873,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "animation",
- "transitions",
- "editor"
- ],
- "declaredTools": 68,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "remove_transition",
- "round": 0,
- "atMs": 1677.0437500000698,
- "execMs": 0.011959000024944544,
- "args": {
- "transitionId": "tr-8"
- }
- },
- {
- "name": "remove_transition",
- "round": 0,
- "atMs": 1677.1779589999933,
- "execMs": 0.0012080000014975667,
- "args": {
- "transitionId": "tr-9"
- }
- }
- ],
- "timings": {
- "gateMs": 235.84699999995064,
- "gateSkipped": false,
- "ttftMs": 2585.261625000043,
- "modelMs": 2500.598875000025,
- "toolMs": 0.01316700002644211,
- "totalMs": 2738.548375000013,
- "judgeMs": 225.56137500004843,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1440.2314999999944,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "remove_transition",
- "execMs": 0.011959000024944544
- },
- {
- "name": "remove_transition",
- "execMs": 0.0012080000014975667
- }
- ]
- },
- {
- "roundMs": 1060.3673750000307,
- "firstDeltaMs": 907.1628329999512,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "errored-call-not-claimed-done",
- "bucket": "single-tool",
- "passRate": 0.5,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 275,
- "p95": 275,
- "mean": 275,
+ "p50": 580,
+ "p95": 580,
+ "mean": 580,
"n": 1
},
"ttftMs": {
- "p50": 2879,
- "p95": 2879,
- "mean": 2879,
+ "p50": 6965,
+ "p95": 6965,
+ "mean": 6965,
"n": 1
},
"totalMs": {
- "p50": 3117,
- "p95": 3117,
- "mean": 3117,
+ "p50": 7433,
+ "p95": 7433,
+ "mean": 7433,
"n": 1
},
"modelMs": {
- "p50": 2839,
- "p95": 2839,
- "mean": 2839,
+ "p50": 6842,
+ "p95": 6842,
+ "mean": 6842,
"n": 1
},
"toolMs": {
@@ -6596,8 +3518,8 @@
"n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
@@ -6610,111 +3532,70 @@
"timeline",
"inspector",
"effects",
- "transitions"
- ],
- "declaredTools": 62,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_transition",
- "round": 0,
- "atMs": 1721.6676659999648,
- "execMs": 0.12704099994152784,
- "args": {
- "clipId": "c2",
- "style": "crossfade",
- "seconds": 0.5
- }
- }
- ],
- "timings": {
- "gateMs": 275.1437089999672,
- "gateSkipped": false,
- "ttftMs": 2879.027124999906,
- "modelMs": 2839.1242499999935,
- "toolMs": 0.12704099994152784,
- "totalMs": 3117.322832999984,
- "judgeMs": 230.7743339999579,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1444.8355410000077,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_transition",
- "execMs": 0.12704099994152784
- }
- ]
- },
- {
- "roundMs": 1394.2887089999858,
- "firstDeltaMs": 1156.2570840000408,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": false,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "transitions",
- "audio"
+ "top_bar",
+ "editor"
],
- "declaredTools": 66,
+ "declaredTools": 71,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "reply did not match /last clip|nothing after|no clip after|can('|no)t|couldn't|didn('|o)t/i"
- ],
+ "notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "set_speed",
"round": 0,
- "atMs": 1536.4541250000475,
- "execMs": 0.01158299995586276,
+ "atMs": 4214.487958999991,
+ "execMs": 0.11812500000814907,
"args": {
- "seconds": 0.5,
- "style": "crossfade",
- "clipId": "c2"
+ "clipId": "c2",
+ "speed": 2
+ }
+ },
+ {
+ "name": "remove_transition",
+ "round": 1,
+ "atMs": 5808.418999999994,
+ "execMs": 0.006374999997206032,
+ "args": {
+ "transitionId": "tr-2"
}
}
],
"timings": {
- "gateMs": 233.51983399991877,
+ "gateMs": 579.9407080000092,
"gateSkipped": false,
- "ttftMs": 2636.6545840000035,
- "modelMs": 2626.2835409999825,
- "toolMs": 0.01158299995586276,
- "totalMs": 2862.516834000009,
- "judgeMs": 252.18566600000486,
+ "ttftMs": 6965.281958999985,
+ "modelMs": 6842.032166999998,
+ "toolMs": 0.1245000000053551,
+ "totalMs": 7433.07362499999,
+ "judgeMs": 607.7482500000042,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1300.7304579999764,
+ "roundMs": 3629.110416999989,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.01158299995586276
+ "name": "set_speed",
+ "execMs": 0.11812500000814907
+ }
+ ]
+ },
+ {
+ "roundMs": 1590.9830829999992,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "remove_transition",
+ "execMs": 0.006374999997206032
}
]
},
{
- "roundMs": 1325.5530830000062,
- "firstDeltaMs": 1099.8327500000596,
+ "roundMs": 1621.9386670000094,
+ "firstDeltaMs": 1155.1537500000268,
"calls": []
}
]
@@ -6723,44 +3604,44 @@
]
},
{
- "name": "auto-continue-no-handoff",
- "bucket": "multi-tool",
+ "name": "speed-ramp-from-description",
+ "bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 301,
- "p95": 358,
- "mean": 329,
- "n": 2
+ "p50": 620,
+ "p95": 620,
+ "mean": 620,
+ "n": 1
},
"ttftMs": {
- "p50": 4262,
- "p95": 4426,
- "mean": 4344,
- "n": 2
+ "p50": 19592,
+ "p95": 19592,
+ "mean": 19592,
+ "n": 1
},
"totalMs": {
- "p50": 4531,
- "p95": 4728,
- "mean": 4629,
- "n": 2
+ "p50": 19752,
+ "p95": 19752,
+ "mean": 19752,
+ "n": 1
},
"modelMs": {
- "p50": 4167,
- "p95": 4425,
- "mean": 4296,
- "n": 2
+ "p50": 19103,
+ "p95": 19103,
+ "mean": 19103,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 13,
+ "max": 13
}
},
"runs": [
@@ -6772,307 +3653,440 @@
"preview",
"timeline",
"inspector",
- "subtitles",
- "library",
+ "effects",
+ "transitions",
"editor"
],
- "declaredTools": 85,
+ "declaredTools": 67,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "update_cue",
+ "name": "set_speed_curve",
"round": 0,
- "atMs": 2072.292832999956,
- "execMs": 0.1140420000301674,
+ "atMs": 2013.8375000000233,
+ "execMs": 0.0559159999829717,
"args": {
- "text": "So today we're",
- "id": "cue1"
+ "smooth": true,
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "at": 3,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2"
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 1,
+ "atMs": 3492.2657090000284,
+ "execMs": 0.004875000013271347,
+ "args": {
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "at": 4.7,
+ "speed": 0.2
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 2,
+ "atMs": 4814.276334000024,
+ "execMs": 0.003958999994210899,
+ "args": {
+ "smooth": true,
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3
+ },
+ {
+ "at": 4.7,
+ "speed": 0.3
+ }
+ ],
+ "clipId": "c2"
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 3,
+ "atMs": 6173.668750000012,
+ "execMs": 0.005207999987760559,
+ "args": {
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 4,
+ "atMs": 7739.629417000018,
+ "execMs": 0.006083999993279576,
+ "args": {
+ "clipId": "c2",
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "speed": 1,
+ "at": 3.5
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 5,
+ "atMs": 9213.063459000026,
+ "execMs": 0.004165999998804182,
+ "args": {
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "at": 4.7,
+ "speed": 0.2
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 6,
+ "atMs": 11171.220250000013,
+ "execMs": 0.007291999994777143,
+ "args": {
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3.5
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 7,
+ "atMs": 12676.369292000018,
+ "execMs": 0.01187499999650754,
+ "args": {
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 8,
+ "atMs": 14145.417709000001,
+ "execMs": 0.006999999983236194,
+ "args": {
+ "clipId": "c2",
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "smooth": true
+ }
+ },
+ {
+ "name": "set_speed_curve",
+ "round": 9,
+ "atMs": 15399.416917000024,
+ "execMs": 0.0019580000080168247,
+ "args": {
+ "smooth": true,
+ "clipId": "c2",
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "speed": 1,
+ "at": 3.5
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ]
}
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 2072.5324580000015,
- "execMs": 0.012875000014901161,
+ "name": "set_speed_curve",
+ "round": 10,
+ "atMs": 16870.353625000018,
+ "execMs": 0.30225000000791624,
"args": {
- "id": "cue3",
- "text": "and they..."
+ "nodes": [
+ {
+ "speed": 1,
+ "at": 0
+ },
+ {
+ "at": 3.5,
+ "speed": 1
+ },
+ {
+ "speed": 0.2,
+ "at": 4.7
+ }
+ ],
+ "clipId": "c2",
+ "smooth": true
}
},
{
- "name": "align_to_audio",
- "round": 1,
- "atMs": 3336.3753750000615,
- "execMs": 0.005250000045634806,
+ "name": "set_speed_curve",
+ "round": 11,
+ "atMs": 18442.462625000015,
+ "execMs": 0.007959000009577721,
"args": {
- "track": 0,
- "target": "captions"
+ "clipId": "c2",
+ "smooth": true,
+ "nodes": [
+ {
+ "at": 0,
+ "speed": 1
+ },
+ {
+ "at": 3,
+ "speed": 1
+ },
+ {
+ "at": 4.7,
+ "speed": 0.2
+ }
+ ]
}
}
],
"timings": {
- "gateMs": 357.8361249999143,
+ "gateMs": 620.2392079999845,
"gateSkipped": false,
- "ttftMs": 4261.617958000046,
- "modelMs": 4167.3184990000445,
- "toolMs": 0.13216700009070337,
- "totalMs": 4530.7826659999555,
- "judgeMs": 579.5385419999948,
+ "ttftMs": 19591.776000000013,
+ "modelMs": 19102.94004200006,
+ "toolMs": 0.4185419999703299,
+ "totalMs": 19751.77587500002,
+ "judgeMs": 557.0688329999975,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1711.5742910000263,
+ "roundMs": 1386.8733749999956,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.1140420000301674
- },
+ "name": "set_speed_curve",
+ "execMs": 0.0559159999829717
+ }
+ ]
+ },
+ {
+ "roundMs": 1476.84733400002,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.012875000014901161
+ "name": "set_speed_curve",
+ "execMs": 0.004875000013271347
}
]
},
{
- "roundMs": 1262.4753750000382,
+ "roundMs": 1320.5820830000157,
"firstDeltaMs": null,
"calls": [
{
- "name": "align_to_audio",
- "execMs": 0.005250000045634806
+ "name": "set_speed_curve",
+ "execMs": 0.003958999994210899
}
]
},
{
- "roundMs": 1193.26883299998,
- "firstDeltaMs": 924.1681249999674,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "subtitles",
- "library",
- "editor"
- ],
- "declaredTools": 73,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1986.8677500000922,
- "execMs": 0.010458000004291534,
- "args": {
- "text": "so today we're",
- "id": "cue1",
- "end": 1.8,
- "start": 0.45
- }
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 1986.9452080000192,
- "execMs": 0.004791999934241176,
- "args": {
- "id": "cue3",
- "end": 5.6,
- "start": 4.2,
- "text": "and"
- }
- },
- {
- "name": "align_to_audio",
- "round": 1,
- "atMs": 3433.6504160000477,
- "execMs": 0.0033340000081807375,
- "args": {
- "track": 0,
- "target": "captions"
- }
- }
- ],
- "timings": {
- "gateMs": 300.56587499997113,
- "gateSkipped": false,
- "ttftMs": 4426.42291600001,
- "modelMs": 4425.475332999951,
- "toolMs": 0.018583999946713448,
- "totalMs": 4727.584375000093,
- "judgeMs": 621.2392080000136,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 1358.1209589999926,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.005207999987760559
+ }
+ ]
+ },
{
- "roundMs": 1685.6646249999758,
+ "roundMs": 1564.153707999998,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.010458000004291534
- },
+ "name": "set_speed_curve",
+ "execMs": 0.006083999993279576
+ }
+ ]
+ },
+ {
+ "roundMs": 1471.576457999996,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "update_cue",
- "execMs": 0.004791999934241176
+ "name": "set_speed_curve",
+ "execMs": 0.004165999998804182
}
]
},
{
- "roundMs": 1446.298999999999,
+ "roundMs": 1956.6746250000142,
"firstDeltaMs": null,
"calls": [
{
- "name": "align_to_audio",
- "execMs": 0.0033340000081807375
+ "name": "set_speed_curve",
+ "execMs": 0.007291999994777143
}
]
},
{
- "roundMs": 1293.5117079999764,
- "firstDeltaMs": 992.4179999999469,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "question-when-ambiguous",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 215,
- "p95": 326,
- "mean": 270,
- "n": 2
- },
- "ttftMs": {
- "p50": 6411,
- "p95": 24262,
- "mean": 15337,
- "n": 2
- },
- "totalMs": {
- "p50": 6511,
- "p95": 24513,
- "mean": 15512,
- "n": 2
- },
- "modelMs": {
- "p50": 6182,
- "p95": 24297,
- "mean": 15239,
- "n": 2
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "skill": "timeline-editing",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "library",
- "editor"
- ],
- "declaredTools": 72,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 214.98616699990816,
- "gateSkipped": false,
- "ttftMs": 24262.332709000097,
- "modelMs": 24296.85908299999,
- "toolMs": 0,
- "totalMs": 24513.341042000102,
- "judgeMs": 538.3621669999557,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 1502.4437920000055,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.01187499999650754
+ }
+ ]
+ },
{
- "roundMs": 1383.499833000009,
+ "roundMs": 1466.7542919999978,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.006999999983236194
+ }
+ ]
},
{
- "roundMs": 22913.35924999998,
- "firstDeltaMs": 22662.75754099991,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "timeline-editing",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "library",
- "top_bar",
- "editor"
- ],
- "declaredTools": 76,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 325.60108399996534,
- "gateSkipped": false,
- "ttftMs": 6411.490166999982,
- "modelMs": 6182.048249999993,
- "toolMs": 0,
- "totalMs": 6511.39374999993,
- "judgeMs": 560.9472079999978,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 1252.5277500000084,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.0019580000080168247
+ }
+ ]
+ },
{
- "roundMs": 4285.07583300001,
+ "roundMs": 1469.813124999986,
"firstDeltaMs": null,
- "calls": []
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.30225000000791624
+ }
+ ]
+ },
+ {
+ "roundMs": 1569.556750000018,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_speed_curve",
+ "execMs": 0.007959000009577721
+ }
+ ]
},
{
- "roundMs": 1896.9724169999827,
- "firstDeltaMs": 1797.1950840000063,
+ "roundMs": 1307.0157910000125,
+ "firstDeltaMs": 1147.6045000000158,
"calls": []
}
]
@@ -7081,40 +4095,40 @@
]
},
{
- "name": "orphaned-overlay-debris",
+ "name": "remove-grey-ones-scope",
"bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 243,
- "p95": 279,
- "mean": 261,
- "n": 2
+ "p50": 524,
+ "p95": 524,
+ "mean": 524,
+ "n": 1
},
"ttftMs": {
- "p50": 2588,
- "p95": 2766,
- "mean": 2677,
- "n": 2
+ "p50": 3189,
+ "p95": 3189,
+ "mean": 3189,
+ "n": 1
},
"totalMs": {
- "p50": 2817,
- "p95": 3017,
- "mean": 2917,
- "n": 2
+ "p50": 3377,
+ "p95": 3377,
+ "mean": 3377,
+ "n": 1
},
"modelMs": {
- "p50": 2570,
- "p95": 2735,
- "mean": 2652,
- "n": 2
+ "p50": 2848,
+ "p95": 2848,
+ "mean": 2848,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
"mean": 2,
@@ -7129,105 +4143,183 @@
"areas": [
"preview",
"timeline",
- "library",
- "top_bar"
+ "inspector",
+ "effects",
+ "animation",
+ "transitions",
+ "side_panel",
+ "editor"
],
- "declaredTools": 60,
+ "declaredTools": 76,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "remove_transition",
"round": 0,
- "atMs": 1449.8045830000192,
- "execMs": 0.16641599999275059,
+ "atMs": 2008.9373750000086,
+ "execMs": 0.19495900001493283,
"args": {
- "clipId": "c1",
- "out": 10
+ "transitionId": "tr-8"
+ }
+ },
+ {
+ "name": "remove_transition",
+ "round": 0,
+ "atMs": 2009.395292000001,
+ "execMs": 0.002749999985098839,
+ "args": {
+ "transitionId": "tr-9"
}
}
],
"timings": {
- "gateMs": 242.78445799998008,
+ "gateMs": 524.1194590000086,
"gateSkipped": false,
- "ttftMs": 2587.984250000096,
- "modelMs": 2569.9313339999644,
- "toolMs": 0.16641599999275059,
- "totalMs": 2817.488333000103,
- "judgeMs": 316.0043750000186,
+ "ttftMs": 3189.453917000006,
+ "modelMs": 2848.478959,
+ "toolMs": 0.19770900000003166,
+ "totalMs": 3376.977083000005,
+ "judgeMs": 275.72816699999385,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1203.449874999933,
+ "roundMs": 1482.5566249999974,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.16641599999275059
+ "name": "remove_transition",
+ "execMs": 0.19495900001493283
+ },
+ {
+ "name": "remove_transition",
+ "execMs": 0.002749999985098839
}
]
},
{
- "roundMs": 1366.4814590000315,
- "firstDeltaMs": 1137.108750000014,
+ "roundMs": 1365.9223340000026,
+ "firstDeltaMs": 1178.8429999999935,
"calls": []
}
]
}
- },
+ }
+ ]
+ },
+ {
+ "name": "errored-call-not-claimed-done",
+ "bucket": "single-tool",
+ "passRate": 0,
+ "budgetBreaches": [],
+ "latency": null,
+ "runs": [
{
- "pass": true,
- "intent": "simple",
- "skill": null,
+ "pass": false,
+ "intent": "complex",
+ "skill": "transitions-and-fades",
"areas": [
"preview",
"timeline",
- "top_bar"
+ "inspector",
+ "effects",
+ "animation",
+ "transitions"
],
- "declaredTools": 53,
+ "declaredTools": 65,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
+ "roundModel": "gemini-3.8-flash",
+ "notes": [
+ "mutating tools called: set_animation, set_clip_keyframes"
+ ],
+ "instant": "set_transition",
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "set_transition",
"round": 0,
- "atMs": 1654.0348329999251,
- "execMs": 0.003874999936670065,
+ "atMs": 660.7137079999957,
+ "execMs": 0.22049999999580905,
"args": {
- "clipId": "c1",
- "out": 10
+ "clipId": "c2",
+ "style": "crossfade",
+ "seconds": 0.2
+ }
+ },
+ {
+ "name": "set_animation",
+ "round": 0,
+ "atMs": 6211.169124999986,
+ "execMs": 0.01041699998313561,
+ "args": {
+ "clipId": "c2",
+ "seconds": 0.5,
+ "style": "fade",
+ "which": "out"
+ }
+ },
+ {
+ "name": "set_clip_keyframes",
+ "round": 1,
+ "atMs": 8729.472875000007,
+ "execMs": 0.027833000000100583,
+ "args": {
+ "keys": [
+ {
+ "opacity": 1,
+ "t": 0
+ },
+ {
+ "opacity": 1,
+ "t": 6
+ },
+ {
+ "opacity": 0,
+ "t": 6.5
+ }
+ ],
+ "clipId": "c2"
}
}
],
"timings": {
- "gateMs": 279.2076250000391,
+ "gateMs": 659.8214159999916,
"gateSkipped": false,
- "ttftMs": 2766.3834999999963,
- "modelMs": 2734.90945799998,
- "toolMs": 0.003874999936670065,
- "totalMs": 3016.5371250000317,
- "judgeMs": 257.99887500004843,
+ "ttftMs": 12099.956416,
+ "modelMs": 11944.831166999938,
+ "toolMs": 0.25874999997904524,
+ "totalMs": 12619.98879100001,
+ "judgeMs": 733.2519999999786,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1373.5055830000201,
+ "roundMs": 5547.543499999971,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.003874999936670065
+ "name": "set_animation",
+ "execMs": 0.01041699998313561
+ }
+ ]
+ },
+ {
+ "roundMs": 2510.194334,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "set_clip_keyframes",
+ "execMs": 0.027833000000100583
}
]
},
{
- "roundMs": 1361.40387499996,
- "firstDeltaMs": 1111.5779999999795,
+ "roundMs": 3887.093332999968,
+ "firstDeltaMs": 3367.6148329999996,
"calls": []
}
]
@@ -7236,44 +4328,44 @@
]
},
{
- "name": "legacy-thread-no-leak",
- "bucket": "chat",
+ "name": "auto-continue-no-handoff",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 266,
- "p95": 326,
- "mean": 296,
- "n": 2
+ "p50": 522,
+ "p95": 522,
+ "mean": 522,
+ "n": 1
},
"ttftMs": {
- "p50": 1521,
- "p95": 1609,
- "mean": 1565,
- "n": 2
+ "p50": 5673,
+ "p95": 5673,
+ "mean": 5673,
+ "n": 1
},
"totalMs": {
- "p50": 1664,
- "p95": 1779,
- "mean": 1722,
- "n": 2
+ "p50": 5895,
+ "p95": 5895,
+ "mean": 5895,
+ "n": 1
},
"modelMs": {
- "p50": 1397,
- "p95": 1451,
- "mean": 1424,
- "n": 2
+ "p50": 5361,
+ "p95": 5361,
+ "mean": 5361,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 4,
+ "max": 4
}
},
"runs": [
@@ -7285,68 +4377,108 @@
"preview",
"timeline",
"inspector",
- "effects",
- "animation",
- "transitions"
+ "subtitles",
+ "library",
+ "editor"
],
- "declaredTools": 64,
+ "declaredTools": 86,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 265.52754099993035,
- "gateSkipped": false,
- "ttftMs": 1520.9474170000758,
- "modelMs": 1396.7702079999726,
- "toolMs": 0,
- "totalMs": 1664.1880420000525,
- "judgeMs": 235.89158299996052,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1396.7702079999726,
- "firstDeltaMs": 1253.8447079999605,
- "calls": []
+ "trace": [
+ {
+ "name": "find_filler",
+ "round": 0,
+ "atMs": 1959.092416999978,
+ "execMs": 0.1156250000349246,
+ "args": {
+ "track": 0,
+ "clip_id": "c1"
}
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "effects",
- "animation",
- "transitions",
- "library"
+ },
+ {
+ "name": "update_cue",
+ "round": 1,
+ "atMs": 3253.112124999985,
+ "execMs": 0.0372499999939464,
+ "args": {
+ "text": "So today we're",
+ "id": "cue1",
+ "start": 0.45
+ }
+ },
+ {
+ "name": "update_cue",
+ "round": 1,
+ "atMs": 3253.4155419999734,
+ "execMs": 0.0047919999924488366,
+ "args": {
+ "id": "cue3",
+ "start": 4.2,
+ "text": "And"
+ }
+ },
+ {
+ "name": "align_to_audio",
+ "round": 2,
+ "atMs": 4449.227374999959,
+ "execMs": 0.018041000002995133,
+ "args": {
+ "track": 0,
+ "target": "captions"
+ }
+ }
],
- "declaredTools": 71,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
"timings": {
- "gateMs": 325.848791000084,
+ "gateMs": 522.4588329999824,
"gateSkipped": false,
- "ttftMs": 1608.6598339999327,
- "modelMs": 1451.1174590000883,
- "toolMs": 0,
- "totalMs": 1779.079625000013,
- "judgeMs": 256.4152920000488,
+ "ttftMs": 5672.658832999994,
+ "modelMs": 5360.827416999906,
+ "toolMs": 0.17570800002431497,
+ "totalMs": 5894.63937499997,
+ "judgeMs": 499.80562499997905,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1451.1174590000883,
- "firstDeltaMs": 1280.7577499999898,
+ "roundMs": 1433.821249999979,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "find_filler",
+ "execMs": 0.1156250000349246
+ }
+ ]
+ },
+ {
+ "roundMs": 1290.9186669999617,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.0372499999939464
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.0047919999924488366
+ }
+ ]
+ },
+ {
+ "roundMs": 1193.7374999999884,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "align_to_audio",
+ "execMs": 0.018041000002995133
+ }
+ ]
+ },
+ {
+ "roundMs": 1442.3499999999767,
+ "firstDeltaMs": 1221.8870419999585,
"calls": []
}
]
@@ -7355,158 +4487,79 @@
]
},
{
- "name": "cinematic-ask-picks-preset",
- "bucket": "single-tool",
+ "name": "question-when-ambiguous",
+ "bucket": "chat",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 272,
- "p95": 441,
- "mean": 356,
- "n": 2
+ "p50": 685,
+ "p95": 685,
+ "mean": 685,
+ "n": 1
},
"ttftMs": {
- "p50": 2899,
- "p95": 4912,
- "mean": 3906,
- "n": 2
+ "p50": 2690,
+ "p95": 2690,
+ "mean": 2690,
+ "n": 1
},
"totalMs": {
- "p50": 3216,
- "p95": 5461,
- "mean": 4338,
- "n": 2
+ "p50": 2778,
+ "p95": 2778,
+ "mean": 2778,
+ "n": 1
},
"modelMs": {
- "p50": 2943,
- "p95": 5051,
- "mean": 3997,
- "n": 2
+ "p50": 2090,
+ "p95": 2090,
+ "mean": 2090,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "skill": null,
- "areas": [
- "preview",
- "inspector",
- "color",
- "effects",
- "side_panel",
- "top_bar"
- ],
- "declaredTools": 52,
- "routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_color_preset",
- "round": 0,
- "atMs": 2445.184041999979,
- "execMs": 0.21729099994990975,
- "args": {
- "preset": "teal-orange",
- "clipId": "c1"
- }
- }
- ],
- "timings": {
- "gateMs": 440.5892499999609,
- "gateSkipped": false,
- "ttftMs": 4912.3126250000205,
- "modelMs": 5050.79520900012,
- "toolMs": 0.21729099994990975,
- "totalMs": 5460.582999999984,
- "judgeMs": 555.5982920000097,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2036.9795840000734,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_preset",
- "execMs": 0.21729099994990975
- }
- ]
- },
- {
- "roundMs": 3013.8156250000466,
- "firstDeltaMs": 2465.772250000038,
- "calls": []
- }
- ]
- }
- },
+ "runs": [
{
"pass": true,
- "intent": "simple",
- "skill": null,
+ "intent": "complex",
+ "skill": "timeline-editing",
"areas": [
"preview",
+ "timeline",
"inspector",
- "color",
- "effects",
- "side_panel",
- "top_bar"
+ "library",
+ "editor"
],
- "declaredTools": 52,
+ "declaredTools": 73,
"routingMisses": [],
- "roundModel": "gemini-3.1-flash-lite",
+ "roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
- "trace": [
- {
- "name": "set_color_preset",
- "round": 0,
- "atMs": 1707.7004170001019,
- "execMs": 0.013374999980442226,
- "args": {
- "preset": "teal-orange",
- "clipId": "c1"
- }
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 271.7244589999318,
+ "gateMs": 684.9555839999812,
"gateSkipped": false,
- "ttftMs": 2898.7929590001004,
- "modelMs": 2943.199459000025,
- "toolMs": 0.013374999980442226,
- "totalMs": 3216.1091250000754,
- "judgeMs": 284.2845420000376,
+ "ttftMs": 2690.4507919999887,
+ "modelMs": 2090.2573329999577,
+ "toolMs": 0,
+ "totalMs": 2777.9478339999914,
+ "judgeMs": 259.8042499999865,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1435.3386669999454,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_preset",
- "execMs": 0.013374999980442226
- }
- ]
- },
- {
- "roundMs": 1507.8607920000795,
- "firstDeltaMs": 1190.7736250000307,
+ "roundMs": 2090.2573329999577,
+ "firstDeltaMs": 2003.0821659999783,
"calls": []
}
]
@@ -7515,44 +4568,44 @@
]
},
{
- "name": "warmer-crushed-blacks-in-range",
- "bucket": "single-tool",
+ "name": "orphaned-overlay-debris",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 245,
- "p95": 265,
- "mean": 255,
- "n": 2
+ "p50": 529,
+ "p95": 529,
+ "mean": 529,
+ "n": 1
},
"ttftMs": {
- "p50": 3704,
- "p95": 5166,
- "mean": 4435,
- "n": 2
+ "p50": 2989,
+ "p95": 2989,
+ "mean": 2989,
+ "n": 1
},
"totalMs": {
- "p50": 4010,
- "p95": 5351,
- "mean": 4680,
- "n": 2
+ "p50": 3194,
+ "p95": 3194,
+ "mean": 3194,
+ "n": 1
},
"modelMs": {
- "p50": 3761,
- "p95": 5080,
- "mean": 4420,
- "n": 2
+ "p50": 2661,
+ "p95": 2661,
+ "mean": 2661,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 3.5,
- "max": 4
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -7562,152 +4615,133 @@
"skill": null,
"areas": [
"preview",
- "inspector",
- "color",
- "effects"
+ "timeline",
+ "inspector"
],
- "declaredTools": 41,
+ "declaredTools": 62,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "set_color_grade",
- "round": 1,
- "atMs": 2790.560208999901,
- "execMs": 0.07479099999181926,
- "args": {
- "clipId": "c1",
- "temperature": 20
- }
- },
- {
- "name": "set_color_curves",
- "round": 1,
- "atMs": 2790.9987089999486,
- "execMs": 0.0023750000400468707,
- "args": {
- "clipId": "c1",
- "curve_contrast": 10
- }
- },
- {
- "name": "set_color_grade",
- "round": 2,
- "atMs": 4113.332999999984,
- "execMs": 0.010834000073373318,
+ "name": "trim_clip",
+ "round": 0,
+ "atMs": 1882.4166250000126,
+ "execMs": 0.1556659999769181,
"args": {
"clipId": "c1",
- "shadows": -10
+ "out": 10
}
}
],
"timings": {
- "gateMs": 264.95016599993687,
+ "gateMs": 529.1502079999773,
"gateSkipped": false,
- "ttftMs": 5166.28583399998,
- "modelMs": 5079.554542000056,
- "toolMs": 0.08800000010523945,
- "totalMs": 5350.7168749999255,
- "judgeMs": 637.7544169999892,
+ "ttftMs": 2989.3134169999976,
+ "modelMs": 2661.2188339999993,
+ "toolMs": 0.1556659999769181,
+ "totalMs": 3193.9387500000303,
+ "judgeMs": 545.0367499999702,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1224.7247500000522,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 1297.2725830000127,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.07479099999181926
- },
- {
- "name": "set_color_curves",
- "execMs": 0.0023750000400468707
- }
- ]
- },
- {
- "roundMs": 1321.6398340000305,
+ "roundMs": 1351.2681250000023,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_color_grade",
- "execMs": 0.010834000073373318
+ "name": "trim_clip",
+ "execMs": 0.1556659999769181
}
]
},
{
- "roundMs": 1235.9173749999609,
- "firstDeltaMs": 1051.7169170000125,
+ "roundMs": 1309.950708999997,
+ "firstDeltaMs": 1105.6849999999977,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "legacy-thread-no-leak",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 330,
+ "p95": 330,
+ "mean": 330,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 1788,
+ "p95": 1788,
+ "mean": 1788,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 2028,
+ "p95": 2028,
+ "mean": 2028,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 1696,
+ "p95": 1696,
+ "mean": 1696,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 1,
+ "max": 1
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "simple",
"skill": null,
"areas": [
"preview",
+ "timeline",
"inspector",
- "color",
- "effects"
+ "effects",
+ "animation",
+ "transitions"
],
- "declaredTools": 41,
+ "declaredTools": 65,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
- "trace": [
- {
- "name": "set_color_grade",
- "round": 1,
- "atMs": 2705.120666999952,
- "execMs": 0.0036250000121071935,
- "args": {
- "clipId": "c1",
- "shadows": -10,
- "temperature": 20
- }
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 245.48908299999312,
+ "gateMs": 330.1852910000016,
"gateSkipped": false,
- "ttftMs": 3704.3957499999087,
- "modelMs": 3761.363999999943,
- "toolMs": 0.0036250000121071935,
- "totalMs": 4010.2012499999255,
- "judgeMs": 214.22733400005382,
+ "ttftMs": 1787.689040999976,
+ "modelMs": 1695.9717499999679,
+ "toolMs": 0,
+ "totalMs": 2027.8691659999895,
+ "judgeMs": 270.4465419999906,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1324.3317919999827,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 1132.6171669999603,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_grade",
- "execMs": 0.0036250000121071935
- }
- ]
- },
- {
- "roundMs": 1304.4150410000002,
- "firstDeltaMs": 998.7812080000294,
+ "roundMs": 1695.9717499999679,
+ "firstDeltaMs": 1456.407917000004,
"calls": []
}
]
@@ -7716,44 +4750,44 @@
]
},
{
- "name": "sky-bluer-single-band",
+ "name": "cinematic-ask-picks-preset",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 241,
- "p95": 300,
- "mean": 271,
- "n": 2
+ "p50": 280,
+ "p95": 280,
+ "mean": 280,
+ "n": 1
},
"ttftMs": {
- "p50": 6287,
- "p95": 6714,
- "mean": 6501,
- "n": 2
+ "p50": 282,
+ "p95": 282,
+ "mean": 282,
+ "n": 1
},
"totalMs": {
- "p50": 6726,
- "p95": 7301,
- "mean": 7013,
- "n": 2
+ "p50": 283,
+ "p95": 283,
+ "mean": 283,
+ "n": 1
},
"modelMs": {
- "p50": 6419,
- "p95": 7051,
- "mean": 6735,
- "n": 2
- },
- "toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 1,
+ "p95": 1,
+ "mean": 1,
+ "n": 1
},
"rounds": {
- "mean": 4,
- "max": 4
+ "mean": 0,
+ "max": 0
}
},
"runs": [
@@ -7766,80 +4800,85 @@
"inspector",
"color",
"effects",
- "image_gen"
+ "side_panel",
+ "top_bar"
],
- "declaredTools": 42,
+ "declaredTools": 52,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": "set_color_preset",
"judgeNote": null,
"trace": [
{
- "name": "seek",
- "round": 1,
- "atMs": 3337.565500000026,
- "execMs": 0.23183299996890128,
- "args": {
- "t": 0
- }
- },
- {
- "name": "set_color_hsl",
- "round": 2,
- "atMs": 5158.769417000003,
- "execMs": 0.013999999966472387,
+ "name": "set_color_preset",
+ "round": 0,
+ "atMs": 281.4969999999739,
+ "execMs": 0.5242079999879934,
"args": {
"clipId": "c1",
- "band": "blue",
- "sat": 25,
- "hue": 15
+ "preset": "teal-orange",
+ "amount": 0.69
}
}
],
"timings": {
- "gateMs": 241.17504200001713,
+ "gateMs": 280.0892500000191,
"gateSkipped": false,
- "ttftMs": 6713.8770830000285,
- "modelMs": 7050.863166999887,
- "toolMs": 0.24583299993537366,
- "totalMs": 7300.524457999971,
- "judgeMs": 584.8700420000823,
+ "ttftMs": 282.4989999999525,
+ "modelMs": 0,
+ "toolMs": 0.5242079999879934,
+ "totalMs": 282.5238749999553,
+ "judgeMs": 247.4059160000179,
"retryMs": 0,
"retries": 0,
- "rounds": [
- {
- "roundMs": 1213.1531249999534,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 1878.9036659999983,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "seek",
- "execMs": 0.23183299996890128
- }
- ]
- },
- {
- "roundMs": 1818.3687919999938,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "set_color_hsl",
- "execMs": 0.013999999966472387
- }
- ]
- },
- {
- "roundMs": 2140.437583999941,
- "firstDeltaMs": 1554.0435419999994,
- "calls": []
- }
- ]
+ "rounds": []
}
+ }
+ ]
+ },
+ {
+ "name": "warmer-crushed-blacks-in-range",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 395,
+ "p95": 395,
+ "mean": 395,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 6540,
+ "p95": 6540,
+ "mean": 6540,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 6710,
+ "p95": 6710,
+ "mean": 6710,
+ "n": 1
},
+ "modelMs": {
+ "p50": 6302,
+ "p95": 6302,
+ "mean": 6302,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
+ },
+ "rounds": {
+ "mean": 3,
+ "max": 3
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "simple",
@@ -7848,76 +4887,56 @@
"preview",
"inspector",
"color",
- "effects",
- "image_gen"
+ "effects"
],
- "declaredTools": 42,
+ "declaredTools": 41,
"routingMisses": [],
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
- "name": "seek",
+ "name": "set_color_grade",
"round": 1,
- "atMs": 3156.8108749999665,
- "execMs": 0.014333999948576093,
- "args": {
- "t": 0
- }
- },
- {
- "name": "set_color_hsl",
- "round": 2,
- "atMs": 5049.040666999994,
- "execMs": 0.01362500002142042,
+ "atMs": 5570.3494170000195,
+ "execMs": 0.3076249999576248,
"args": {
- "hue": 20,
- "sat": 25,
- "band": "blue",
- "clipId": "c1"
+ "temperature": 20,
+ "clipId": "c1",
+ "shadows": -15
}
}
],
"timings": {
- "gateMs": 300.2105000000447,
+ "gateMs": 395.3481669999892,
"gateSkipped": false,
- "ttftMs": 6287.196249999921,
- "modelMs": 6418.930500999908,
- "toolMs": 0.027958999969996512,
- "totalMs": 6725.693250000011,
- "judgeMs": 541.2030830000294,
+ "ttftMs": 6540.034542000038,
+ "modelMs": 6301.72504099994,
+ "toolMs": 0.3076249999576248,
+ "totalMs": 6710.445959000033,
+ "judgeMs": 576.8358749999898,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1158.9759589999449,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 1693.9996249999385,
+ "roundMs": 3856.6763330000103,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "seek",
- "execMs": 0.014333999948576093
- }
- ]
+ "calls": []
},
{
- "roundMs": 1890.6579589999747,
+ "roundMs": 1307.2124159999657,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_color_hsl",
- "execMs": 0.01362500002142042
+ "name": "set_color_grade",
+ "execMs": 0.3076249999576248
}
]
},
{
- "roundMs": 1675.29695800005,
- "firstDeltaMs": 1236.9017080001067,
+ "roundMs": 1137.8362919999636,
+ "firstDeltaMs": 967.8776249999646,
"calls": []
}
]
@@ -7926,124 +4945,135 @@
]
},
{
- "name": "reference-photo-matches-grade",
+ "name": "sky-bluer-single-band",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 255,
- "p95": 264,
- "mean": 259,
- "n": 2
+ "p50": 451,
+ "p95": 451,
+ "mean": 451,
+ "n": 1
},
"ttftMs": {
- "p50": 10077,
- "p95": 20151,
- "mean": 15114,
- "n": 2
+ "p50": 452,
+ "p95": 452,
+ "mean": 452,
+ "n": 1
},
"totalMs": {
- "p50": 10240,
- "p95": 20436,
- "mean": 15338,
- "n": 2
+ "p50": 452,
+ "p95": 452,
+ "mean": 452,
+ "n": 1
},
"modelMs": {
- "p50": 9969,
- "p95": 20176,
- "mean": 15072,
- "n": 2
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 1
},
"rounds": {
- "mean": 3,
- "max": 3
+ "mean": 0,
+ "max": 0
}
},
"runs": [
{
"pass": true,
"intent": "complex",
- "skill": null,
+ "skill": "color-grading",
"areas": [
"preview",
"inspector",
"color",
"effects",
- "side_panel",
- "top_bar"
+ "image_gen",
+ "editor"
],
- "declaredTools": 52,
+ "declaredTools": 46,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": "set_color_hsl",
"judgeNote": null,
"trace": [
{
- "name": "match_color_grade",
+ "name": "set_color_hsl",
"round": 0,
- "atMs": 5722.50245800009,
- "execMs": 0.23270799999590963,
- "args": {
- "ref_asset_id": "a-i1",
- "clipId": "c1"
- }
- },
- {
- "name": "read_color_stats",
- "round": 1,
- "atMs": 8448.422292000032,
- "execMs": 0.053958000033162534,
+ "atMs": 451.54308299999684,
+ "execMs": 0.17145800002617761,
"args": {
- "clipId": "c1"
+ "clipId": "c1",
+ "band": "blue",
+ "sat": 29
}
}
],
"timings": {
- "gateMs": 263.6963749999413,
- "gateSkipped": true,
- "ttftMs": 10076.608708000043,
- "modelMs": 9968.938291000086,
- "toolMs": 0.28666600002907217,
- "totalMs": 10239.683917000075,
- "judgeMs": 499.6667080000043,
+ "gateMs": 450.66599999996834,
+ "gateSkipped": false,
+ "ttftMs": 451.9269579999964,
+ "modelMs": 0,
+ "toolMs": 0.17145800002617761,
+ "totalMs": 451.9367499999935,
+ "judgeMs": 232.18725000001723,
"retryMs": 0,
"retries": 0,
- "rounds": [
- {
- "roundMs": 5455.68979199999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "match_color_grade",
- "execMs": 0.23270799999590963
- }
- ]
- },
- {
- "roundMs": 2723.8698330000043,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_color_stats",
- "execMs": 0.053958000033162534
- }
- ]
- },
- {
- "roundMs": 1789.3786660000915,
- "firstDeltaMs": 1626.7485410000663,
- "calls": []
- }
- ]
+ "rounds": []
}
+ }
+ ]
+ },
+ {
+ "name": "reference-photo-matches-grade",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 322,
+ "p95": 322,
+ "mean": 322,
+ "n": 1
+ },
+ "ttftMs": {
+ "p50": 4720,
+ "p95": 4720,
+ "mean": 4720,
+ "n": 1
+ },
+ "totalMs": {
+ "p50": 4898,
+ "p95": 4898,
+ "mean": 4898,
+ "n": 1
+ },
+ "modelMs": {
+ "p50": 4569,
+ "p95": 4569,
+ "mean": 4569,
+ "n": 1
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
"intent": "complex",
@@ -8054,68 +5084,51 @@
"color",
"effects",
"side_panel",
- "top_bar"
+ "top_bar",
+ "editor"
],
- "declaredTools": 52,
+ "declaredTools": 56,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
+ "instant": null,
"judgeNote": null,
"trace": [
{
"name": "match_color_grade",
"round": 0,
- "atMs": 16618.874959000037,
- "execMs": 0.015000000013969839,
+ "atMs": 2269.434332999983,
+ "execMs": 0.15854199999012053,
"args": {
"ref_asset_id": "a-i1",
"clipId": "c1"
}
- },
- {
- "name": "read_color_stats",
- "round": 1,
- "atMs": 18550.126249999972,
- "execMs": 0.008917000028304756,
- "args": {
- "clipId": "c1"
- }
}
],
"timings": {
- "gateMs": 255.05629099998623,
+ "gateMs": 322.3672089999891,
"gateSkipped": true,
- "ttftMs": 20151.05587499996,
- "modelMs": 20175.66770799982,
- "toolMs": 0.023917000042274594,
- "totalMs": 20436.48333399999,
- "judgeMs": 523.6447499999776,
+ "ttftMs": 4719.961791999987,
+ "modelMs": 4569.165667000052,
+ "toolMs": 0.15854199999012053,
+ "totalMs": 4897.899708000012,
+ "judgeMs": 598.573040999996,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 16361.533707999974,
+ "roundMs": 1942.8682090000366,
"firstDeltaMs": null,
"calls": [
{
"name": "match_color_grade",
- "execMs": 0.015000000013969839
- }
- ]
- },
- {
- "roundMs": 1929.4221249999246,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_color_stats",
- "execMs": 0.008917000028304756
+ "execMs": 0.15854199999012053
}
]
},
{
- "roundMs": 1884.7118749999208,
- "firstDeltaMs": 1599.4977499999804,
+ "roundMs": 2626.2974580000155,
+ "firstDeltaMs": 2448.595250000013,
"calls": []
}
]
@@ -8130,38 +5143,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 256,
- "p95": 275,
- "mean": 265,
- "n": 2
+ "p50": 476,
+ "p95": 476,
+ "mean": 476,
+ "n": 1
},
"ttftMs": {
- "p50": 13186,
- "p95": 13852,
- "mean": 13519,
- "n": 2
+ "p50": 8314,
+ "p95": 8314,
+ "mean": 8314,
+ "n": 1
},
"totalMs": {
- "p50": 13680,
- "p95": 14212,
- "mean": 13946,
- "n": 2
+ "p50": 8798,
+ "p95": 8798,
+ "mean": 8798,
+ "n": 1
},
"modelMs": {
- "p50": 13386,
- "p95": 13935,
- "mean": 13660,
- "n": 2
+ "p50": 8311,
+ "p95": 8311,
+ "mean": 8311,
+ "n": 1
},
"toolMs": {
"p50": 0,
- "p95": 1,
- "mean": 1,
- "n": 2
+ "p95": 0,
+ "mean": 0,
+ "n": 1
},
"rounds": {
- "mean": 5.5,
- "max": 6
+ "mean": 3,
+ "max": 3
}
},
"runs": [
@@ -8174,7 +5187,6 @@
"timeline",
"inspector",
"color",
- "removal",
"effects",
"animation",
"transitions",
@@ -8187,248 +5199,76 @@
"top_bar",
"editor"
],
- "declaredTools": 121,
+ "declaredTools": 119,
"routingMisses": [],
"roundModel": "gemini-3.8-flash",
"notes": [],
- "judgeNote": "upsell 0.54",
+ "instant": null,
+ "judgeNote": "upsell 0.57",
"trace": [
{
"name": "read_project",
- "round": 1,
- "atMs": 7373.890207999968,
- "execMs": 0.689790999982506,
+ "round": 0,
+ "atMs": 3115.3913750000065,
+ "execMs": 0.20216700003948063,
"args": {
"link": "https://donkeycut.com/app/p/ref-123"
}
},
{
"name": "replicate_project",
- "round": 2,
- "atMs": 9963.025707999943,
- "execMs": 0.1279169999761507,
+ "round": 1,
+ "atMs": 6177.915750000044,
+ "execMs": 0.1061250000493601,
"args": {
"link": "https://donkeycut.com/app/p/ref-123",
"media": [
{
- "source_asset_id": "r-v1",
- "asset_id": "a-v1"
- },
- {
- "asset_id": "a-v2",
- "source_asset_id": "r-v2"
- }
- ]
- }
- },
- {
- "name": "update_overlay",
- "round": 3,
- "atMs": 11836.166540999897,
- "execMs": 0.03116699995007366,
- "args": {
- "id": "t-n1",
- "text": "MY DESK SETUP"
- }
- },
- {
- "name": "subtitles_generate",
- "round": 3,
- "atMs": 11836.586374999955,
- "execMs": 0.03554099996108562,
- "args": {}
- }
- ],
- "timings": {
- "gateMs": 255.5660420000786,
- "gateSkipped": false,
- "ttftMs": 13851.970332999947,
- "modelMs": 13934.569001000025,
- "toolMs": 0.884415999869816,
- "totalMs": 14211.674124999903,
- "judgeMs": 628.9623330000322,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4637.172499999986,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2474.225375000038,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_project",
- "execMs": 0.689790999982506
- }
- ]
- },
- {
- "roundMs": 2583.4949999999953,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "replicate_project",
- "execMs": 0.1279169999761507
- }
- ]
- },
- {
- "roundMs": 1867.8827089999104,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_overlay",
- "execMs": 0.03116699995007366
+ "asset_id": "a-v1",
+ "source_asset_id": "r-v1"
},
{
- "name": "subtitles_generate",
- "execMs": 0.03554099996108562
+ "source_asset_id": "r-v2",
+ "asset_id": "a-v2"
}
]
- },
- {
- "roundMs": 2371.7934170000954,
- "firstDeltaMs": 2013.3481250000186,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "skill": "replicating-a-project",
- "areas": [
- "preview",
- "timeline",
- "inspector",
- "color",
- "effects",
- "animation",
- "transitions",
- "subtitles",
- "audio",
- "video_gen",
- "scene",
- "library",
- "side_panel",
- "top_bar",
- "editor"
- ],
- "declaredTools": 118,
- "routingMisses": [],
- "roundModel": "gemini-3.8-flash",
- "notes": [],
- "judgeNote": "upsell 0.57",
- "trace": [
- {
- "name": "read_project",
- "round": 1,
- "atMs": 4354.051875000005,
- "execMs": 0.0339169999351725,
- "args": {
- "link": "https://donkeycut.com/app/p/ref-123"
- }
- },
- {
- "name": "replicate_project",
- "round": 2,
- "atMs": 7220.381125000073,
- "execMs": 0.11149999999906868,
- "args": {
- "media": [
- {
- "source_asset_id": "r-v1",
- "asset_id": "a-v1"
- },
- {
- "asset_id": "a-v2",
- "source_asset_id": "r-v2"
- }
- ],
- "link": "https://donkeycut.com/app/p/ref-123"
- }
- },
- {
- "name": "update_overlay",
- "round": 3,
- "atMs": 9275.000750000007,
- "execMs": 0.033500000019557774,
- "args": {
- "text": "MY DESK SETUP",
- "id": "t-n1"
}
- },
- {
- "name": "subtitles_generate",
- "round": 4,
- "atMs": 11213.23350000009,
- "execMs": 0.03175000008195639,
- "args": {}
}
],
"timings": {
- "gateMs": 275.12804199999664,
+ "gateMs": 475.94483300001593,
"gateSkipped": false,
- "ttftMs": 13185.857250000001,
- "modelMs": 13385.525876000058,
- "toolMs": 0.21066700003575534,
- "totalMs": 13680.230250000022,
- "judgeMs": 590.8087079999968,
+ "ttftMs": 8313.829958000046,
+ "modelMs": 8311.124376000022,
+ "toolMs": 0.3082920000888407,
+ "totalMs": 8798.196000000054,
+ "judgeMs": 562.6585830000113,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1735.5736670000479,
- "firstDeltaMs": null,
- "calls": []
- },
- {
- "roundMs": 2335.4039169999305,
+ "roundMs": 2634.9706250000163,
"firstDeltaMs": null,
"calls": [
{
"name": "read_project",
- "execMs": 0.0339169999351725
+ "execMs": 0.20216700003948063
}
]
},
{
- "roundMs": 2863.430708000087,
+ "roundMs": 3058.2170840000035,
"firstDeltaMs": null,
"calls": [
{
"name": "replicate_project",
- "execMs": 0.11149999999906868
- }
- ]
- },
- {
- "roundMs": 2051.527374999947,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "update_overlay",
- "execMs": 0.033500000019557774
- }
- ]
- },
- {
- "roundMs": 1935.511792000034,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "subtitles_generate",
- "execMs": 0.03175000008195639
+ "execMs": 0.1061250000493601
}
]
},
{
- "roundMs": 2464.0784170000115,
- "firstDeltaMs": 1970.1614590000827,
+ "roundMs": 2617.9366670000018,
+ "firstDeltaMs": 2133.771374999953,
"calls": []
}
]
diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.ts
index 4641bc79..2b462399 100644
--- a/site/scripts/eval-cut-chat.ts
+++ b/site/scripts/eval-cut-chat.ts
@@ -140,6 +140,7 @@ async function runConfig(
` ${report.passRate === 1 ? "ok " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` +
` pass ${pct(report.passRate)} total p50 ${ms(l?.totalMs.p50)} ttft p50 ${ms(l?.ttftMs?.p50)}` +
` rounds ${l ? l.rounds.mean : "—"}` +
+ (runs.find((r) => r.instant) ? ` instant ${runs.find((r) => r.instant)?.instant}` : "") +
(report.budgetBreaches.length > 0 ? ` BUDGET: ${report.budgetBreaches.join("; ")}` : "")
);
for (const r of runs.filter((r) => !r.pass)) {
diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.ts
index a3e52f39..3fc54bc0 100644
--- a/site/scripts/lib/cut-eval/cases.ts
+++ b/site/scripts/lib/cut-eval/cases.ts
@@ -4,6 +4,7 @@
* project-mutating tool calls, edit requests still reach for tools.
*
* Every case carries a latency bucket:
+ * instant — the judgment settles the whole turn: no model round at all
* chat — the fast path: gated greetings and question turns
* single-tool — one decisive call reaches the right tool
* multi-tool — composed edits over several calls
@@ -37,7 +38,7 @@ import {
userTurn,
} from "./fixtures";
-export type Bucket = "chat" | "single-tool" | "multi-tool";
+export type Bucket = "instant" | "chat" | "single-tool" | "multi-tool";
export interface EvalCase {
name: string;
@@ -53,6 +54,10 @@ export interface EvalCase {
/** Latency guard: the turn fails if the trace exceeds this many calls. A
* simple ask must not detour through skill reads or state polls. */
maxToolCalls?: number;
+ /** The registry action the instant path must settle this turn to, with no
+ * model round at all. `null` pins the opposite: the turn must reach the
+ * model. Left out, either is allowed. */
+ instant?: string | null;
/** The gate side the turn must classify to: "chat" turns run with no tool
* declarations (tool calls become impossible), "work" covers both routing
* verdicts (simple and complex). */
@@ -95,6 +100,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// subtitles_generate): a bare greeting asks for nothing, so the turn
// must classify "chat" and run with every tool declaration withheld.
name: "greeting-is-gated",
+ instant: null,
bucket: "chat",
input: () => [userTurn("hi")],
reply: /help|what would you/i,
@@ -105,6 +111,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
{
// Same gate mid-conversation: thanks after landed work requests nothing.
name: "thanks-is-gated",
+ instant: null,
bucket: "chat",
input: () => [
plainUserTurn("trim the first clip down to 5 seconds"),
@@ -185,6 +192,9 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
stubs: {
set_transition: { ok: true },
add_title: { ok: true },
+ // "Short" is part of the ask, and a photo lands at the default still
+ // length, so trimming the pair down is the movie being made.
+ trim_clip: { in: 0, out: 3.5, len: 3.5 },
},
},
{
@@ -261,6 +271,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// to "make room" is wrong — the trap word "instead"/"version" is not a
// delete instruction.
name: "iterate-keeps-existing-clip",
+ instant: null,
bucket: "single-tool",
input: () => [
userTurn("the beach clip feels off — make me a better version, more of a golden-hour sunset"),
@@ -296,6 +307,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// then narrates that as the answer. The snapshot names every clip and
// its muted flag, so the mute needs no reads either.
name: "mute-all-does-not-redo-prior-turn",
+ instant: null,
bucket: "single-tool",
input: () => [
plainUserTurn(
@@ -410,6 +422,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
{
// Fast path: a direct edit is one trim_clip on the clip in the snapshot.
name: "trim-ask-single-tool",
+ instant: null,
bucket: "single-tool",
input: () => [userTurn("trim the first clip down to 5 seconds")],
reply: /trim|5|second/i,
@@ -546,6 +559,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// "Pop" exists only as an animation, so the routing is unambiguous
// (a wipe ask could legitimately become a wipe *transition* instead).
name: "animation-overrides-transition",
+ instant: null,
bucket: "single-tool",
input: () => [
userTurn("get rid of the crossfade — make the second clip pop in instead", {
@@ -774,6 +788,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// A described ramp is a curve, laid in one call: the shape the user
// asked for lands as nodes over the clip's footage.
name: "speed-ramp-from-description",
+ instant: null,
bucket: "single-tool",
// A clean timeline: the debris cases own the parked bars, and a turn
// that can see them is right to tidy them.
@@ -927,6 +942,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// blacks" pushes shadows down (or deepens the low end of the master
// curve) — the interceptor fails any move in the wrong direction.
name: "warmer-crushed-blacks-in-range",
+ instant: null,
bucket: "single-tool",
input: () => [userTurn("make the beach clip warmer and crush the blacks a little")],
reply: /warm|black|shadow|grade/i,
@@ -957,6 +973,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
// set_color_hsl on aqua or blue — a whole-frame saturation push leaks
// onto everything else and fails.
name: "sky-bluer-single-band",
+ instant: "set_color_hsl",
bucket: "single-tool",
input: () => [userTurn("make the sky bluer without changing anything else")],
reply: /sky|blue/i,
diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.ts
index 98140477..3783afe4 100644
--- a/site/scripts/lib/cut-eval/fixtures.ts
+++ b/site/scripts/lib/cut-eval/fixtures.ts
@@ -36,6 +36,9 @@ export const SAFE_TOOLS = new Set([
"list_voices",
"seek",
"select",
+ // Finding the items a description covers reads the timeline; the change
+ // that follows is what edits.
+ "select_items",
"set_playing",
"set_view",
]);
@@ -746,7 +749,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
}
/** Serve a read-only tool from the fixture snapshot. */
-export function serveSafeTool(name: string, state: unknown): unknown {
+export function serveSafeTool(name: string, state: unknown, args: Record<string, unknown> = {}): unknown {
if (name === "get_state") {
// The fixture snapshot is frozen — it can't reflect this turn's stubbed
// edits. Say so, or the model sees its adds "missing" and re-adds in a
@@ -763,5 +766,21 @@ export function serveSafeTool(name: string, state: unknown): unknown {
if (name === "find_filler") return { fillers: FILLER_WORDS, count: FILLER_WORDS.length, track: 0 };
if (name === "measure_level") return { targetId: "", clips: [] };
if (name === "detect_beats") return { bpm: 0, beats: [] };
+ if (name === "select_items") {
+ // The find half of a sweep, served off the fixture: every item of that
+ // kind matches, so a case asserting the apply half gets the whole row.
+ const s = (state ?? {}) as Record<string, unknown>;
+ const kind = String(args.kind ?? "clip");
+ const rows =
+ kind === "audio" ? s.soundtrack
+ : kind === "overlay" ? s.overlays
+ : kind === "cue" ? ((s.subtitles ?? {}) as Record<string, unknown>).cues
+ : kind === "transition" ? s.transitions
+ : s.videoTrack;
+ const ids = (Array.isArray(rows) ? rows : [])
+ .map((r) => (r as { id?: unknown }).id)
+ .filter((id): id is string => typeof id === "string");
+ return { kind, searched: ids.length, ids, matched: ids.map((id) => ({ id })) };
+ }
return { ok: true };
}
diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.ts
index 6740b7c5..243c4cfa 100644
--- a/site/scripts/lib/cut-eval/harness.ts
+++ b/site/scripts/lib/cut-eval/harness.ts
@@ -100,6 +100,8 @@ export interface CaseResult {
routingMisses: string[];
/** The model the rounds ran on — differs per turn under a router config. */
roundModel: string;
+ /** The action the instant path carried the turn out with, if any. */
+ instant: string | null;
timings: RunTimings;
}
@@ -125,6 +127,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
let reply = "";
let streamError: string | null = null;
let extensions = 0;
+ let instant: string | null = null;
const devPost =
(path: string) =>
@@ -168,7 +171,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
throw new Error(String((stub as { __error: unknown }).__error));
return stub;
}
- if (SAFE_TOOLS.has(name)) return serveSafeTool(name, c.state ?? EDITOR_STATE);
+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, c.state ?? EDITOR_STATE, args);
violations.push(name);
return { error: "eval: this tool is disabled for this turn — do not retry it" };
} finally {
@@ -187,6 +190,9 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
gateSkipped = skipped;
route = { skill: r.skill, areas: r.areas, declaredTools: r.declaredTools };
},
+ onInstant: (action) => {
+ instant = action?.id ?? null;
+ },
onRequestTools: (areas) => routingMisses.push(...areas),
onRound: (ms, firstDeltaMs) => {
modelMs += ms;
@@ -237,6 +243,12 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
// "work" expectation.
const gateSide = intent === "chat" ? "chat" : "work";
if (c.gate && gateSide !== c.gate) notes.push(`gate said ${intent}, expected ${c.gate}`);
+ // An instant turn is the whole point of the path: the judgment settled it
+ // and no model round ran. A case that pins one asserts both.
+ if (c.instant !== undefined) {
+ if (instant !== c.instant) notes.push(`instant was ${instant ?? "none"}, expected ${c.instant ?? "none"}`);
+ if (c.instant !== null && rounds.length > 0) notes.push(`instant turn still spent ${rounds.length} model round(s)`);
+ }
if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);
for (const t of c.requiredTools ?? []) {
if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);
@@ -299,6 +311,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
areas: route?.areas ?? [],
declaredTools: route?.declaredTools ?? 0,
routingMisses,
+ instant,
roundModel: resolveGeminiModel(intent === "simple" ? cfg.simpleModel : cfg.complexModel),
timings,
};
diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.ts
index 5ab5b724..68fe26b2 100644
--- a/site/scripts/lib/cut-eval/report.ts
+++ b/site/scripts/lib/cut-eval/report.ts
@@ -26,6 +26,8 @@ export interface RunReport {
/** The model the rounds ran on — differs per turn under a router config. */
roundModel?: string;
notes: string[];
+ /** The registry action the turn ran with no model round, if any. */
+ instant?: string | null;
/** The voice/taste judge's complaint. Report-only: never part of pass. */
judgeNote?: string | null;
trace: TraceEntry[];
@@ -107,6 +109,7 @@ export function toRunReport(result: CaseResult | null, error?: string): RunRepor
routingMisses: result.routingMisses,
roundModel: result.roundModel,
notes: result.notes,
+ instant: result.instant,
judgeNote: result.judgeNote,
trace: result.trace,
timings: result.timings,