Open source CapCut alternative.
TypeScript
test(site): the chat eval pins which turns go instant
David Lef80e0c1573d3
diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.jsonindex 23970b4a..2b9b3ecb 100644--- a/evals/cut-chat.latest-report.json+++ b/evals/cut-chat.latest-report.json@@ -1,8 +1,8 @@{"schema": "cut-chat-eval/v1",- "generatedAt": "2026-09-17T12:17:25.851Z",+ "generatedAt": "2026-09-18T03:44:43.038Z","base": "http://localhost:3000",- "runsPerCase": 2,+ "runsPerCase": 1,"configs": [{"label": "config",@@ -12,70 +12,70 @@"cases": 9,"passRate": 1,"gateMs": {- "p50": 251,- "p95": 304,- "mean": 249,+ "p50": 529,+ "p95": 742,+ "mean": 504,"n": 9},"ttftMs": {- "p50": 2728,- "p95": 6411,- "mean": 3041,+ "p50": 2656,+ "p95": 5759,+ "mean": 3124,"n": 9},"totalMs": {- "p50": 3280,- "p95": 6511,- "mean": 3372,+ "p50": 3768,+ "p95": 5943,+ "mean": 3533,"n": 9},- "meanRounds": 1.5+ "meanRounds": 1},"multi-tool": {"cases": 9,- "passRate": 1,+ "passRate": 0.89,"gateMs": {- "p50": 245,- "p95": 301,- "mean": 252,- "n": 9+ "p50": 476,+ "p95": 580,+ "mean": 461,+ "n": 8},"ttftMs": {- "p50": 5792,- "p95": 32285,- "mean": 10035,- "n": 9+ "p50": 5673,+ "p95": 28945,+ "mean": 9066,+ "n": 8},"totalMs": {- "p50": 6118,- "p95": 32550,- "mean": 10335,- "n": 9+ "p50": 5895,+ "p95": 29504,+ "mean": 9412,+ "n": 8},- "meanRounds": 4.8+ "meanRounds": 3},"single-tool": {"cases": 24,- "passRate": 0.94,+ "passRate": 0.96,"gateMs": {- "p50": 255,- "p95": 319,- "mean": 289,- "n": 24+ "p50": 452,+ "p95": 620,+ "mean": 444,+ "n": 23},"ttftMs": {- "p50": 3166,- "p95": 11762,- "mean": 5085,- "n": 24+ "p50": 4551,+ "p95": 19592,+ "mean": 5992,+ "n": 23},"totalMs": {- "p50": 3433,- "p95": 12041,- "mean": 5332,- "n": 24+ "p50": 4756,+ "p95": 19752,+ "mean": 6183,+ "n": 23},- "meanRounds": 2.6+ "meanRounds": 2.3}},"cases": [@@ -86,34 +86,34 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 304,- "p95": 812,- "mean": 558,- "n": 2+ "p50": 742,+ "p95": 742,+ "mean": 742,+ "n": 1},"ttftMs": {- "p50": 2728,- "p95": 3610,- "mean": 3169,- "n": 2+ "p50": 3664,+ "p95": 3664,+ "mean": 3664,+ "n": 1},"totalMs": {- "p50": 2928,- "p95": 3713,- "mean": 3320,- "n": 2+ "p50": 3768,+ "p95": 3768,+ "mean": 3768,+ "n": 1},"modelMs": {- "p50": 2622,- "p95": 2891,- "mean": 2756,- "n": 2+ "p50": 3017,+ "p95": 3017,+ "mean": 3017,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 1,@@ -121,44 +121,6 @@}},"runs": [- {- "pass": true,- "intent": "complex",- "skill": null,- "areas": [- "timeline",- "inspector",- "subtitles",- "audio",- "library",- "side_panel",- "editor"- ],- "declaredTools": 91,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 811.7028329999999,- "gateSkipped": true,- "ttftMs": 3609.8061660000003,- "modelMs": 2891.1585,- "toolMs": 0,- "totalMs": 3712.7595410000004,- "judgeMs": 637.5777499999995,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2891.1585,- "firstDeltaMs": 2789.6068339999997,- "calls": []- }- ]- }- },{"pass": true,"intent": "complex",@@ -175,22 +137,23 @@"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [],"timings": {- "gateMs": 304.1766669999997,+ "gateMs": 742.2675000000002,"gateSkipped": true,- "ttftMs": 2727.847667,- "modelMs": 2621.7646250000007,+ "ttftMs": 3663.9621669999997,+ "modelMs": 3016.7649169999995,"toolMs": 0,- "totalMs": 2927.532292000001,- "judgeMs": 206.19129099999918,+ "totalMs": 3768.323375,+ "judgeMs": 228.9335000000001,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2621.7646250000007,- "firstDeltaMs": 2422.3260420000006,+ "roundMs": 3016.7649169999995,+ "firstDeltaMs": 2913.461125,"calls": []}]@@ -205,38 +168,38 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 263,- "p95": 316,- "mean": 290,- "n": 2+ "p50": 529,+ "p95": 529,+ "mean": 529,+ "n": 1},"ttftMs": {- "p50": 4319,- "p95": 30883,- "mean": 17601,- "n": 2+ "p50": 2484,+ "p95": 2484,+ "mean": 2484,+ "n": 1},"totalMs": {- "p50": 4319,- "p95": 31109,- "mean": 17714,- "n": 2+ "p50": 2533,+ "p95": 2533,+ "mean": 2533,+ "n": 1},"modelMs": {- "p50": 4053,- "p95": 30789,- "mean": 17421,- "n": 2+ "p50": 2002,+ "p95": 2002,+ "mean": 2002,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 1,+ "max": 1}},"runs": [@@ -249,62 +212,23 @@"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [],"timings": {- "gateMs": 316.32133299999987,- "gateSkipped": false,- "ttftMs": 30882.879916,- "modelMs": 30789.048667000003,- "toolMs": 0,- "totalMs": 31108.895458000003,- "judgeMs": 538.7172090000022,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3538.2138749999995,- "firstDeltaMs": 3307.114125,- "calls": []- },- {- "roundMs": 27250.834792,- "firstDeltaMs": 27026.010374999998,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "chat",- "skill": null,- "areas": [],- "declaredTools": 0,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 263.0132920000033,+ "gateMs": 528.8679589999992,"gateSkipped": false,- "ttftMs": 4318.823958000001,- "modelMs": 4053.2067920000045,+ "ttftMs": 2483.8594999999996,+ "modelMs": 2002.343833,"toolMs": 0,- "totalMs": 4319.161749999999,- "judgeMs": 606.3485000000001,+ "totalMs": 2533.1364169999997,+ "judgeMs": 346.0425000000005,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1134.1152920000022,- "firstDeltaMs": 1008.7466249999998,- "calls": []- },- {- "roundMs": 2919.0915000000023,- "firstDeltaMs": 2918.957416999998,+ "roundMs": 2002.343833,+ "firstDeltaMs": 1953.405417,"calls": []}]@@ -319,38 +243,38 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 251,- "p95": 282,- "mean": 266,- "n": 2+ "p50": 376,+ "p95": 376,+ "mean": 376,+ "n": 1},"ttftMs": {- "p50": 4282,- "p95": 4923,- "mean": 4603,- "n": 2+ "p50": 4685,+ "p95": 4685,+ "mean": 4685,+ "n": 1},"totalMs": {- "p50": 4458,- "p95": 4972,- "mean": 4715,- "n": 2+ "p50": 4741,+ "p95": 4741,+ "mean": 4741,+ "n": 1},"modelMs": {- "p50": 4206,- "p95": 4677,- "mean": 4441,- "n": 2+ "p50": 4363,+ "p95": 4363,+ "mean": 4363,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 1,+ "max": 1}},"runs": [@@ -363,62 +287,23 @@"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [],"timings": {- "gateMs": 281.59308299999975,- "gateSkipped": false,- "ttftMs": 4922.769625000001,- "modelMs": 4676.804083000003,- "toolMs": 0,- "totalMs": 4972.416833000003,- "judgeMs": 489.70624999999563,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1204.658457999998,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 3472.1456250000047,- "firstDeltaMs": 3422.9637500000026,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "chat",- "skill": null,- "areas": [],- "declaredTools": 0,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 251.05025000000023,+ "gateMs": 375.9370420000005,"gateSkipped": false,- "ttftMs": 4282.4438329999975,- "modelMs": 4205.652916999999,+ "ttftMs": 4684.84525,+ "modelMs": 4363.426416,"toolMs": 0,- "totalMs": 4458.468416999996,- "judgeMs": 1035.0701249999984,+ "totalMs": 4740.879457999999,+ "judgeMs": 600.5082500000008,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1077.3654590000006,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 3128.287457999999,- "firstDeltaMs": 2952.6718330000003,+ "roundMs": 4363.426416,+ "firstDeltaMs": 4307.648416,"calls": []}]@@ -433,34 +318,34 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 291,- "p95": 332,- "mean": 311,- "n": 2+ "p50": 447,+ "p95": 447,+ "mean": 447,+ "n": 1},"ttftMs": {- "p50": 2914,- "p95": 3019,- "mean": 2967,- "n": 2+ "p50": 3329,+ "p95": 3329,+ "mean": 3329,+ "n": 1},"totalMs": {- "p50": 3090,- "p95": 3278,- "mean": 3184,- "n": 2+ "p50": 3512,+ "p95": 3512,+ "mean": 3512,+ "n": 1},"modelMs": {- "p50": 2795,- "p95": 2941,- "mean": 2868,- "n": 2+ "p50": 3050,+ "p95": 3050,+ "mean": 3050,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -480,136 +365,38 @@"library","editor"],- "declaredTools": 85,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "update_cue",- "round": 0,- "atMs": 1818.1259169999976,- "execMs": 0.28608299999905284,- "args": {- "text": "so today we're",- "id": "cue1"- }- },- {- "name": "update_cue",- "round": 0,- "atMs": 1818.9567919999972,- "execMs": 0.004207999998470768,- "args": {- "id": "cue3",- "text": "and they absolutely love the water"- }- },- {- "name": "delete_cue",- "round": 0,- "atMs": 1819.2722079999949,- "execMs": 0.02195899999787798,- "args": {- "id": "cue4"- }- },- {- "name": "update_cue",- "round": 0,- "atMs": 1819.3593330000003,- "execMs": 0.002542000001994893,- "args": {- "text": "so let's watch them play",- "id": "cue5"- }- }- ],- "timings": {- "gateMs": 331.9095410000009,- "gateSkipped": false,- "ttftMs": 3018.7580420000013,- "modelMs": 2940.6652909999975,- "toolMs": 0.3147919999973965,- "totalMs": 3277.793249999995,- "judgeMs": 304.1790420000034,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1483.0979999999981,- "firstDeltaMs": null,- "calls": [- {- "name": "update_cue",- "execMs": 0.28608299999905284- },- {- "name": "update_cue",- "execMs": 0.004207999998470768- },- {- "name": "delete_cue",- "execMs": 0.02195899999787798- },- {- "name": "update_cue",- "execMs": 0.002542000001994893- }- ]- },- {- "roundMs": 1457.5672909999994,- "firstDeltaMs": 1198.645958000001,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "subtitles",- "library",- "editor"- ],- "declaredTools": 85,+ "declaredTools": 86,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "update_cue","round": 0,- "atMs": 1847.401791999997,- "execMs": 0.007083999997121282,+ "atMs": 2172.778875,+ "execMs": 0.29262500000004366,"args": {- "text": "So today we're",- "id": "cue1"+ "id": "cue1",+ "text": "So today we're"}},{"name": "update_cue","round": 0,- "atMs": 1847.5505419999972,- "execMs": 0.00391700000182027,+ "atMs": 2173.8610420000005,+ "execMs": 0.005166999999346444,"args": {"id": "cue3",- "text": "and it's great."+ "text": "And..."}},{"name": "align_to_audio","round": 0,- "atMs": 1848.802791999995,- "execMs": 0.005500000006577466,+ "atMs": 2174.714249999999,+ "execMs": 0.003582999999707681,"args": {"target": "captions","track": 0@@ -617,37 +404,37 @@}],"timings": {- "gateMs": 291.03254200000083,+ "gateMs": 447.4327080000003,"gateSkipped": false,- "ttftMs": 2914.2705419999984,- "modelMs": 2794.726251,- "toolMs": 0.016501000005519018,- "totalMs": 3090.3525420000005,- "judgeMs": 351.53241700000217,+ "ttftMs": 3328.927875000001,+ "modelMs": 3049.699666999999,+ "toolMs": 0.3013749999990978,+ "totalMs": 3512.293834,+ "judgeMs": 268.95491700000275,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1554.4014590000006,+ "roundMs": 1714.9200419999997,"firstDeltaMs": null,"calls": [{"name": "update_cue",- "execMs": 0.007083999997121282+ "execMs": 0.29262500000004366},{"name": "update_cue",- "execMs": 0.00391700000182027+ "execMs": 0.005166999999346444},{"name": "align_to_audio",- "execMs": 0.005500000006577466+ "execMs": 0.003582999999707681}]},{- "roundMs": 1240.3247919999994,- "firstDeltaMs": 1064.6275000000023,+ "roundMs": 1334.7796249999992,+ "firstDeltaMs": 1153.1853330000013,"calls": []}]@@ -662,34 +449,34 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 239,- "p95": 253,- "mean": 246,- "n": 2+ "p50": 697,+ "p95": 697,+ "mean": 697,+ "n": 1},"ttftMs": {- "p50": 1176,- "p95": 1269,- "mean": 1222,- "n": 2+ "p50": 1735,+ "p95": 1735,+ "mean": 1735,+ "n": 1},"totalMs": {- "p50": 1276,- "p95": 1444,- "mean": 1360,- "n": 2+ "p50": 1905,+ "p95": 1905,+ "mean": 1905,+ "n": 1},"modelMs": {- "p50": 1035,- "p95": 1187,- "mean": 1111,- "n": 2+ "p50": 1201,+ "p95": 1201,+ "mean": 1201,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 1,@@ -702,60 +489,30 @@"intent": "simple","skill": null,"areas": [- "timeline"- ],- "declaredTools": 44,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 253.24716699999408,- "gateSkipped": false,- "ttftMs": 1269.1092500000013,- "modelMs": 1187.2047910000038,- "toolMs": 0,- "totalMs": 1444.1679999999978,- "judgeMs": 244.8036659999998,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1187.2047910000038,- "firstDeltaMs": 1012.2492910000001,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "timeline"+ "timeline",+ "top_bar"],- "declaredTools": 44,+ "declaredTools": 49,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [],"timings": {- "gateMs": 238.75079099999857,+ "gateMs": 697.3682919999992,"gateSkipped": false,- "ttftMs": 1175.5901660000018,- "modelMs": 1035.3704170000055,+ "ttftMs": 1735.3238330000022,+ "modelMs": 1201.3779170000016,"toolMs": 0,- "totalMs": 1276.0852499999964,- "judgeMs": 271.54412500000035,+ "totalMs": 1905.153666000002,+ "judgeMs": 242.05804200000057,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1035.3704170000055,- "firstDeltaMs": 935.2327500000029,+ "roundMs": 1201.3779170000016,+ "firstDeltaMs": 1032.3630840000005,"calls": []}]@@ -770,34 +527,34 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 250,- "p95": 274,- "mean": 262,- "n": 2+ "p50": 324,+ "p95": 324,+ "mean": 324,+ "n": 1},"ttftMs": {- "p50": 5792,- "p95": 7322,- "mean": 6557,- "n": 2+ "p50": 28945,+ "p95": 28945,+ "mean": 28945,+ "n": 1},"totalMs": {- "p50": 6118,- "p95": 7661,- "mean": 6889,- "n": 2+ "p50": 29504,+ "p95": 29504,+ "mean": 29504,+ "n": 1},"modelMs": {- "p50": 5840,- "p95": 7407,- "mean": 6624,- "n": 2+ "p50": 29170,+ "p95": 29170,+ "mean": 29170,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -819,23 +576,23 @@"audio","video_gen","scene",- "stock","library","side_panel","top_bar","editor"],- "declaredTools": 112,+ "declaredTools": 111,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "add_clip","round": 0,- "atMs": 3654.9874589999963,- "execMs": 0.09112500000628643,+ "atMs": 23231.843917,+ "execMs": 0.12508299999899464,"args": {"asset_id": "a-i1"}@@ -843,163 +600,83 @@{"name": "add_clip","round": 0,- "atMs": 3655.246874999997,- "execMs": 0.003874999994877726,+ "atMs": 23232.451457999996,+ "execMs": 0.00466700000106357,"args": {"asset_id": "a-i2"}}],"timings": {- "gateMs": 274.00900000000547,+ "gateMs": 323.57904100000087,"gateSkipped": true,- "ttftMs": 5791.829333999995,- "modelMs": 5840.383915999992,- "toolMs": 0.09500000000116415,- "totalMs": 6118.179999999993,- "judgeMs": 772.9698329999956,+ "ttftMs": 28944.707083,+ "modelMs": 29170.149917000006,+ "toolMs": 0.1297500000000582,+ "totalMs": 29504.305291999997,+ "judgeMs": 501.3070000000007,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 3378.5229580000014,+ "roundMs": 22899.947292,"firstDeltaMs": null,"calls": [{"name": "add_clip",- "execMs": 0.09112500000628643+ "execMs": 0.12508299999899464},{"name": "add_clip",- "execMs": 0.003874999994877726+ "execMs": 0.00466700000106357}]},{- "roundMs": 2461.86095799999,- "firstDeltaMs": 2135.6040829999984,+ "roundMs": 6270.202625000005,+ "firstDeltaMs": 5710.906500000005,"calls": []}]}- },- {- "pass": true,- "intent": "complex",- "skill": "timeline-editing",- "areas": [- "preview",- "timeline",- "inspector",- "animation",- "transitions",- "subtitles",- "audio",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 112,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "add_clip",- "round": 0,- "atMs": 5664.788457999995,- "execMs": 0.009791000004042871,- "args": {- "asset_id": "a-i1",- "index": 1- }- },- {- "name": "add_clip",- "round": 0,- "atMs": 5665.031625000003,- "execMs": 0.0037089999968884513,- "args": {- "index": 2,- "asset_id": "a-i2"- }- }- ],- "timings": {- "gateMs": 249.76866600000358,- "gateSkipped": true,- "ttftMs": 7321.556125000003,- "modelMs": 7407.458249999996,- "toolMs": 0.013500000000931323,- "totalMs": 7660.595541000002,- "judgeMs": 519.989167000007,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 5413.399124999996,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.009791000004042871- },- {- "name": "add_clip",- "execMs": 0.0037089999968884513- }- ]- },- {- "roundMs": 1994.0591249999998,- "firstDeltaMs": 1655.4207499999902,- "calls": []- }- ]- }- }- ]- },- {- "name": "photos-question-stays-in-chat",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 230,- "p95": 254,- "mean": 242,- "n": 2+ }+ ]+ },+ {+ "name": "photos-question-stays-in-chat",+ "bucket": "chat",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 273,+ "p95": 273,+ "mean": 273,+ "n": 1},"ttftMs": {- "p50": 2151,- "p95": 2347,- "mean": 2249,- "n": 2+ "p50": 2656,+ "p95": 2656,+ "mean": 2656,+ "n": 1},"totalMs": {- "p50": 3280,- "p95": 3711,- "mean": 3495,- "n": 2+ "p50": 3898,+ "p95": 3898,+ "mean": 3898,+ "n": 1},"modelMs": {- "p50": 3046,- "p95": 3453,- "mean": 3250,- "n": 2+ "p50": 3621,+ "p95": 3621,+ "mean": 3621,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 1,@@ -1028,71 +705,27 @@"top_bar","editor"],- "declaredTools": 112,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": "concise factual 0.21; upsell 0.84; long 1.9",- "trace": [],- "timings": {- "gateMs": 230.47950000000128,- "gateSkipped": true,- "ttftMs": 2150.78966699999,- "modelMs": 3045.852583,- "toolMs": 0,- "totalMs": 3279.6020419999986,- "judgeMs": 226.97287500000675,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3045.852583,- "firstDeltaMs": 1917.5350000000035,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "elements",- "effects",- "transitions",- "subtitles",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 110,+ "declaredTools": 113,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],- "judgeNote": "concise factual 0.20; upsell 0.55; long 2.0",+ "instant": null,+ "judgeNote": "concise factual 0.17; upsell 0.89; long 1.9","trace": [],"timings": {- "gateMs": 254.1837910000031,+ "gateMs": 272.50187499999447,"gateSkipped": true,- "ttftMs": 2346.775207999992,- "modelMs": 3453.3652920000022,+ "ttftMs": 2656.004375000004,+ "modelMs": 3621.348416999994,"toolMs": 0,- "totalMs": 3711.0452919999952,- "judgeMs": 332.4827920000098,+ "totalMs": 3898.3247919999994,+ "judgeMs": 286.1975420000017,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 3453.3652920000022,- "firstDeltaMs": 2090.231500000009,+ "roundMs": 3621.348416999994,+ "firstDeltaMs": 2379.258916999999,"calls": []}]@@ -1107,38 +740,38 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 259,- "p95": 263,- "mean": 261,- "n": 2+ "p50": 375,+ "p95": 375,+ "mean": 375,+ "n": 1},"ttftMs": {- "p50": 1434,- "p95": 8678,- "mean": 5056,- "n": 2+ "p50": 5759,+ "p95": 5759,+ "mean": 5759,+ "n": 1},"totalMs": {- "p50": 1696,- "p95": 8806,- "mean": 5251,- "n": 2+ "p50": 5943,+ "p95": 5943,+ "mean": 5943,+ "n": 1},"modelMs": {- "p50": 1432,- "p95": 8539,- "mean": 4985,- "n": 2+ "p50": 5565,+ "p95": 5565,+ "mean": 5565,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 1.5,- "max": 2+ "mean": 1,+ "max": 1}},"runs": [@@ -1156,62 +789,23 @@"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [],"timings": {- "gateMs": 259.3814999999886,- "gateSkipped": false,- "ttftMs": 8677.94183299999,- "modelMs": 8538.731751,- "toolMs": 0,- "totalMs": 8805.912207999994,- "judgeMs": 548.3632499999949,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3911.315333999999,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 4627.416417,- "firstDeltaMs": 4499.9619590000075,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 38,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 262.62966599999345,+ "gateMs": 375.1201250000013,"gateSkipped": false,- "ttftMs": 1433.572291000004,- "modelMs": 1431.7364170000073,+ "ttftMs": 5759.179917000001,+ "modelMs": 5564.7642080000005,"toolMs": 0,- "totalMs": 1696.436333000005,- "judgeMs": 273.283208000008,+ "totalMs": 5943.383834,+ "judgeMs": 589.8631250000035,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1431.7364170000073,- "firstDeltaMs": 1169.2181249999994,+ "roundMs": 5564.7642080000005,+ "firstDeltaMs": 5380.985207999998,"calls": []}]@@ -1226,34 +820,34 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 263,- "p95": 266,- "mean": 265,- "n": 2+ "p50": 506,+ "p95": 506,+ "mean": 506,+ "n": 1},"ttftMs": {- "p50": 3166,- "p95": 3209,- "mean": 3188,- "n": 2+ "p50": 6569,+ "p95": 6569,+ "mean": 6569,+ "n": 1},"totalMs": {- "p50": 3433,- "p95": 3436,- "mean": 3434,- "n": 2+ "p50": 6693,+ "p95": 6693,+ "mean": 6693,+ "n": 1},"modelMs": {- "p50": 3158,- "p95": 3166,- "mean": 3162,- "n": 2+ "p50": 6178,+ "p95": 6178,+ "mean": 6178,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -1269,7 +863,6 @@"preview","timeline","inspector",- "animation","transitions","subtitles","audio",@@ -1279,54 +872,98 @@"stock","library","side_panel",- "top_bar",- "editor"+ "top_bar"],- "declaredTools": 113,+ "declaredTools": 108,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "generate_video","round": 0,- "atMs": 1752.7311659999978,- "execMs": 0.01779099999112077,+ "atMs": 5054.001750000003,+ "execMs": 0.008249999998952262,"args": {- "prompt": "A joyful dog running and playing in a sunny park, cinematic lighting."+ "prompt": "A playful dog running happily through a green park, sunny day, cinematic style."}}],"timings": {- "gateMs": 266.34341600000334,+ "gateMs": 505.6267499999958,"gateSkipped": false,- "ttftMs": 3209.0444579999894,- "modelMs": 3158.371082999991,- "toolMs": 0.01779099999112077,- "totalMs": 3432.8083329999936,- "judgeMs": 260.37766700000793,+ "ttftMs": 6569.255584000006,+ "modelMs": 6178.490166999996,+ "toolMs": 0.008249999998952262,+ "totalMs": 6692.863583999999,+ "judgeMs": 530.565208,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1480.4649579999968,+ "roundMs": 4541.619249999996,"firstDeltaMs": null,"calls": [{"name": "generate_video",- "execMs": 0.01779099999112077+ "execMs": 0.008249999998952262}]},{- "roundMs": 1677.906124999994,- "firstDeltaMs": 1454.6185830000031,+ "roundMs": 1636.8709170000002,+ "firstDeltaMs": 1513.5462920000064,"calls": []}]}+ }+ ]+ },+ {+ "name": "video-ask-with-size-and-length",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 452,+ "p95": 452,+ "mean": 452,+ "n": 1+ },+ "ttftMs": {+ "p50": 3412,+ "p95": 3412,+ "mean": 3412,+ "n": 1+ },+ "totalMs": {+ "p50": 3868,+ "p95": 3868,+ "mean": 3868,+ "n": 1+ },+ "modelMs": {+ "p50": 3405,+ "p95": 3405,+ "mean": 3405,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,"intent": "simple",@@ -1335,57 +972,60 @@"preview","timeline","inspector",- "transitions",- "subtitles",- "audio",+ "effects",+ "animation","image_gen","video_gen","scene","stock","library","side_panel",- "top_bar"+ "top_bar",+ "editor"],- "declaredTools": 107,+ "declaredTools": 98,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "generate_video","round": 0,- "atMs": 1754.778250000003,- "execMs": 0.009125000011408702,+ "atMs": 2082.371459000002,+ "execMs": 0.2822920000035083,"args": {- "prompt": "A cheerful dog running happily through a lush green park, sunny day, high definition, slow motion."+ "prompt": "A paper boat drifting across a puddle, cinematic lighting, 1080p.",+ "resolution": "1080p",+ "duration_seconds": 6}}],"timings": {- "gateMs": 263.310083999997,+ "gateMs": 451.9445839999971,"gateSkipped": false,- "ttftMs": 3166.4713750000083,- "modelMs": 3166.453999999998,- "toolMs": 0.009125000011408702,- "totalMs": 3435.7610419999983,- "judgeMs": 277.9326669999864,+ "ttftMs": 3412.0437919999968,+ "modelMs": 3405.1449589999975,+ "toolMs": 0.2822920000035083,+ "totalMs": 3868.4129590000084,+ "judgeMs": 276.29754200000025,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1488.7930000000051,+ "roundMs": 1621.674708999999,"firstDeltaMs": null,"calls": [{"name": "generate_video",- "execMs": 0.009125000011408702+ "execMs": 0.2822920000035083}]},{- "roundMs": 1677.6609999999928,- "firstDeltaMs": 1408.8366249999963,+ "roundMs": 1783.4702499999985,+ "firstDeltaMs": 1327.4159169999912,"calls": []}]@@ -1394,33 +1034,33 @@]},{- "name": "video-ask-with-size-and-length",+ "name": "iterate-keeps-existing-clip","bucket": "single-tool",- "passRate": 0.5,+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1124,- "p95": 1124,- "mean": 1124,+ "p50": 465,+ "p95": 465,+ "mean": 465,"n": 1},"ttftMs": {- "p50": 15281,- "p95": 15281,- "mean": 15281,+ "p50": 5161,+ "p95": 5161,+ "mean": 5161,"n": 1},"totalMs": {- "p50": 16012,- "p95": 16012,- "mean": 16012,+ "p50": 5478,+ "p95": 5478,+ "mean": 5478,"n": 1},"modelMs": {- "p50": 15599,- "p95": 15599,- "mean": 15599,+ "p50": 5007,+ "p95": 5007,+ "mean": 5007,"n": 1},"toolMs": {@@ -1435,75 +1075,65 @@}},"runs": [- {- "pass": false,- "intent": "work",- "notes": [],- "trace": [],- "timings": null,- "error": "This operation was aborted"- },{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "color-grading","areas": ["preview","timeline","inspector",+ "color","effects",- "animation",- "image_gen","video_gen","scene",- "stock",- "library","side_panel","top_bar","editor"],- "declaredTools": 97,+ "declaredTools": 93,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "generate_video","round": 0,- "atMs": 10523.85370800001,- "execMs": 0.0022080000053392723,+ "atMs": 3063.68908299999,+ "execMs": 0.20266699999046978,"args": {- "prompt": "a paper boat drifting across a puddle, cinematic lighting, 1080p",- "duration_seconds": 6,- "resolution": "1080p"+ "prompt": "Scenic beach at golden-hour sunset, warm glowing light, gentle ocean waves rolling onto the sand, sun dipping toward the horizon, cinematic camera movement",+ "reference_asset_id": "a-v1",+ "aspect": "9:16"}}],"timings": {- "gateMs": 1124.1443750000035,+ "gateMs": 464.5534590000025,"gateSkipped": false,- "ttftMs": 15281.440125000008,- "modelMs": 15598.731249999997,- "toolMs": 0.0022080000053392723,- "totalMs": 16012.236250000002,- "judgeMs": 711.551833999998,+ "ttftMs": 5160.882582999999,+ "modelMs": 5007.143416999999,+ "toolMs": 0.20266699999046978,+ "totalMs": 5478.474665999995,+ "judgeMs": 531.4503750000003,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 10118.337124999991,+ "roundMs": 2595.5232920000126,"firstDeltaMs": null,"calls": [{"name": "generate_video",- "execMs": 0.0022080000053392723+ "execMs": 0.20266699999046978}]},{- "roundMs": 5480.394125000006,- "firstDeltaMs": 4750.2749579999945,+ "roundMs": 2411.6201249999867,+ "firstDeltaMs": 2094.7140409999993,"calls": []}]@@ -1512,40 +1142,40 @@]},{- "name": "iterate-keeps-existing-clip",+ "name": "mute-all-does-not-redo-prior-turn","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 303,- "p95": 322,- "mean": 313,- "n": 2+ "p50": 537,+ "p95": 537,+ "mean": 537,+ "n": 1},"ttftMs": {- "p50": 6209,- "p95": 11595,- "mean": 8902,- "n": 2+ "p50": 4551,+ "p95": 4551,+ "mean": 4551,+ "n": 1},"totalMs": {- "p50": 6431,- "p95": 11925,- "mean": 9178,- "n": 2+ "p50": 4697,+ "p95": 4697,+ "mean": 4697,+ "n": 1},"modelMs": {- "p50": 6118,- "p95": 11573,- "mean": 8846,- "n": 2+ "p50": 4146,+ "p95": 4146,+ "mean": 4146,+ "n": 1},"toolMs": {- "p50": 0,- "p95": 3,+ "p50": 1,+ "p95": 1,"mean": 1,- "n": 2+ "n": 1},"rounds": {"mean": 3,@@ -1555,134 +1185,194 @@"runs": [{"pass": true,- "intent": "complex",- "skill": "color-grading",+ "intent": "simple",+ "skill": null,"areas": [- "preview","timeline","inspector",- "color",- "effects",- "video_gen",- "scene",- "side_panel",- "top_bar"+ "audio",+ "editor"],- "declaredTools": 88,+ "declaredTools": 65,"routingMisses": [],- "roundModel": "gemini-3.8-flash",+ "roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "generate_video",+ "name": "select_items",+ "round": 0,+ "atMs": 2080.143790999995,+ "execMs": 0.6971659999981057,+ "args": {+ "describe": "all clips",+ "kind": "clip"+ }+ },+ {+ "name": "set_clip_muted","round": 1,- "atMs": 7802.556916000001,- "execMs": 2.6554999999934807,+ "atMs": 3460.424374999988,+ "execMs": 0.011958000002778135,"args": {- "reference_asset_id": "a-v1",- "prompt": "Scenic beach at golden-hour sunset, warm glowing sunlight reflecting across ocean waves and wet sand, gentle cinematic camera motion, peaceful evening light"+ "muted": true,+ "ids": [+ "c1",+ "c2",+ "c3"+ ]}}],"timings": {- "gateMs": 321.89491600000474,+ "gateMs": 536.8160420000058,"gateSkipped": false,- "ttftMs": 11595.309333000012,- "modelMs": 11573.099707999994,- "toolMs": 2.6554999999934807,- "totalMs": 11924.54104099999,- "judgeMs": 540.6497089999903,+ "ttftMs": 4550.54912499999,+ "modelMs": 4145.760459000012,+ "toolMs": 0.7091240000008838,+ "totalMs": 4696.913625000001,+ "judgeMs": 582.5177919999987,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1767.5983750000014,+ "roundMs": 1536.341292000012,"firstDeltaMs": null,- "calls": []- },- {- "roundMs": 5688.663333000004,+ "calls": [+ {+ "name": "select_items",+ "execMs": 0.6971659999981057+ }+ ]+ },+ {+ "roundMs": 1374.874125000002,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 2.6554999999934807+ "name": "set_clip_muted",+ "execMs": 0.011958000002778135}]},{- "roundMs": 4116.837999999989,- "firstDeltaMs": 3787.974208999978,+ "roundMs": 1234.545041999998,+ "firstDeltaMs": 1088.6765840000007,"calls": []}]}+ }+ ]+ },+ {+ "name": "imported-photo-edit-references-photo",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 514,+ "p95": 514,+ "mean": 514,+ "n": 1+ },+ "ttftMs": {+ "p50": 4378,+ "p95": 4378,+ "mean": 4378,+ "n": 1+ },+ "totalMs": {+ "p50": 4756,+ "p95": 4756,+ "mean": 4756,+ "n": 1+ },+ "modelMs": {+ "p50": 4237,+ "p95": 4237,+ "mean": 4237,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,- "intent": "complex",- "skill": "color-grading",+ "intent": "simple",+ "skill": null,"areas": ["preview","timeline","inspector",- "color",+ "removal",+ "elements","effects",+ "animation",+ "image_gen","video_gen","scene","stock","library","side_panel",- "top_bar"+ "top_bar",+ "editor"],- "declaredTools": 97,+ "declaredTools": 104,"routingMisses": [],- "roundModel": "gemini-3.8-flash",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": null,+ "instant": null,+ "judgeNote": "concise factual 0.46","trace": [{- "name": "generate_video",- "round": 1,- "atMs": 4222.986291999987,- "execMs": 0.014374999998835847,+ "name": "generate_image",+ "round": 0,+ "atMs": 2897.7629999999917,+ "execMs": 0.08483300000079907,"args": {- "reference_asset_id": "a-v1",- "prompt": "A scenic beach bathed in warm golden-hour sunset light, gentle ocean waves lapping at the shore, soft warm orange and pink horizon, cinematic camera motion",- "aspect": "9:16"+ "reference_asset_ids": [+ "a-tw1"+ ],+ "prompt": "The girl in the photo from this post, wearing a Santa hat. The style, lighting, and composition are identical to the original image."}}],"timings": {- "gateMs": 303.25975000002654,+ "gateMs": 514.291750000004,"gateSkipped": false,- "ttftMs": 6209.459000000003,- "modelMs": 6117.910833000002,- "toolMs": 0.014374999998835847,- "totalMs": 6431.130208999995,- "judgeMs": 497.73662499999045,+ "ttftMs": 4377.539250000002,+ "modelMs": 4237.1919999999955,+ "toolMs": 0.08483300000079907,+ "totalMs": 4756.461207999993,+ "judgeMs": 547.1037919999944,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1569.9868330000027,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2341.4284159999806,+ "roundMs": 2380.1294999999955,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.014374999998835847+ "name": "generate_image",+ "execMs": 0.08483300000079907}]},{- "roundMs": 2206.4955840000184,- "firstDeltaMs": 1984.9706250000163,+ "roundMs": 1857.0625,+ "firstDeltaMs": 1478.8639589999948,"calls": []}]@@ -1691,40 +1381,40 @@]},{- "name": "mute-all-does-not-redo-prior-turn",+ "name": "audio-ask-single-generate","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 217,- "p95": 227,- "mean": 222,- "n": 2+ "p50": 621,+ "p95": 621,+ "mean": 621,+ "n": 1},"ttftMs": {- "p50": 2833,- "p95": 2954,- "mean": 2894,- "n": 2+ "p50": 4195,+ "p95": 4195,+ "mean": 4195,+ "n": 1},"totalMs": {- "p50": 3014,- "p95": 3059,- "mean": 3037,- "n": 2+ "p50": 4339,+ "p95": 4339,+ "mean": 4339,+ "n": 1},"modelMs": {- "p50": 2793,- "p95": 2823,- "mean": 2808,- "n": 2+ "p50": 3711,+ "p95": 3711,+ "mean": 3711,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -1740,163 +1430,165 @@"preview","timeline","inspector",+ "subtitles","audio",- "editor"+ "video_gen",+ "scene",+ "stock",+ "library",+ "side_panel",+ "top_bar"],- "declaredTools": 69,+ "declaredTools": 107,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_clip_muted",- "round": 0,- "atMs": 1902.9193749999977,- "execMs": 0.06995800000731833,- "args": {- "clipId": "c1",- "muted": true- }- },- {- "name": "set_clip_muted",- "round": 0,- "atMs": 1903.2229999999981,- "execMs": 0.0027500000142026693,- "args": {- "muted": true,- "clipId": "c2"- }- },- {- "name": "set_clip_muted",+ "name": "voiceover_generate","round": 0,- "atMs": 1903.465750000003,- "execMs": 0.003874999994877726,+ "atMs": 2717.6584580000053,+ "execMs": 0.01224999999976717,"args": {- "muted": true,- "clipId": "c3"+ "script": "Welcome to the dog show."}}],"timings": {- "gateMs": 226.62166699999943,+ "gateMs": 621.4086249999964,"gateSkipped": false,- "ttftMs": 2953.701915999991,- "modelMs": 2823.3891669999866,- "toolMs": 0.07658300001639873,- "totalMs": 3058.836290999985,- "judgeMs": 206.96162500002538,+ "ttftMs": 4194.916041999997,+ "modelMs": 3711.3718760000047,+ "toolMs": 0.01224999999976717,+ "totalMs": 4339.3164169999945,+ "judgeMs": 711.4737920000043,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1669.7185839999875,+ "roundMs": 2091.9540419999976,"firstDeltaMs": null,"calls": [{- "name": "set_clip_muted",- "execMs": 0.06995800000731833- },- {- "name": "set_clip_muted",- "execMs": 0.0027500000142026693- },- {- "name": "set_clip_muted",- "execMs": 0.003874999994877726+ "name": "voiceover_generate",+ "execMs": 0.01224999999976717}]},{- "roundMs": 1153.6705829999992,- "firstDeltaMs": 1048.986957999994,+ "roundMs": 1619.4178340000071,+ "firstDeltaMs": 1475.3925420000014,"calls": []}]}+ }+ ]+ },+ {+ "name": "music-ask-single-generate",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 311,+ "p95": 311,+ "mean": 311,+ "n": 1},+ "ttftMs": {+ "p50": 5346,+ "p95": 5346,+ "mean": 5346,+ "n": 1+ },+ "totalMs": {+ "p50": 5580,+ "p95": 5580,+ "mean": 5580,+ "n": 1+ },+ "modelMs": {+ "p50": 5262,+ "p95": 5262,+ "mean": 5262,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "audio-and-subtitles","areas": ["preview","timeline","inspector",+ "subtitles","audio",- "editor"+ "stock",+ "library",+ "side_panel"],- "declaredTools": 69,+ "declaredTools": 95,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_clip_muted",- "round": 0,- "atMs": 1580.9433750000026,- "execMs": 0.01179200000478886,- "args": {- "clipId": "c1",- "muted": true- }- },- {- "name": "set_clip_muted",- "round": 0,- "atMs": 1581.1917919999978,- "execMs": 0.0027920000138692558,- "args": {- "clipId": "c2",- "muted": true- }- },- {- "name": "set_clip_muted",+ "name": "generate_music","round": 0,- "atMs": 1581.402833,- "execMs": 0.002207999990787357,+ "atMs": 2909.1016669999954,+ "execMs": 0.07000000000698492,"args": {- "muted": true,- "clipId": "c3"+ "volume": 0.4,+ "start": 0,+ "instrumental": true,+ "add_to_timeline": true,+ "prompt": "Chill, mellow lofi hip hop background music with soft electric piano, smooth ambient textures, and gentle downtempo beat",+ "length": "clip"}}],"timings": {- "gateMs": 217.39104200000293,+ "gateMs": 311.4786249999888,"gateSkipped": false,- "ttftMs": 2833.381625000009,- "modelMs": 2792.722999999998,- "toolMs": 0.016792000009445474,- "totalMs": 3014.457208000007,- "judgeMs": 218.88420800000313,+ "ttftMs": 5345.563500000004,+ "modelMs": 5262.396999999997,+ "toolMs": 0.07000000000698492,+ "totalMs": 5580.217499999999,+ "judgeMs": 525.0467920000083,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1361.5271249999932,+ "roundMs": 2593.245041999995,"firstDeltaMs": null,"calls": [{- "name": "set_clip_muted",- "execMs": 0.01179200000478886- },- {- "name": "set_clip_muted",- "execMs": 0.0027920000138692558- },- {- "name": "set_clip_muted",- "execMs": 0.002207999990787357+ "name": "generate_music",+ "execMs": 0.07000000000698492}]},{- "roundMs": 1431.195875000005,- "firstDeltaMs": 1250.511415999994,+ "roundMs": 2669.151958000002,+ "firstDeltaMs": 2434.863249999995,"calls": []}]@@ -1905,40 +1597,40 @@]},{- "name": "imported-photo-edit-references-photo",+ "name": "trim-ask-single-tool","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 220,- "p95": 266,- "mean": 243,- "n": 2+ "p50": 476,+ "p95": 476,+ "mean": 476,+ "n": 1},"ttftMs": {- "p50": 3314,- "p95": 3858,- "mean": 3586,- "n": 2+ "p50": 3297,+ "p95": 3297,+ "mean": 3297,+ "n": 1},"totalMs": {- "p50": 3667,- "p95": 4203,- "mean": 3935,- "n": 2+ "p50": 3723,+ "p95": 3723,+ "mean": 3723,+ "n": 1},"modelMs": {- "p50": 3438,- "p95": 3931,- "mean": 3685,- "n": 2+ "p50": 3240,+ "p95": 3240,+ "mean": 3240,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -1954,132 +1646,157 @@"preview","timeline","inspector",- "removal",- "elements",- "effects",- "animation",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar","editor"],- "declaredTools": 103,+ "declaredTools": 66,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "generate_image",+ "name": "trim_clip","round": 0,- "atMs": 1988.851915999985,- "execMs": 0.2047080000047572,+ "atMs": 1824.863083999997,+ "execMs": 0.013250000003608875,"args": {- "reference_asset_ids": [- "a-tw1"- ],- "prompt": "a photo of the person in the reference image wearing a santa hat, maintaining the style, lighting, and composition of the original photo"+ "in": 0,+ "out": 5,+ "clipId": "c1"}}],"timings": {- "gateMs": 219.92966699998942,+ "gateMs": 475.8945420000091,"gateSkipped": false,- "ttftMs": 3313.9157499999856,- "modelMs": 3438.0066660000302,- "toolMs": 0.2047080000047572,- "totalMs": 3667.044790999993,- "judgeMs": 244.47749999997905,+ "ttftMs": 3297.279917000007,+ "modelMs": 3239.899292000002,+ "toolMs": 0.013250000003608875,+ "totalMs": 3722.5478339999972,+ "judgeMs": 244.66629200000898,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1761.965166000009,+ "roundMs": 1343.831416999994,"firstDeltaMs": null,"calls": [{- "name": "generate_image",- "execMs": 0.2047080000047572+ "name": "trim_clip",+ "execMs": 0.013250000003608875}]},{- "roundMs": 1676.0415000000212,- "firstDeltaMs": 1323.256625000009,+ "roundMs": 1896.067875000008,+ "firstDeltaMs": 1471.1177080000052,"calls": []}]}+ }+ ]+ },+ {+ "name": "transition-ask-styled",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 433,+ "p95": 433,+ "mean": 433,+ "n": 1+ },+ "ttftMs": {+ "p50": 8104,+ "p95": 8104,+ "mean": 8104,+ "n": 1+ },+ "totalMs": {+ "p50": 8252,+ "p95": 8252,+ "mean": 8252,+ "n": 1+ },+ "modelMs": {+ "p50": 7814,+ "p95": 7814,+ "mean": 7814,+ "n": 1},+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "transitions-and-fades","areas": ["preview","timeline","inspector",- "removal",- "elements","effects","animation",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",+ "transitions","top_bar","editor"],- "declaredTools": 103,+ "declaredTools": 73,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "generate_image",+ "name": "set_transition","round": 0,- "atMs": 2701.744250000018,- "execMs": 0.012959000014234334,+ "atMs": 6044.441207999989,+ "execMs": 0.06787500000791624,"args": {- "reference_asset_ids": [- "a-tw1"- ],- "prompt": "A high-quality recreation of the girl in the \"first snow of the year\" photo, now wearing a festive red Santa hat, maintaining the original composition and aesthetic.",- "aspect": "9:16"+ "style": "pushdown",+ "seconds": 0.5,+ "clipId": "c1"}}],"timings": {- "gateMs": 266.1987909999734,+ "gateMs": 433.28800000000047,"gateSkipped": false,- "ttftMs": 3858.036208000005,- "modelMs": 3931.0774989999773,- "toolMs": 0.012959000014234334,- "totalMs": 4203.301917000004,- "judgeMs": 224.05791699999827,+ "ttftMs": 8103.729582999993,+ "modelMs": 7813.888875000019,+ "toolMs": 0.06787500000791624,+ "totalMs": 8251.897332999986,+ "judgeMs": 508.81104099999357,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2431.2394159999967,+ "roundMs": 5607.726459000012,"firstDeltaMs": null,"calls": [{- "name": "generate_image",- "execMs": 0.012959000014234334+ "name": "set_transition",+ "execMs": 0.06787500000791624}]},{- "roundMs": 1499.8380829999805,- "firstDeltaMs": 1154.8791249999776,+ "roundMs": 2206.1624160000065,+ "firstDeltaMs": 2058.428791000013,"calls": []}]@@ -2088,44 +1805,44 @@]},{- "name": "audio-ask-single-generate",+ "name": "slide-in-direction-correct","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 224,- "p95": 248,- "mean": 236,- "n": 2+ "p50": 532,+ "p95": 532,+ "mean": 532,+ "n": 1},"ttftMs": {- "p50": 2787,- "p95": 3334,- "mean": 3061,- "n": 2+ "p50": 534,+ "p95": 534,+ "mean": 534,+ "n": 1},"totalMs": {- "p50": 3004,- "p95": 3585,- "mean": 3295,- "n": 2+ "p50": 534,+ "p95": 534,+ "mean": 534,+ "n": 1},"modelMs": {- "p50": 2750,- "p95": 3355,- "mean": 3052,- "n": 2+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 0,+ "max": 0}},"runs": [@@ -2137,60 +1854,181 @@"preview","timeline","inspector",- "subtitles",- "audio",+ "effects",+ "animation",+ "transitions",+ "editor"+ ],+ "declaredTools": 69,+ "routingMisses": [],+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "instant": "set_animation",+ "judgeNote": "failures owned 0.44",+ "trace": [+ {+ "name": "set_animation",+ "round": 0,+ "atMs": 533.5733749999927,+ "execMs": 0.06333299999823794,+ "args": {+ "clipId": "c2",+ "which": "in",+ "style": "slideright",+ "seconds": 0.47+ }+ }+ ],+ "timings": {+ "gateMs": 532.2724170000001,+ "gateSkipped": false,+ "ttftMs": 534.0615419999958,+ "modelMs": 0,+ "toolMs": 0.06333299999823794,+ "totalMs": 534.0719169999938,+ "judgeMs": 198.63141700001142,+ "retryMs": 0,+ "retries": 0,+ "rounds": []+ }+ }+ ]+ },+ {+ "name": "look-ask-single-tool",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 366,+ "p95": 366,+ "mean": 366,+ "n": 1+ },+ "ttftMs": {+ "p50": 367,+ "p95": 367,+ "mean": 367,+ "n": 1+ },+ "totalMs": {+ "p50": 367,+ "p95": 367,+ "mean": 367,+ "n": 1+ },+ "modelMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "rounds": {+ "mean": 0,+ "max": 0+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "simple",+ "skill": null,+ "areas": [+ "preview",+ "timeline",+ "inspector",+ "color",+ "effects","video_gen","scene",- "stock",- "library","side_panel",- "top_bar"+ "top_bar",+ "editor"],- "declaredTools": 106,+ "declaredTools": 93,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": "add_effect","judgeNote": null,"trace": [{- "name": "voiceover_generate",+ "name": "add_effect","round": 0,- "atMs": 1881.8186250000144,- "execMs": 0.007541999977547675,+ "atMs": 366.4008329999924,+ "execMs": 0.18120900000212714,"args": {- "script": "Welcome to the dog show."+ "effect": "vhs",+ "amount": 0.67,+ "start": 6,+ "end": 12.5}}],"timings": {- "gateMs": 223.6105830000015,+ "gateMs": 365.6525419999962,"gateSkipped": false,- "ttftMs": 3334.4220000000205,- "modelMs": 3355.0260420000122,- "toolMs": 0.007541999977547675,- "totalMs": 3584.7433330000204,- "judgeMs": 254.3882910000102,+ "ttftMs": 366.85558299999684,+ "modelMs": 0,+ "toolMs": 0.18120900000212714,+ "totalMs": 366.87566700000025,+ "judgeMs": 271.4133749999892,"retryMs": 0,"retries": 0,- "rounds": [- {- "roundMs": 1653.8843749999942,- "firstDeltaMs": null,- "calls": [- {- "name": "voiceover_generate",- "execMs": 0.007541999977547675- }- ]- },- {- "roundMs": 1701.141667000018,- "firstDeltaMs": 1451.2494170000136,- "calls": []- }- ]+ "rounds": []}+ }+ ]+ },+ {+ "name": "audio-effect-single-tool",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 302,+ "p95": 302,+ "mean": 302,+ "n": 1+ },+ "ttftMs": {+ "p50": 303,+ "p95": 303,+ "mean": 303,+ "n": 1+ },+ "totalMs": {+ "p50": 303,+ "p95": 303,+ "mean": 303,+ "n": 1+ },+ "modelMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 0,+ "max": 0+ }+ },+ "runs": [{"pass": true,"intent": "simple",@@ -2199,90 +2037,73 @@"preview","timeline","inspector",- "subtitles",+ "effects","audio",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar"+ "side_panel"],- "declaredTools": 106,+ "declaredTools": 74,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": "add_effect","judgeNote": null,"trace": [{- "name": "voiceover_generate",+ "name": "add_effect","round": 0,- "atMs": 1573.5448749999923,- "execMs": 0.00987500001792796,+ "atMs": 302.74691699999676,+ "execMs": 0.3786249999975553,"args": {- "script": "Welcome to the dog show."+ "effect": "telephone",+ "amount": 0.68,+ "start": 6,+ "end": 12.5}}],"timings": {- "gateMs": 248.26612499999464,+ "gateMs": 302.0796249999985,"gateSkipped": false,- "ttftMs": 2787.0850840000203,- "modelMs": 2749.9022919999843,- "toolMs": 0.00987500001792796,- "totalMs": 3004.278250000003,- "judgeMs": 221.06387500002165,+ "ttftMs": 303.43887499999255,+ "modelMs": 0,+ "toolMs": 0.3786249999975553,+ "totalMs": 303.4515839999949,+ "judgeMs": 281.96937499998603,"retryMs": 0,"retries": 0,- "rounds": [- {- "roundMs": 1322.1817919999885,- "firstDeltaMs": null,- "calls": [- {- "name": "voiceover_generate",- "execMs": 0.00987500001792796- }- ]- },- {- "roundMs": 1427.7204999999958,- "firstDeltaMs": 1212.0876250000147,- "calls": []- }- ]+ "rounds": []}}]},{- "name": "music-ask-single-generate",+ "name": "sound-dissolve-single-tool","bucket": "single-tool",- "passRate": 0.5,+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 268,- "p95": 268,- "mean": 268,+ "p50": 340,+ "p95": 340,+ "mean": 340,"n": 1},"ttftMs": {- "p50": 9835,- "p95": 9835,- "mean": 9835,+ "p50": 4533,+ "p95": 4533,+ "mean": 4533,"n": 1},"totalMs": {- "p50": 10184,- "p95": 10184,- "mean": 10184,+ "p50": 4686,+ "p95": 4686,+ "mean": 4686,"n": 1},"modelMs": {- "p50": 9912,- "p95": 9912,- "mean": 9912,+ "p50": 4336,+ "p95": 4336,+ "mean": 4336,"n": 1},"toolMs": {@@ -2298,179 +2119,66 @@},"runs": [{- "pass": false,- "intent": "complex",- "skill": "audio-and-subtitles",+ "pass": true,+ "intent": "simple",+ "skill": null,"areas": ["preview","timeline","inspector",- "subtitles",+ "effects",+ "transitions","audio",- "stock",- "library",- "side_panel"+ "editor"],- "declaredTools": 94,+ "declaredTools": 71,"routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [- "mutating tools called: add_clip"- ],+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "generate_music",+ "name": "set_transition","round": 1,- "atMs": 5008.586624999996,- "execMs": 0.17216600000392646,- "args": {- "instrumental": true,- "length": "clip",- "prompt": "Chill, relaxed lofi ambient acoustic background music, warm gentle guitar and soft synth pads, mellow and calming vibe",- "start": 0,- "add_to_timeline": true,- "volume": 0.35- }- },- {- "name": "get_state",- "round": 2,- "atMs": 19322.093041999993,- "execMs": 0.1537909999897238,- "args": {}- },- {- "name": "add_clip",- "round": 3,- "atMs": 28052.555291999975,- "execMs": 0.008332999976119027,+ "atMs": 3238.128500000006,+ "execMs": 0.3135829999955604,"args": {- "lane": 1,- "asset_id": "a-music1",- "start": 0+ "seconds": 0.3,+ "style": "audiocross",+ "clipId": "c1"}}],"timings": {- "gateMs": 246.73874999998952,+ "gateMs": 340.3528329999972,"gateSkipped": false,- "ttftMs": 41920.403416999994,- "modelMs": 41960.425292,- "toolMs": 0.33428999996976927,- "totalMs": 42219.28408299998,- "judgeMs": 554.3412499999977,+ "ttftMs": 4532.745125000001,+ "modelMs": 4335.873957000003,+ "toolMs": 0.3135829999955604,+ "totalMs": 4685.774916000009,+ "judgeMs": 657.7384170000005,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1830.7174170000071,+ "roundMs": 1343.5783749999973,"firstDeltaMs": null,"calls": []},{- "roundMs": 2924.800416000013,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_music",- "execMs": 0.17216600000392646- }- ]- },- {- "roundMs": 14310.745166999986,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.1537909999897238- }- ]- },- {- "roundMs": 8728.772792000003,+ "roundMs": 1546.9686660000007,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.008332999976119027+ "name": "set_transition",+ "execMs": 0.3135829999955604}]},{- "roundMs": 14165.38949999999,- "firstDeltaMs": 13866.961291999993,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "audio-and-subtitles",- "areas": [- "preview",- "timeline",- "inspector",- "subtitles",- "audio",- "stock",- "library",- "side_panel",- "top_bar"- ],- "declaredTools": 98,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "generate_music",- "round": 1,- "atMs": 7789.254709000001,- "execMs": 0.013916999974753708,- "args": {- "length": "clip",- "add_to_timeline": true,- "volume": 0.4,- "start": 0,- "prompt": "Chill lofi ambient background music, relaxed acoustic beats, mellow guitar and soft synth, warm summer beach vibe",- "instrumental": true- }- }- ],- "timings": {- "gateMs": 268.3392500000191,- "gateSkipped": false,- "ttftMs": 9835.427792000002,- "modelMs": 9911.988834999967,- "toolMs": 0.013916999974753708,- "totalMs": 10184.317209,- "judgeMs": 459.09216700002435,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4441.42420899999,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 3076.8457089999865,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_music",- "execMs": 0.013916999974753708- }- ]- },- {- "roundMs": 2393.718916999991,- "firstDeltaMs": 2044.9508330000099,+ "roundMs": 1445.3269160000054,+ "firstDeltaMs": 1292.6651660000061,"calls": []}]@@ -2479,40 +2187,40 @@]},{- "name": "trim-ask-single-tool",+ "name": "animation-overrides-transition","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 218,- "p95": 287,- "mean": 252,- "n": 2+ "p50": 506,+ "p95": 506,+ "mean": 506,+ "n": 1},"ttftMs": {- "p50": 2617,- "p95": 2766,- "mean": 2691,- "n": 2+ "p50": 5866,+ "p95": 5866,+ "mean": 5866,+ "n": 1},"totalMs": {- "p50": 2761,- "p95": 2916,- "mean": 2838,- "n": 2+ "p50": 6147,+ "p95": 6147,+ "mean": 6147,+ "n": 1},"modelMs": {- "p50": 2538,- "p95": 2627,- "mean": 2582,- "n": 2+ "p50": 5634,+ "p95": 5634,+ "mean": 5634,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -2528,108 +2236,67 @@"preview","timeline","inspector",- "library",+ "animation",+ "transitions","editor"],- "declaredTools": 72,+ "declaredTools": 68,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "remove_transition","round": 0,- "atMs": 1551.334583000018,- "execMs": 0.005499999999301508,+ "atMs": 2514.8120419999905,+ "execMs": 0.1587089999957243,"args": {- "clipId": "c1",- "in": 0,- "out": 5- }- }- ],- "timings": {- "gateMs": 217.80354200000875,- "gateSkipped": false,- "ttftMs": 2617.0373330000148,- "modelMs": 2538.0882920000004,- "toolMs": 0.005499999999301508,- "totalMs": 2761.014583000011,- "judgeMs": 221.55958299999475,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1329.2464169999876,- "firstDeltaMs": null,- "calls": [- {- "name": "trim_clip",- "execMs": 0.005499999999301508- }- ]- },- {- "roundMs": 1208.8418750000128,- "firstDeltaMs": 1064.9870419999934,- "calls": []+ "transitionId": "T-c1-c2"}- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "library",- "editor"- ],- "declaredTools": 72,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [+ },{- "name": "trim_clip",+ "name": "set_animation","round": 0,- "atMs": 1514.858332999982,- "execMs": 0.004165999998804182,+ "atMs": 2516.7521669999987,+ "execMs": 0.009333000009064563,"args": {- "out": 5,- "clipId": "c1"+ "seconds": 0.5,+ "which": "in",+ "style": "pop",+ "clipId": "c2"}}],"timings": {- "gateMs": 286.5764579999959,+ "gateMs": 505.9929159999883,"gateSkipped": false,- "ttftMs": 2765.8231249999953,- "modelMs": 2626.754917000013,- "toolMs": 0.004165999998804182,- "totalMs": 2915.7682909999858,- "judgeMs": 177.7972909999953,+ "ttftMs": 5865.710541999986,+ "modelMs": 5633.522166999988,+ "toolMs": 0.16804200000478886,+ "totalMs": 6147.075707999989,+ "judgeMs": 566.6127080000006,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1226.7500419999997,+ "roundMs": 2004.978959,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.004165999998804182+ "name": "remove_transition",+ "execMs": 0.1587089999957243+ },+ {+ "name": "set_animation",+ "execMs": 0.009333000009064563}]},{- "roundMs": 1400.0048750000133,- "firstDeltaMs": 1250.1533750000235,+ "roundMs": 3628.543207999988,+ "firstDeltaMs": 3347.3651669999963,"calls": []}]@@ -2638,336 +2305,311 @@]},{- "name": "transition-ask-styled",- "bucket": "single-tool",- "passRate": 1,+ "name": "filler-words-cut-with-editor",+ "bucket": "multi-tool",+ "passRate": 0,"budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 215,- "p95": 215,- "mean": 215,- "n": 2- },- "ttftMs": {- "p50": 2857,- "p95": 6292,- "mean": 4574,- "n": 2- },- "totalMs": {- "p50": 2966,- "p95": 6440,- "mean": 4703,- "n": 2- },- "modelMs": {- "p50": 2748,- "p95": 6221,- "mean": 4484,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2.5,- "max": 3- }- },+ "latency": null,"runs": [{- "pass": true,- "intent": "simple",- "skill": null,+ "pass": false,+ "intent": "complex",+ "skill": "watching-and-cutting","areas": ["preview","timeline","inspector",- "effects",- "animation","transitions",+ "subtitles",+ "audio",+ "library",+ "side_panel",+ "top_bar","editor"],- "declaredTools": 68,+ "declaredTools": 101,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],+ "roundModel": "gemini-3.8-flash",+ "notes": [+ "mutating tools called: remove_gap"+ ],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "find_filler","round": 0,- "atMs": 1766.6607920000097,- "execMs": 0.08154099999228492,+ "atMs": 3767.003042000011,+ "execMs": 0.0957909999997355,"args": {- "style": "pushdown",- "clipId": "c1",- "seconds": 0.5- }- }- ],- "timings": {- "gateMs": 215.1701250000042,- "gateSkipped": false,- "ttftMs": 2856.6242499999935,- "modelMs": 2747.8471250000002,- "toolMs": 0.08154099999228492,- "totalMs": 2966.304708999989,- "judgeMs": 262.8814999999886,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1549.3415410000016,- "firstDeltaMs": null,- "calls": [- {- "name": "set_transition",- "execMs": 0.08154099999228492- }- ]- },- {- "roundMs": 1198.5055839999986,- "firstDeltaMs": 1088.9722919999913,- "calls": []+ "track": 0}- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "transitions-and-fades",- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "animation",- "transitions",- "library",- "editor"- ],- "declaredTools": 75,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [+ },{- "name": "set_transition",+ "name": "detect_silence","round": 1,- "atMs": 3791.700832999981,- "execMs": 0.009999999980209395,+ "atMs": 6030.388542000015,+ "execMs": 0.024291999987326562,"args": {- "seconds": 0.5,- "clipId": "c1",- "style": "pushdown"+ "clip_id": "c1"}- }- ],- "timings": {- "gateMs": 215.35533300001407,- "gateSkipped": false,- "ttftMs": 6291.746541999979,- "modelMs": 6221.118248999963,- "toolMs": 0.009999999980209395,- "totalMs": 6440.362166999985,- "judgeMs": 518.4899999999907,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1501.8468329999887,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2071.82699999999,- "firstDeltaMs": null,- "calls": [+ },+ {+ "name": "watch_video",+ "round": 2,+ "atMs": 7921.313875000007,+ "execMs": 0.02404200000455603,+ "args": {+ "to": 6,+ "clip_id": "c1",+ "interval_seconds": 1,+ "from": 0+ }+ },+ {+ "name": "note_source",+ "round": 3,+ "atMs": 11197.174667,+ "execMs": 0.030583000014303252,+ "args": {+ "clip_id": "c1",+ "notes": [{- "name": "set_transition",- "execMs": 0.009999999980209395+ "from": 0,+ "text": "Beach footage with dogs playing by the water",+ "to": 6}]- },- {- "roundMs": 2647.444415999984,- "firstDeltaMs": 2499.012290999992,- "calls": []}- ]- }- }- ]- },- {- "name": "slide-in-direction-correct",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 260,- "p95": 263,- "mean": 261,- "n": 2- },- "ttftMs": {- "p50": 2700,- "p95": 4221,- "mean": 3460,- "n": 2- },- "totalMs": {- "p50": 2788,- "p95": 4408,- "mean": 3598,- "n": 2- },- "modelMs": {- "p50": 2526,- "p95": 4140,- "mean": 3333,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2.5,- "max": 3- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "animation",- "transitions"- ],- "declaredTools": 63,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [+ },{- "name": "set_animation",- "round": 1,- "atMs": 3143.269540999987,- "execMs": 0.07474999999976717,+ "name": "trim_clip",+ "round": 4,+ "atMs": 13853.136291999996,+ "execMs": 1.1199999999953434,"args": {- "which": "in",- "style": "slideright",- "clipId": "c2",- "seconds": 0.5+ "clipId": "c1",+ "in": 0.45+ }+ },+ {+ "name": "split_at",+ "round": 4,+ "atMs": 13855.264875000008,+ "execMs": 0.17829199999687262,+ "args": {+ "t": 4.1+ }+ },+ {+ "name": "split_at",+ "round": 4,+ "atMs": 13856.234000000011,+ "execMs": 0.02383300001383759,+ "args": {+ "t": 5.15+ }+ },+ {+ "name": "delete_item",+ "round": 5,+ "atMs": 17126.910042000018,+ "execMs": 0.11554200001410209,+ "args": {+ "kind": "clip",+ "id": "c1-2-3"+ }+ },+ {+ "name": "remove_gap",+ "round": 5,+ "atMs": 17128.52962500001,+ "execMs": 0.03500000000349246,+ "args": {+ "at": 4.5+ }+ },+ {+ "name": "get_state",+ "round": 6,+ "atMs": 18953.044083,+ "execMs": 0.1336670000164304,+ "args": {}+ },+ {+ "name": "refine_speech_cuts",+ "round": 7,+ "atMs": 20914.840666999997,+ "execMs": 0.06129099999088794,+ "args": {+ "pace": "natural",+ "clip_ids": [+ "c1-1",+ "c1-2-4"+ ]+ }+ },+ {+ "name": "align_to_audio",+ "round": 8,+ "atMs": 22686.507750000004,+ "execMs": 0.024167000025045127,+ "args": {+ "track": 0,+ "target": "captions"+ }+ },+ {+ "name": "update_cue",+ "round": 9,+ "atMs": 24849.964374999996,+ "execMs": 0.03858300001593307,+ "args": {+ "text": "So today we're",+ "id": "cue1"+ }+ },+ {+ "name": "update_cue",+ "round": 9,+ "atMs": 24850.235375000004,+ "execMs": 0.013750000012805685,+ "args": {+ "id": "cue3",+ "text": "And"}}],"timings": {- "gateMs": 263.201040999993,+ "gateMs": 481.8056660000002,"gateSkipped": false,- "ttftMs": 4220.703832999978,- "modelMs": 4139.535084000032,- "toolMs": 0.07474999999976717,- "totalMs": 4408.397790999996,- "judgeMs": 542.0186660000181,+ "ttftMs": 26443.069042000003,+ "modelMs": 26186.005707999953,+ "toolMs": 1.9188330000906717,+ "totalMs": 26703.704042000012,+ "judgeMs": 511.13716700000805,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1399.340125000017,+ "roundMs": 3280.327833000003,"firstDeltaMs": null,- "calls": []+ "calls": [+ {+ "name": "find_filler",+ "execMs": 0.0957909999997355+ }+ ]},{- "roundMs": 1476.0649590000103,+ "roundMs": 2260.191166999997,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.07474999999976717+ "name": "detect_silence",+ "execMs": 0.024291999987326562}]},{- "roundMs": 1264.1300000000047,- "firstDeltaMs": 1076.5062500000058,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "animation",- "transitions"- ],- "declaredTools": 64,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_animation",- "round": 0,- "atMs": 1625.3120000000054,- "execMs": 0.003915999986929819,- "args": {- "seconds": 0.5,- "clipId": "c2",- "style": "slideright",- "which": "in"- }- }- ],- "timings": {- "gateMs": 259.50512499999604,- "gateSkipped": false,- "ttftMs": 2700.167665999994,- "modelMs": 2526.1960839999956,- "toolMs": 0.003915999986929819,- "totalMs": 2787.675125000009,- "judgeMs": 325.18249999999534,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 1887.7098330000008,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "watch_video",+ "execMs": 0.02404200000455603+ }+ ]+ },{- "roundMs": 1364.479833999998,+ "roundMs": 3272.8237089999893,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.003915999986929819+ "name": "note_source",+ "execMs": 0.030583000014303252+ }+ ]+ },+ {+ "roundMs": 2653.259625000006,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "trim_clip",+ "execMs": 1.1199999999953434+ },+ {+ "name": "split_at",+ "execMs": 0.17829199999687262+ },+ {+ "name": "split_at",+ "execMs": 0.02383300001383759+ }+ ]+ },+ {+ "roundMs": 3267.363291000016,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "delete_item",+ "execMs": 0.11554200001410209+ },+ {+ "name": "remove_gap",+ "execMs": 0.03500000000349246+ }+ ]+ },+ {+ "roundMs": 1822.3442089999735,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "get_state",+ "execMs": 0.1336670000164304+ }+ ]+ },+ {+ "roundMs": 1959.774624999991,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "refine_speech_cuts",+ "execMs": 0.06129099999088794+ }+ ]+ },+ {+ "roundMs": 1769.8817500000005,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "align_to_audio",+ "execMs": 0.024167000025045127+ }+ ]+ },+ {+ "roundMs": 2160.8513329999987,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "update_cue",+ "execMs": 0.03858300001593307+ },+ {+ "name": "update_cue",+ "execMs": 0.013750000012805685}]},{- "roundMs": 1161.7162499999977,- "firstDeltaMs": 1074.3126660000125,+ "roundMs": 1851.4783329999773,+ "firstDeltaMs": 1591.298415999976,"calls": []}]@@ -2976,40 +2618,40 @@]},{- "name": "look-ask-single-tool",+ "name": "edit-request-still-uses-tools","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 240,- "p95": 290,- "mean": 265,- "n": 2+ "p50": 509,+ "p95": 509,+ "mean": 509,+ "n": 1},"ttftMs": {- "p50": 2938,- "p95": 2994,- "mean": 2966,- "n": 2+ "p50": 3190,+ "p95": 3190,+ "mean": 3190,+ "n": 1},"totalMs": {- "p50": 3166,- "p95": 3201,- "mean": 3184,- "n": 2+ "p50": 3277,+ "p95": 3277,+ "mean": 3277,+ "n": 1},"modelMs": {- "p50": 2910,- "p95": 2926,- "mean": 2918,- "n": 2+ "p50": 2762,+ "p95": 2762,+ "mean": 2762,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -3025,121 +2667,53 @@"preview","timeline","inspector",- "color",- "effects",- "video_gen",- "scene",+ "subtitles","library","side_panel",- "top_bar"+ "top_bar",+ "editor"],- "declaredTools": 95,+ "declaredTools": 97,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "add_effect",+ "name": "subtitles_generate","round": 0,- "atMs": 1861.2977920000267,- "execMs": 0.037958000000799075,+ "atMs": 1846.7781249999825,+ "execMs": 0.01699999999254942,"args": {- "effect": "vhs",- "start": 6,- "end": 12.5,- "amount": 0.5+ "track": 0}}],"timings": {- "gateMs": 289.69391599998926,+ "gateMs": 508.5648339999898,"gateSkipped": false,- "ttftMs": 2994.052250000008,- "modelMs": 2909.613624999998,- "toolMs": 0.037958000000799075,- "totalMs": 3200.9923750000016,- "judgeMs": 234.41870800001197,+ "ttftMs": 3189.7560829999857,+ "modelMs": 2761.642583000008,+ "toolMs": 0.01699999999254942,+ "totalMs": 3277.0722079999978,+ "judgeMs": 375.7734590000182,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1570.4394169999869,+ "roundMs": 1334.210208000004,"firstDeltaMs": null,"calls": [{- "name": "add_effect",- "execMs": 0.037958000000799075+ "name": "subtitles_generate",+ "execMs": 0.01699999999254942}]},{- "roundMs": 1339.1742080000113,- "firstDeltaMs": 1132.3141659999965,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "color",- "effects",- "video_gen",- "scene",- "library",- "side_panel",- "top_bar"- ],- "declaredTools": 95,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "add_effect",- "round": 0,- "atMs": 1858.4160839999968,- "execMs": 0.0029999999969732016,- "args": {- "end": 12.5,- "start": 6,- "amount": 0.5,- "effect": "vhs"- }- }- ],- "timings": {- "gateMs": 239.67337499998393,- "gateSkipped": false,- "ttftMs": 2937.9317089999968,- "modelMs": 2925.514500999998,- "toolMs": 0.0029999999969732016,- "totalMs": 3166.4014170000155,- "judgeMs": 265.204000000027,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1618.0305419999931,- "firstDeltaMs": null,- "calls": [- {- "name": "add_effect",- "execMs": 0.0029999999969732016- }- ]- },- {- "roundMs": 1307.4839590000047,- "firstDeltaMs": 1079.0800839999865,+ "roundMs": 1427.432375000004,+ "firstDeltaMs": 1340.4036250000063,"calls": []}]@@ -3148,158 +2722,151 @@]},{- "name": "audio-effect-single-tool",+ "name": "cartoon-from-audio-plans-scene","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 251,- "p95": 356,- "mean": 304,- "n": 2+ "p50": 437,+ "p95": 437,+ "mean": 437,+ "n": 1},"ttftMs": {- "p50": 2636,- "p95": 2729,- "mean": 2683,- "n": 2+ "p50": 27617,+ "p95": 27617,+ "mean": 27617,+ "n": 1},"totalMs": {- "p50": 2894,- "p95": 2894,- "mean": 2894,- "n": 2+ "p50": 28057,+ "p95": 28057,+ "mean": 28057,+ "n": 1},"modelMs": {- "p50": 2534,- "p95": 2640,- "mean": 2587,- "n": 2+ "p50": 27607,+ "p95": 27607,+ "mean": 27607,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 4,+ "max": 4}},"runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "scene-productions","areas": ["preview","timeline","inspector",+ "color",+ "removal",+ "elements","effects",+ "animation",+ "transitions",+ "subtitles","audio",- "side_panel"+ "image_gen",+ "video_gen",+ "scene",+ "stock",+ "library",+ "side_panel",+ "top_bar",+ "editor"],- "declaredTools": 73,+ "declaredTools": 128,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_clip_sound",+ "name": "listen_audio","round": 0,- "atMs": 1633.934459000011,- "execMs": 0.11125000001629815,+ "atMs": 3699.636708999984,+ "execMs": 0.06904200001736172,"args": {- "clipId": "c2",- "preset": "phone"+ "asset_id": "a-au1"+ }+ },+ {+ "name": "get_state",+ "round": 1,+ "atMs": 6497.891041999974,+ "execMs": 0.08341700001619756,+ "args": {}+ },+ {+ "name": "generate_scene",+ "round": 2,+ "atMs": 10014.750249999983,+ "execMs": 0.09912499997881241,+ "args": {+ "style": "smooth 2D cartoon animation, clean line art, vibrant flat colors, fluid traditional animation movement",+ "brief": "A story following the speaker's narration, rendered as a smooth 2D cartoon.",+ "from_audio_asset_id": "a-au1"}}],"timings": {- "gateMs": 250.75916599997436,+ "gateMs": 436.6330420000013,"gateSkipped": false,- "ttftMs": 2729.4537919999857,- "modelMs": 2640.348874999967,- "toolMs": 0.11125000001629815,- "totalMs": 2893.5675420000043,- "judgeMs": 215.6016670000099,+ "ttftMs": 27616.98225,+ "modelMs": 27606.593416000018,+ "toolMs": 0.2515840000123717,+ "totalMs": 28056.71645899999,+ "judgeMs": 782.0522090000159,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1381.5887089999742,+ "roundMs": 3259.5898330000055,"firstDeltaMs": null,"calls": [{- "name": "set_clip_sound",- "execMs": 0.11125000001629815+ "name": "listen_audio",+ "execMs": 0.06904200001736172}]},{- "roundMs": 1258.7601659999928,- "firstDeltaMs": 1094.755749999982,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "audio",- "side_panel"- ],- "declaredTools": 73,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_clip_sound",- "round": 0,- "atMs": 1617.3443329999864,- "execMs": 0.021084000007249415,- "args": {- "preset": "phone",- "clipId": "c2"- }- }- ],- "timings": {- "gateMs": 356.3399999999965,- "gateSkipped": false,- "ttftMs": 2636.099332999991,- "modelMs": 2534.3065840000054,- "toolMs": 0.021084000007249415,- "totalMs": 2893.990832999989,- "judgeMs": 229.26095799999894,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 2796.185375000001,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "get_state",+ "execMs": 0.08341700001619756+ }+ ]+ },{- "roundMs": 1259.4970420000027,+ "roundMs": 3511.175749999995,"firstDeltaMs": null,"calls": [{- "name": "set_clip_sound",- "execMs": 0.021084000007249415+ "name": "generate_scene",+ "execMs": 0.09912499997881241}]},{- "roundMs": 1274.8095420000027,- "firstDeltaMs": 1016.9855420000094,+ "roundMs": 18039.642458000017,+ "firstDeltaMs": 17600.159249999997,"calls": []}]@@ -3308,162 +2875,134 @@]},{- "name": "sound-dissolve-single-tool",+ "name": "cartoon-korean-audio-passes-language","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 225,- "p95": 296,- "mean": 261,- "n": 2+ "p50": 524,+ "p95": 524,+ "mean": 524,+ "n": 1},"ttftMs": {- "p50": 2623,- "p95": 2923,- "mean": 2773,- "n": 2+ "p50": 10025,+ "p95": 10025,+ "mean": 10025,+ "n": 1},"totalMs": {- "p50": 2867,- "p95": 3142,- "mean": 3005,- "n": 2+ "p50": 10196,+ "p95": 10196,+ "mean": 10196,+ "n": 1},"modelMs": {- "p50": 2640,- "p95": 2844,- "mean": 2742,- "n": 2+ "p50": 9663,+ "p95": 9663,+ "mean": 9663,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 3}},"runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "scene-productions","areas": ["preview","timeline","inspector",+ "removal",+ "elements","effects",+ "animation","transitions",+ "subtitles","audio",+ "image_gen",+ "video_gen",+ "scene",+ "stock",+ "library",+ "side_panel",+ "top_bar","editor"],- "declaredTools": 70,+ "declaredTools": 121,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "listen_audio","round": 0,- "atMs": 1747.8170419999806,- "execMs": 0.07725000000209548,+ "atMs": 4475.160665999982,+ "execMs": 0.034499999979743734,"args": {- "clipId": "c1",- "style": "audiocross",- "seconds": 0.3+ "asset_id": "a-au1"+ }+ },+ {+ "name": "generate_scene",+ "round": 1,+ "atMs": 7409.686082999979,+ "execMs": 0.46141599997645244,+ "args": {+ "from_audio_asset_id": "a-au1",+ "audio_language": "ko-KR",+ "brief": "A charming and smooth story animated to the Korean narration audio. Flat hand-drawn 2D cartoon animation, fluid motion, clean outlines, vibrant soft color palette.",+ "style": "smooth 2D cartoon animation, flat clean outlines, fluid character motion, vibrant and soft color palette"}}],"timings": {- "gateMs": 296.3960420000076,+ "gateMs": 523.9296669999894,"gateSkipped": false,- "ttftMs": 2922.7595419999852,- "modelMs": 2843.9709999999905,- "toolMs": 0.07725000000209548,- "totalMs": 3141.900667000009,- "judgeMs": 302.0467920000083,+ "ttftMs": 10025.095915999991,+ "modelMs": 9662.929749000032,+ "toolMs": 0.4959159999561962,+ "totalMs": 10196.3125,+ "judgeMs": 562.0726250000007,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1450.6633750000037,+ "roundMs": 3947.449708,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.07725000000209548+ "name": "listen_audio",+ "execMs": 0.034499999979743734}]},{- "roundMs": 1393.3076249999867,- "firstDeltaMs": 1174.4249589999963,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "transitions",- "audio",- "editor"- ],- "declaredTools": 70,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_transition",- "round": 0,- "atMs": 1571.1429999999818,- "execMs": 0.006624999979976565,- "args": {- "clipId": "c1",- "seconds": 0.3,- "style": "audiocross"- }- }- ],- "timings": {- "gateMs": 225.32741700002225,- "gateSkipped": false,- "ttftMs": 2623.195624999993,- "modelMs": 2639.6257500000065,- "toolMs": 0.006624999979976565,- "totalMs": 2867.185999999987,- "judgeMs": 279.27512499998556,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1344.689000000013,+ "roundMs": 2932.496833000012,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.006624999979976565+ "name": "generate_scene",+ "execMs": 0.46141599997645244}]},{- "roundMs": 1294.9367499999935,- "firstDeltaMs": 1051.1058339999872,+ "roundMs": 2782.9832080000197,+ "firstDeltaMs": 2612.181292000023,"calls": []}]@@ -3472,194 +3011,230 @@]},{- "name": "animation-overrides-transition",+ "name": "cartoon-style-reference-anchors-look","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 255,- "p95": 301,- "mean": 278,- "n": 2+ "p50": 306,+ "p95": 306,+ "mean": 306,+ "n": 1},"ttftMs": {- "p50": 4192,- "p95": 6068,- "mean": 5130,- "n": 2+ "p50": 8790,+ "p95": 8790,+ "mean": 8790,+ "n": 1},"totalMs": {- "p50": 4350,- "p95": 6422,- "mean": 5386,- "n": 2+ "p50": 9161,+ "p95": 9161,+ "mean": 9161,+ "n": 1},"modelMs": {- "p50": 4091,- "p95": 6119,- "mean": 5105,- "n": 2+ "p50": 8847,+ "p95": 8847,+ "mean": 8847,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2.5,+ "mean": 3,"max": 3}},"runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "scene-productions","areas": ["preview","timeline","inspector",+ "color",+ "removal",+ "elements",+ "effects","animation","transitions",+ "subtitles",+ "audio",+ "image_gen",+ "video_gen",+ "scene",+ "stock",+ "library",+ "side_panel",+ "top_bar","editor"],- "declaredTools": 67,+ "declaredTools": 128,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "remove_transition",+ "name": "listen_audio","round": 0,- "atMs": 1751.9912089999998,- "execMs": 0.09441600000718608,+ "atMs": 3228.2281660000153,+ "execMs": 0.015541000000666827,"args": {- "transitionId": "tr-c1"+ "asset_id": "a-au1"}},{- "name": "set_animation",+ "name": "generate_scene","round": 1,- "atMs": 3176.6528340000077,- "execMs": 0.0044169999891892076,+ "atMs": 6621.747415999998,+ "execMs": 0.22937500002444722,"args": {- "clipId": "c2",- "style": "pop",- "seconds": 0.5,- "which": "in"- }- }- ],+ "style": "Colorful 2D cartoon animation matching the illustration style of dog-park.jpg, clean linework, vibrant friendly colors, expressive characters",+ "from_audio_asset_id": "a-au1",+ "reference_asset_ids": [+ "a-i1"+ ],+ "brief": "A charming cartoon story following the narration, featuring playful animated dogs in a colorful park and beach setting with clean lines, expressive character animation, and vibrant colors."+ }+ }+ ],"timings": {- "gateMs": 254.50633299996844,- "gateSkipped": false,- "ttftMs": 4192.076667000016,- "modelMs": 4090.9922099999967,- "toolMs": 0.09883299999637529,- "totalMs": 4349.64837499999,- "judgeMs": 589.4487920000101,+ "gateMs": 305.56641699999454,+ "gateSkipped": true,+ "ttftMs": 8790.16737499999,+ "modelMs": 8846.805790999992,+ "toolMs": 0.24491600002511404,+ "totalMs": 9160.907750000013,+ "judgeMs": 519.856375000003,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1495.0590840000077,+ "roundMs": 2919.6338749999995,"firstDeltaMs": null,"calls": [{- "name": "remove_transition",- "execMs": 0.09441600000718608+ "name": "listen_audio",+ "execMs": 0.015541000000666827}]},{- "roundMs": 1423.5612919999985,+ "roundMs": 3390.7651250000054,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.0044169999891892076+ "name": "generate_scene",+ "execMs": 0.22937500002444722}]},{- "roundMs": 1172.3718339999905,- "firstDeltaMs": 1014.9619999999995,+ "roundMs": 2536.4067909999867,+ "firstDeltaMs": 2165.980958,"calls": []}]}+ }+ ]+ },+ {+ "name": "cartoon-question-stays-in-chat",+ "bucket": "chat",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 531,+ "p95": 531,+ "mean": 531,+ "n": 1+ },+ "ttftMs": {+ "p50": 2651,+ "p95": 2651,+ "mean": 2651,+ "n": 1+ },+ "totalMs": {+ "p50": 4200,+ "p95": 4200,+ "mean": 4200,+ "n": 1+ },+ "modelMs": {+ "p50": 3665,+ "p95": 3665,+ "mean": 3665,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 1,+ "max": 1+ }+ },+ "runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "scene-productions","areas": ["preview","timeline","inspector",+ "removal",+ "elements",+ "effects","animation","transitions",- "editor"+ "subtitles",+ "audio",+ "image_gen",+ "video_gen",+ "scene",+ "stock",+ "library",+ "side_panel",+ "top_bar"],- "declaredTools": 67,+ "declaredTools": 117,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],- "judgeNote": null,- "trace": [- {- "name": "remove_transition",- "round": 0,- "atMs": 2244.4309999999823,- "execMs": 0.0015410000341944396,- "args": {- "transitionId": "c1-transition-0"- }- },- {- "name": "set_animation",- "round": 0,- "atMs": 2244.4865839999984,- "execMs": 0.000792000035289675,- "args": {- "style": "pop",- "clipId": "c2",- "seconds": 0.2,- "which": "in"- }- }- ],+ "instant": null,+ "judgeNote": "concise factual 0.35; long 2.0",+ "trace": [],"timings": {- "gateMs": 301.0832919999957,+ "gateMs": 531.2116249999963,"gateSkipped": false,- "ttftMs": 6067.817124999943,- "modelMs": 6119.131208999956,- "toolMs": 0.0023330000694841146,- "totalMs": 6421.992083999969,- "judgeMs": 509.14308300003177,+ "ttftMs": 2650.5627919999824,+ "modelMs": 3664.653541000007,+ "toolMs": 0,+ "totalMs": 4199.933374999993,+ "judgeMs": 263.8937080000178,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1942.0425839999807,- "firstDeltaMs": null,- "calls": [- {- "name": "remove_transition",- "execMs": 0.0015410000341944396- },- {- "name": "set_animation",- "execMs": 0.000792000035289675- }- ]- },- {- "roundMs": 4177.088624999975,- "firstDeltaMs": 3822.990208000003,+ "roundMs": 3664.653541000007,+ "firstDeltaMs": 2116.5879159999895,"calls": []}]@@ -3668,51 +3243,51 @@]},{- "name": "filler-words-cut-with-editor",+ "name": "clip-mentions-redo-their-shots","bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 218,- "p95": 290,- "mean": 254,- "n": 2+ "p50": 289,+ "p95": 289,+ "mean": 289,+ "n": 1},"ttftMs": {- "p50": 32285,- "p95": 38799,- "mean": 35542,- "n": 2+ "p50": 13125,+ "p95": 13125,+ "mean": 13125,+ "n": 1},"totalMs": {- "p50": 32550,- "p95": 39155,- "mean": 35852,- "n": 2+ "p50": 13580,+ "p95": 13580,+ "mean": 13580,+ "n": 1},"modelMs": {- "p50": 32237,- "p95": 38907,- "mean": 35572,- "n": 2+ "p50": 13266,+ "p95": 13266,+ "mean": 13266,+ "n": 1},"toolMs": {"p50": 1,"p95": 1,"mean": 1,- "n": 2+ "n": 1},"rounds": {- "mean": 16,- "max": 17+ "mean": 6,+ "max": 6}},"runs": [{"pass": true,"intent": "complex",- "skill": "watching-and-cutting",+ "skill": "audio-and-subtitles","areas": ["preview","timeline",@@ -3720,2652 +3295,185 @@"transitions","subtitles","audio",+ "video_gen",+ "scene",+ "stock","library","side_panel",- "top_bar","editor"],- "declaredTools": 100,+ "declaredTools": 107,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],- "judgeNote": "upsell 0.54",+ "instant": null,+ "judgeNote": null,"trace": [{- "name": "find_filler",- "round": 1,- "atMs": 7303.833249999967,- "execMs": 0.2982499999925494,- "args": {- "clip_id": "c1",- "track": 0- }- },- {- "name": "detect_silence",- "round": 2,- "atMs": 10485.969624999969,- "execMs": 0.019415999995544553,+ "name": "listen_audio",+ "round": 0,+ "atMs": 2591.3057499999995,+ "execMs": 0.5308329999970738,"args": {"to": 6,- "clip_id": "c1",+ "clip_id": "au-n","from": 0}},{"name": "watch_video",- "round": 3,- "atMs": 12500.72104199999,- "execMs": 0.029291999991983175,+ "round": 0,+ "atMs": 2592.829792000004,+ "execMs": 0.025999999983469024,"args": {- "clip_id": "c1",- "from": 0,- "interval_seconds": 1,- "to": 6+ "clip_id": "sc-1"}},{"name": "note_source",- "round": 4,- "atMs": 14568.631832999992,- "execMs": 0.014416999998502433,+ "round": 1,+ "atMs": 5092.499958,+ "execMs": 0.021791999984998256,"args": {- "clip_id": "c1",+ "clip_id": "sc-1","notes": [{+ "to": 3,"from": 0,- "to": 6,- "text": "Speaker by the beach talking with dogs in view"+ "text": "Shot 1 take showing scene 1 visual"}]}},{- "name": "select",- "round": 5,- "atMs": 16454.25408299995,- "execMs": 0.024625000020023435,- "args": {- "kind": "clip",- "id": "c1"- }- },- {- "name": "trim_clip",- "round": 6,- "atMs": 18193.951166999992,- "execMs": 0.5514580000308342,- "args": {- "clipId": "c1",- "in": 0.45- }- },- {- "name": "split_at",- "round": 7,- "atMs": 19819.45170799998,- "execMs": 0.08112499996786937,- "args": {- "t": 4.1- }- },- {- "name": "trim_clip",- "round": 8,- "atMs": 21819.94316699996,- "execMs": 0.018207999994046986,- "args": {- "in": 5.6,- "clipId": "c1-2"- }- },- {- "name": "get_state",- "round": 9,- "atMs": 24824.277000000002,- "execMs": 0.14995900000212714,- "args": {}- },- {- "name": "refine_speech_cuts",- "round": 10,- "atMs": 26635.33041699999,- "execMs": 0.03062500001396984,- "args": {- "clip_ids": [- "c1-1",- "c1-2"- ],- "pace": "natural"- }- },- {- "name": "update_cue",- "round": 11,- "atMs": 28624.792749999964,- "execMs": 0.013500000000931323,+ "name": "watch_video",+ "round": 1,+ "atMs": 5092.804499999998,+ "execMs": 0.004249999998137355,"args": {- "end": 1.8,- "text": "So today we're",- "id": "cue1",- "start": 0+ "clip_id": "sc-2"}},{- "name": "align_to_audio",- "round": 12,- "atMs": 30336.36599999998,- "execMs": 0.006457999988924712,+ "name": "note_source",+ "round": 2,+ "atMs": 7075.75787500001,+ "execMs": 0.020499999984167516,"args": {- "track": 0,- "target": "captions"+ "clip_id": "sc-2",+ "notes": [+ {+ "text": "Shot 2 take showing scene 2 visual",+ "from": 0,+ "to": 3+ }+ ]}},{"name": "get_state",- "round": 13,- "atMs": 32870.604082999984,- "execMs": 0.09083299996564165,+ "round": 2,+ "atMs": 7076.017167000013,+ "execMs": 0.01737499999580905,"args": {}},{- "name": "delete_cue",- "round": 14,- "atMs": 34794.35333299998,- "execMs": 0.061999999976251274,+ "name": "regenerate_shot",+ "round": 3,+ "atMs": 9038.298708000017,+ "execMs": 0.02412499999627471,"args": {- "id": "cue3"+ "note": "Match narration for first segment with distinct visual composition",+ "n": 1}},{- "name": "listen_audio",- "round": 15,- "atMs": 36722.856582999986,- "execMs": 0.021999999997206032,+ "name": "regenerate_shot",+ "round": 4,+ "atMs": 11026.740292000002,+ "execMs": 0.03508400000282563,"args": {- "from": 3.5,- "to": 4.55,- "clip_id": "c1-1"+ "note": "Match narration for second segment with distinct visual composition from shot 1",+ "n": 2}}],"timings": {- "gateMs": 217.8545419999864,- "gateSkipped": false,- "ttftMs": 38799.28816699999,- "modelMs": 38907.04075100011,- "toolMs": 1.4121659999364056,- "totalMs": 39155.218957999954,- "judgeMs": 582.8360840000096,+ "gateMs": 289.43787500000326,+ "gateSkipped": true,+ "ttftMs": 13125.493792000023,+ "modelMs": 13266.17620799999,+ "toolMs": 0.6799589999427553,+ "totalMs": 13580.435958000016,+ "judgeMs": 560.3659170000174,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1595.4662499999977,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 5485.319834000024,- "firstDeltaMs": null,- "calls": [- {- "name": "find_filler",- "execMs": 0.2982499999925494- }- ]- },- {- "roundMs": 3179.2717909999774,+ "roundMs": 2296.8954169999924,"firstDeltaMs": null,"calls": [{- "name": "detect_silence",- "execMs": 0.019415999995544553- }- ]- },- {- "roundMs": 2012.2938750000321,- "firstDeltaMs": null,- "calls": [+ "name": "listen_audio",+ "execMs": 0.5308329999970738+ },{"name": "watch_video",- "execMs": 0.029291999991983175+ "execMs": 0.025999999983469024}]},{- "roundMs": 2065.308707999997,+ "roundMs": 2495.436708000023,"firstDeltaMs": null,"calls": [{"name": "note_source",- "execMs": 0.014416999998502433- }- ]- },- {- "roundMs": 1882.5462919999845,- "firstDeltaMs": null,- "calls": [+ "execMs": 0.021791999984998256+ },{- "name": "select",- "execMs": 0.024625000020023435+ "name": "watch_video",+ "execMs": 0.004249999998137355}]},{- "roundMs": 1738.1774159999914,+ "roundMs": 1980.9661669999768,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.5514580000308342- }- ]- },- {- "roundMs": 1623.5362920000334,- "firstDeltaMs": null,- "calls": [+ "name": "note_source",+ "execMs": 0.020499999984167516+ },{- "name": "split_at",- "execMs": 0.08112499996786937+ "name": "get_state",+ "execMs": 0.01737499999580905}]},{- "roundMs": 1999.857958000037,+ "roundMs": 1956.2055830000027,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.018207999994046986+ "name": "regenerate_shot",+ "execMs": 0.02412499999627471}]},{- "roundMs": 3003.715124999988,+ "roundMs": 1985.682124999992,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.14995900000212714+ "name": "regenerate_shot",+ "execMs": 0.03508400000282563}]},{- "roundMs": 1809.7702920000302,- "firstDeltaMs": null,- "calls": [- {- "name": "refine_speech_cuts",- "execMs": 0.03062500001396984- }- ]- },- {- "roundMs": 1988.8142089999747,- "firstDeltaMs": null,- "calls": [- {- "name": "update_cue",- "execMs": 0.013500000000931323- }- ]- },- {- "roundMs": 1710.788750000007,- "firstDeltaMs": null,- "calls": [- {- "name": "align_to_audio",- "execMs": 0.006457999988924712- }- ]- },- {- "roundMs": 2533.3838749999995,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.09083299996564165- }- ]- },- {- "roundMs": 1922.700708999997,- "firstDeltaMs": null,- "calls": [- {- "name": "delete_cue",- "execMs": 0.061999999976251274- }- ]- },- {- "roundMs": 1925.6313749999972,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.021999999997206032- }- ]- },- {- "roundMs": 2430.4580000000424,- "firstDeltaMs": 2075.1564160000416,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "watching-and-cutting",- "areas": [- "preview",- "timeline",- "inspector",- "transitions",- "subtitles",- "audio",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 100,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "find_filler",- "round": 1,- "atMs": 4396.68987500004,- "execMs": 0.059084000007715076,- "args": {- "track": 0,- "clip_id": "c1"- }- },- {- "name": "detect_silence",- "round": 2,- "atMs": 6149.900499999989,- "execMs": 0.040542000031564385,- "args": {- "clip_id": "c1"- }- },- {- "name": "watch_video",- "round": 3,- "atMs": 8156.194791000045,- "execMs": 0.017125000013038516,- "args": {- "clip_id": "c1",- "to": 6,- "interval_seconds": 1,- "from": 0- }- },- {- "name": "note_source",- "round": 4,- "atMs": 10120.79916600004,- "execMs": 0.017749999999068677,- "args": {- "asset_id": "a-v1",- "notes": [- {- "from": 0,- "text": "Woman at beach talking to camera with dogs in background",- "to": 6- }- ]- }- },- {- "name": "split_at",- "round": 5,- "atMs": 11832.032666000014,- "execMs": 0.05033300002105534,- "args": {- "t": 4.55- }- },- {- "name": "split_at",- "round": 6,- "atMs": 13592.981583000044,- "execMs": 0.08833399997092783,- "args": {- "t": 5.6- }- },- {- "name": "trim_clip",- "round": 7,- "atMs": 15339.422750000027,- "execMs": 0.055333999975118786,- "args": {- "clipId": "c1-1",- "in": 0.45- }- },- {- "name": "delete_item",- "round": 8,- "atMs": 18217.488500000036,- "execMs": 0.022624999983236194,- "args": {- "kind": "clip",- "id": "c1-2-3"- }- },- {- "name": "get_state",- "round": 9,- "atMs": 19943.600416,- "execMs": 0.0782500000204891,- "args": {}- },- {- "name": "refine_speech_cuts",- "round": 10,- "atMs": 21950.37174999999,- "execMs": 0.04920799995306879,- "args": {- "pace": "natural",- "clip_ids": [- "c1-1",- "c1-2-4"- ]- }- },- {- "name": "update_cue",- "round": 11,- "atMs": 23760.78470800002,- "execMs": 0.024000000033993274,- "args": {- "end": 1.35,- "start": 0,- "id": "cue1",- "text": "so today we're"- }- },- {- "name": "get_state",- "round": 12,- "atMs": 28713.041333,- "execMs": 0.0811659999890253,- "args": {}- },- {- "name": "update_cue",- "round": 13,- "atMs": 30496.30125000002,- "execMs": 0.017583000008016825,- "args": {- "end": 4.1,- "start": 4.1,- "text": "and",- "id": "cue3"- }- }- ],- "timings": {- "gateMs": 289.78583300003083,- "gateSkipped": false,- "ttftMs": 32285.455958000035,- "modelMs": 32237.16462699999,- "toolMs": 0.6013340000063181,- "totalMs": 32549.717833000002,- "judgeMs": 500.9048329999787,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1260.811875000014,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2842.517625000037,- "firstDeltaMs": null,- "calls": [- {- "name": "find_filler",- "execMs": 0.059084000007715076- }- ]- },- {- "roundMs": 1751.425709000032,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.040542000031564385- }- ]- },- {- "roundMs": 2004.4235419999459,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.017125000013038516- }- ]- },- {- "roundMs": 1963.215749999974,- "firstDeltaMs": null,- "calls": [- {- "name": "note_source",- "execMs": 0.017749999999068677- }- ]- },- {- "roundMs": 1710.0438339999528,- "firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.05033300002105534- }- ]- },- {- "roundMs": 1759.844959000009,- "firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.08833399997092783- }- ]- },- {- "roundMs": 1744.530040999991,- "firstDeltaMs": null,- "calls": [- {- "name": "trim_clip",- "execMs": 0.055333999975118786- }- ]- },- {- "roundMs": 2876.2964999999967,- "firstDeltaMs": null,- "calls": [- {- "name": "delete_item",- "execMs": 0.022624999983236194- }- ]- },- {- "roundMs": 1725.439167000004,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.0782500000204891- }- ]- },- {- "roundMs": 2004.9705840000533,- "firstDeltaMs": null,- "calls": [- {- "name": "refine_speech_cuts",- "execMs": 0.04920799995306879- }- ]- },- {- "roundMs": 1808.8036250000005,- "firstDeltaMs": null,- "calls": [- {- "name": "update_cue",- "execMs": 0.024000000033993274- }- ]- },- {- "roundMs": 4950.779749999987,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.0811659999890253- }- ]- },- {- "roundMs": 1781.9015409999993,- "firstDeltaMs": null,- "calls": [- {- "name": "update_cue",- "execMs": 0.017583000008016825- }- ]- },- {- "roundMs": 2052.160124999995,- "firstDeltaMs": 1788.3024160000496,- "calls": []- }- ]- }- }- ]- },- {- "name": "edit-request-still-uses-tools",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 269,- "p95": 291,- "mean": 280,- "n": 2- },- "ttftMs": {- "p50": 2915,- "p95": 5602,- "mean": 4258,- "n": 2- },- "totalMs": {- "p50": 2930,- "p95": 5753,- "mean": 4342,- "n": 2- },- "modelMs": {- "p50": 2637,- "p95": 5480,- "mean": 4059,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "subtitles",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 96,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "subtitles_generate",- "round": 0,- "atMs": 4294.591166999948,- "execMs": 0.00483300001360476,- "args": {- "track": 0- }- }- ],- "timings": {- "gateMs": 269.41512499999953,- "gateSkipped": false,- "ttftMs": 5601.570207999961,- "modelMs": 5480.3979999999865,- "toolMs": 0.00483300001360476,- "totalMs": 5753.372499999998,- "judgeMs": 520.054833000002,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4022.840124999988,- "firstDeltaMs": null,- "calls": [- {- "name": "subtitles_generate",- "execMs": 0.00483300001360476- }- ]- },- {- "roundMs": 1457.5578749999986,- "firstDeltaMs": 1306.034290999989,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "subtitles",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 96,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "subtitles_generate",- "round": 0,- "atMs": 1606.8932919999934,- "execMs": 0.006709000037517399,- "args": {- "track": 0- }- }- ],- "timings": {- "gateMs": 290.6009159999667,- "gateSkipped": false,- "ttftMs": 2914.5904579999624,- "modelMs": 2636.953916999977,- "toolMs": 0.006709000037517399,- "totalMs": 2929.628708000004,- "judgeMs": 228.50675000000047,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1314.8173749999842,- "firstDeltaMs": null,- "calls": [- {- "name": "subtitles_generate",- "execMs": 0.006709000037517399- }- ]- },- {- "roundMs": 1322.136541999993,- "firstDeltaMs": 1307.2522920000483,- "calls": []- }- ]- }- }- ]- },- {- "name": "cartoon-from-audio-plans-scene",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 275,- "p95": 470,- "mean": 373,- "n": 2- },- "ttftMs": {- "p50": 7286,- "p95": 23363,- "mean": 15325,- "n": 2- },- "totalMs": {- "p50": 7461,- "p95": 23869,- "mean": 15665,- "n": 2- },- "modelMs": {- "p50": 7051,- "p95": 23588,- "mean": 15319,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 4,- "max": 4- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "color",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 127,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 1,- "atMs": 4857.906708000053,- "execMs": 0.07583400001749396,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 2,- "atMs": 7902.72316600004,- "execMs": 0.08183300000382587,- "args": {- "style": "Smooth 2D cartoon animation, hand-drawn vector aesthetic, clean lines, vibrant palette, fluid character animation",- "brief": "A smooth, fluid 2D hand-drawn animated cartoon illustrating the narration: clean bold outlines, vibrant flat colors, rounded friendly character shapes, continuous animated movement.",- "from_audio_asset_id": "a-au1"- }- }- ],- "timings": {- "gateMs": 274.9295830000192,- "gateSkipped": false,- "ttftMs": 23363.34275000001,- "modelMs": 23587.79954300006,- "toolMs": 0.15766700002131984,- "totalMs": 23869.057291000034,- "judgeMs": 579.0723330000183,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1727.5787920000148,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2850.7872920000227,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.07583400001749396- }- ]- },- {- "roundMs": 3043.916917000024,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.08183300000382587- }- ]- },- {- "roundMs": 15965.516541999998,- "firstDeltaMs": 15459.97320800001,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "color",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 127,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 1,- "atMs": 2976.7070419999654,- "execMs": 0.0313750000204891,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 2,- "atMs": 5711.230125000002,- "execMs": 0.005917000002227724,- "args": {- "from_audio_asset_id": "a-au1",- "brief": "A charming story following the narration, animated as smooth 2D cartoon animation with clean ink line art, soft vibrant colors, fluid hand-drawn motion, and consistent character design.",- "style": "Smooth 2D cartoon animation, hand-drawn vector art style, clean linework, vibrant flat colors, fluid motion"- }- }- ],- "timings": {- "gateMs": 470.4172920000274,- "gateSkipped": false,- "ttftMs": 7285.673084000009,- "modelMs": 7050.987667999987,- "toolMs": 0.03729200002271682,- "totalMs": 7460.546999999962,- "judgeMs": 555.2023749999935,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 66.9012090000324,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2504.117499999993,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.0313750000204891- }- ]- },- {- "roundMs": 2732.1188749999856,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.005917000002227724- }- ]- },- {- "roundMs": 1747.850083999976,- "firstDeltaMs": 1573.3913339999854,- "calls": []- }- ]- }- }- ]- },- {- "name": "cartoon-korean-audio-passes-language",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 319,- "p95": 320,- "mean": 319,- "n": 2- },- "ttftMs": {- "p50": 11762,- "p95": 12573,- "mean": 12168,- "n": 2- },- "totalMs": {- "p50": 12041,- "p95": 12908,- "mean": 12474,- "n": 2- },- "modelMs": {- "p50": 11714,- "p95": 12580,- "mean": 12147,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 4,- "max": 4- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 120,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 1,- "atMs": 4272.4063750000205,- "execMs": 0.009041999990586191,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 2,- "atMs": 7042.81270900002,- "execMs": 0.3398750000051223,- "args": {- "audio_language": "ko-KR",- "from_audio_asset_id": "a-au1",- "style": "smooth 2D cartoon animation, clean line art, vibrant pastel colors, fluid motion",- "brief": "A smooth 2D cartoon animation visualizing the Korean narration about a dog's adventure in a park and beach. Clean lines, charming rounded 2D animated characters, soft vibrant colors, fluid cartoon motion."- }- }- ],- "timings": {- "gateMs": 319.008833000029,- "gateSkipped": false,- "ttftMs": 11761.898417000019,- "modelMs": 11713.877665999986,- "toolMs": 0.34891699999570847,- "totalMs": 12041.062666999991,- "judgeMs": 539.3162920000032,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1689.5204999999842,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2259.5719999999856,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.009041999990586191- }- ]- },- {- "roundMs": 2769.332875000022,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.3398750000051223- }- ]- },- {- "roundMs": 4995.452290999994,- "firstDeltaMs": 4716.576457999996,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 120,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 1,- "atMs": 6614.068084000028,- "execMs": 0.019958999997470528,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 2,- "atMs": 8982.310875000025,- "execMs": 0.013042000005953014,- "args": {- "style": "smooth 2D cartoon animation, clean line art, vibrant colors, fluid character motion",- "from_audio_asset_id": "a-au1",- "brief": "A smooth 2D cartoon animation depicting the story narrated in the audio, featuring expressive characters, clean linework, vibrant colors, and fluid motion.",- "audio_language": "ko-KR"- }- }- ],- "timings": {- "gateMs": 319.92545899999095,- "gateSkipped": false,- "ttftMs": 12573.294542000047,- "modelMs": 12580.106417999952,- "toolMs": 0.03300100000342354,- "totalMs": 12907.581750000012,- "judgeMs": 569.1341249999823,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1661.9575839999598,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 4627.9814590000315,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.019958999997470528- }- ]- },- {- "roundMs": 2366.469333999965,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.013042000005953014- }- ]- },- {- "roundMs": 3923.698040999996,- "firstDeltaMs": 3589.924874999968,- "calls": []- }- ]- }- }- ]- },- {- "name": "cartoon-style-reference-anchors-look",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 252,- "p95": 260,- "mean": 256,- "n": 2- },- "ttftMs": {- "p50": 5897,- "p95": 8931,- "mean": 7414,- "n": 2- },- "totalMs": {- "p50": 6194,- "p95": 9198,- "mean": 7696,- "n": 2- },- "modelMs": {- "p50": 5928,- "p95": 8937,- "mean": 7432,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 3,- "max": 3- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "color",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 127,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 0,- "atMs": 3098.703125,- "execMs": 0.04220800002804026,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 1,- "atMs": 6774.4283339999965,- "execMs": 0.1091249999590218,- "args": {- "style": "flat hand-drawn 2D cartoon animation, colorful storybook illustration style, clean vibrant outlines, lively fluid motion",- "from_audio_asset_id": "a-au1",- "reference_asset_ids": [- "a-i1"- ]- }- }- ],- "timings": {- "gateMs": 251.5820839999942,- "gateSkipped": true,- "ttftMs": 8930.976375000027,- "modelMs": 8936.575916999951,- "toolMs": 0.15133299998706207,- "totalMs": 9198.011667000013,- "judgeMs": 587.8315410000505,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2842.6163329999545,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.04220800002804026- }- ]- },- {- "roundMs": 3672.175458999991,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.1091249999590218- }- ]- },- {- "roundMs": 2421.7841250000056,- "firstDeltaMs": 2154.885249999992,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "color",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 127,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 0,- "atMs": 1975.7431670000078,- "execMs": 0.006290999997872859,- "args": {- "asset_id": "a-au1"- }- },- {- "name": "generate_scene",- "round": 1,- "atMs": 4181.843500000017,- "execMs": 0.011375000001862645,- "args": {- "style": "vibrant 2D cartoon animation style matching the attached illustration",- "reference_asset_ids": [- "a-i1"- ],- "from_audio_asset_id": "a-au1"- }- }- ],- "timings": {- "gateMs": 260.4637910000165,- "gateSkipped": true,- "ttftMs": 5897.35820800002,- "modelMs": 5928.399665999983,- "toolMs": 0.017665999999735504,- "totalMs": 6194.27816700004,- "judgeMs": 574.2261669999571,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1712.2283329999773,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.006290999997872859- }- ]- },- {- "roundMs": 2205.2085829999996,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_scene",- "execMs": 0.011375000001862645- }- ]- },- {- "roundMs": 2010.962750000006,- "firstDeltaMs": 1714.131457999989,- "calls": []- }- ]- }- }- ]- },- {- "name": "cartoon-question-stays-in-chat",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 210,- "p95": 246,- "mean": 228,- "n": 2- },- "ttftMs": {- "p50": 3343,- "p95": 6764,- "mean": 5053,- "n": 2- },- "totalMs": {- "p50": 4215,- "p95": 8147,- "mean": 6181,- "n": 2- },- "modelMs": {- "p50": 4001,- "p95": 7895,- "mean": 5948,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 120,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": "concise factual 0.46; long 2.0",- "trace": [],- "timings": {- "gateMs": 246.42304099997273,- "gateSkipped": false,- "ttftMs": 6763.740624999977,- "modelMs": 7895.323125000054,- "toolMs": 0,- "totalMs": 8147.037375000014,- "judgeMs": 486.8101669999887,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4289.968167000043,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 3605.354958000011,- "firstDeltaMs": 2222.187957999995,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "scene-productions",- "areas": [- "preview",- "timeline",- "inspector",- "removal",- "elements",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "image_gen",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 120,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 210.49154099996667,- "gateSkipped": false,- "ttftMs": 3342.778667000006,- "modelMs": 4000.9623339999234,- "toolMs": 0,- "totalMs": 4215.245208000008,- "judgeMs": 598.4915000000037,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1541.7603749999544,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2459.201958999969,- "firstDeltaMs": 1587.3116670000018,- "calls": []- }- ]- }- }- ]- },- {- "name": "clip-mentions-redo-their-shots",- "bucket": "multi-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 245,- "p95": 256,- "mean": 251,- "n": 2- },- "ttftMs": {- "p50": 19034,- "p95": 26135,- "mean": 22584,- "n": 2- },- "totalMs": {- "p50": 19271,- "p95": 26690,- "mean": 22980,- "n": 2- },- "modelMs": {- "p50": 19020,- "p95": 26415,- "mean": 22717,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 7.5,- "max": 10- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "audio-and-subtitles",- "areas": [- "preview",- "timeline",- "inspector",- "transitions",- "subtitles",- "audio",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "editor"- ],- "declaredTools": 106,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 0,- "atMs": 2614.5971659999923,- "execMs": 0.10375000000931323,- "args": {- "to": 6,- "clip_id": "au-n",- "from": 0- }- },- {- "name": "watch_video",- "round": 0,- "atMs": 2614.805083000043,- "execMs": 0.002582999994046986,- "args": {- "clip_id": "sc-1"- }- },- {- "name": "watch_video",- "round": 0,- "atMs": 2614.8421250000247,- "execMs": 0.0005830000154674053,- "args": {- "clip_id": "sc-2"- }- },- {- "name": "note_source",- "round": 1,- "atMs": 5099.87737500004,- "execMs": 0.0039160000160336494,- "args": {- "clip_id": "sc-1",- "notes": [- {- "text": "Shot 1 of generated scene: dog running in park/field",- "to": 3,- "from": 0- }- ]- }- },- {- "name": "note_source",- "round": 1,- "atMs": 5099.935500000021,- "execMs": 0.0006249999860301614,- "args": {- "clip_id": "sc-2",- "notes": [- {- "to": 3,- "text": "Shot 2 of generated scene: very similar angle of dog running in park/field",- "from": 0- }- ]- }- },- {- "name": "get_state",- "round": 2,- "atMs": 7577.166541000013,- "execMs": 0.008750000037252903,- "args": {}- },- {- "name": "regenerate_shot",- "round": 3,- "atMs": 10322.591208000027,- "execMs": 0.009959000046364963,- "args": {- "note": "Close-up reaction shot that matches the narration",- "n": 2- }- }- ],- "timings": {- "gateMs": 245.41316700004973,- "gateSkipped": true,- "ttftMs": 19034.05487500003,- "modelMs": 19020.037291000015,- "toolMs": 0.1301660001045093,- "totalMs": 19271.390000000014,- "judgeMs": 577.3008750000154,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2368.2152079999796,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.10375000000931323- },- {- "name": "watch_video",- "execMs": 0.002582999994046986- },- {- "name": "watch_video",- "execMs": 0.0005830000154674053- }- ]- },- {- "roundMs": 2484.524792000011,- "firstDeltaMs": null,- "calls": [- {- "name": "note_source",- "execMs": 0.0039160000160336494- },- {- "name": "note_source",- "execMs": 0.0006249999860301614- }- ]- },- {- "roundMs": 2476.674291000003,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.008750000037252903- }- ]- },- {- "roundMs": 2743.2271670000046,- "firstDeltaMs": null,- "calls": [- {- "name": "regenerate_shot",- "execMs": 0.009959000046364963- }- ]- },- {- "roundMs": 8947.395833000017,- "firstDeltaMs": 8710.480333000014,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "audio-and-subtitles",- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "transitions",- "subtitles",- "audio",- "video_gen",- "scene",- "stock",- "library",- "side_panel",- "editor"- ],- "declaredTools": 107,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "listen_audio",- "round": 0,- "atMs": 7032.2287079999805,- "execMs": 0.01087499997811392,- "args": {- "from": 0,- "asset_id": "a-au1",- "to": 6- }- },- {- "name": "watch_video",- "round": 1,- "atMs": 9419.263541999971,- "execMs": 0.010249999992083758,- "args": {- "clip_id": "sc-1"- }- },- {- "name": "note_source",- "round": 2,- "atMs": 12498.851542000019,- "execMs": 0.016582999960519373,- "args": {- "clip_id": "sc-1",- "notes": [- {- "from": 0,- "to": 3,- "text": "Close-up shot of a golden retriever puppy looking at the camera on grass"- }- ]- }- },- {- "name": "watch_video",- "round": 3,- "atMs": 14469.211249999993,- "execMs": 0.016583000018727034,- "args": {- "clip_id": "sc-2"- }- },- {- "name": "note_source",- "round": 4,- "atMs": 16545.08166700002,- "execMs": 0.01554200000828132,- "args": {- "notes": [- {- "to": 3,- "text": "Close-up shot of the same golden retriever puppy sitting in the grass looking at the camera",- "from": 0- }- ],- "clip_id": "sc-2"- }- },- {- "name": "get_state",- "round": 5,- "atMs": 18451.339666999993,- "execMs": 0.0050830000545829535,- "args": {}- },- {- "name": "detect_silence",- "round": 6,- "atMs": 20426.178917000012,- "execMs": 0.02245799999218434,- "args": {- "asset_id": "a-au1",- "from": 0,- "to": 9- }- },- {- "name": "regenerate_shot",- "round": 7,- "atMs": 22340.80920800002,- "execMs": 0.028541000036057085,- "args": {- "note": "Golden retriever puppy running through the park playing fetch",- "n": 1- }- },- {- "name": "regenerate_shot",- "round": 8,- "atMs": 24551.809917000006,- "execMs": 0.025666999979875982,- "args": {- "note": "Golden retriever puppy running along the beach splashing in the shallow water",- "n": 2- }- }- ],- "timings": {- "gateMs": 255.9873329999973,- "gateSkipped": true,- "ttftMs": 26134.942541999975,- "modelMs": 26414.743376999977,- "toolMs": 0.15158200002042577,- "totalMs": 26689.50004199997,- "judgeMs": 539.7487500000279,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 6773.223625000042,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.01087499997811392- }- ]- },- {- "roundMs": 2385.6892089999747,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.010249999992083758- }- ]- },- {- "roundMs": 3077.8324169999687,- "firstDeltaMs": null,- "calls": [- {- "name": "note_source",- "execMs": 0.016582999960519373- }- ]- },- {- "roundMs": 1968.5407499999856,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.016583000018727034- }- ]- },- {- "roundMs": 2073.7079579999554,- "firstDeltaMs": null,- "calls": [- {- "name": "note_source",- "execMs": 0.01554200000828132- }- ]- },- {- "roundMs": 1904.663125000021,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.0050830000545829535- }- ]- },- {- "roundMs": 1973.7992920000106,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.02245799999218434- }- ]- },- {- "roundMs": 1912.4533330000122,- "firstDeltaMs": null,- "calls": [- {- "name": "regenerate_shot",- "execMs": 0.028541000036057085- }- ]- },- {- "roundMs": 2208.867084000027,- "firstDeltaMs": null,- "calls": [- {- "name": "regenerate_shot",- "execMs": 0.025666999979875982- }- ]- },- {- "roundMs": 2135.96658399998,- "firstDeltaMs": 1581.7539169999654,- "calls": []- }- ]- }- }- ]- },- {- "name": "parked-debris-cleanup-after-retime",- "bucket": "multi-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 228,- "p95": 264,- "mean": 246,- "n": 2- },- "ttftMs": {- "p50": 7665,- "p95": 7873,- "mean": 7769,- "n": 2- },- "totalMs": {- "p50": 8215,- "p95": 8422,- "mean": 8318,- "n": 2- },- "modelMs": {- "p50": 7948,- "p95": 8190,- "mean": 8069,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 3,- "max": 3- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "library",- "top_bar",- "editor"- ],- "declaredTools": 77,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_speed",- "round": 0,- "atMs": 4707.735999999975,- "execMs": 0.041333000001031905,- "args": {- "clipId": "c2",- "speed": 2- }- },- {- "name": "remove_transition",- "round": 1,- "atMs": 6412.277999999991,- "execMs": 0.0020840000361204147,- "args": {- "transitionId": "tr-2"- }- }- ],- "timings": {- "gateMs": 264.01820799999405,- "gateSkipped": false,- "ttftMs": 7664.744999999995,- "modelMs": 7948.246749999991,- "toolMs": 0.04341700003715232,- "totalMs": 8215.141708999989,- "judgeMs": 641.9884580000071,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4441.949957999983,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed",- "execMs": 0.041333000001031905- }- ]- },- {- "roundMs": 1703.8798750000424,- "firstDeltaMs": null,- "calls": [- {- "name": "remove_transition",- "execMs": 0.0020840000361204147- }- ]- },- {- "roundMs": 1802.416916999966,- "firstDeltaMs": 1252.0935839999584,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "top_bar",- "editor"- ],- "declaredTools": 70,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_speed",- "round": 0,- "atMs": 1598.6035830000183,- "execMs": 0.0044169999891892076,- "args": {- "speed": 2,- "clipId": "c2"- }- },- {- "name": "remove_transition",- "round": 1,- "atMs": 3753.244958000025,- "execMs": 0.00266700005158782,- "args": {- "transitionId": "tr-2"- }- }- ],- "timings": {- "gateMs": 227.9594999999972,- "gateSkipped": false,- "ttftMs": 7873.432917000027,- "modelMs": 8190.208416000009,- "toolMs": 0.007084000040777028,- "totalMs": 8421.521833000006,- "judgeMs": 540.487958999991,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1369.1699999999837,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed",- "execMs": 0.0044169999891892076- }- ]- },- {- "roundMs": 2153.7447500000126,- "firstDeltaMs": null,- "calls": [- {- "name": "remove_transition",- "execMs": 0.00266700005158782- }- ]- },- {- "roundMs": 4667.293666000012,- "firstDeltaMs": 4119.487374999968,- "calls": []- }- ]- }- }- ]- },- {- "name": "speed-ramp-from-description",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 259,- "p95": 283,- "mean": 271,- "n": 2- },- "ttftMs": {- "p50": 4348,- "p95": 9251,- "mean": 6800,- "n": 2- },- "totalMs": {- "p50": 4502,- "p95": 9478,- "mean": 6990,- "n": 2- },- "modelMs": {- "p50": 4238,- "p95": 9186,- "mean": 6712,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 4.5,- "max": 6- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "transitions",- "library"- ],- "declaredTools": 69,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_speed_curve",- "round": 0,- "atMs": 1928.2015830000164,- "execMs": 0.22504200000548735,- "args": {- "clipId": "c2",- "nodes": [- {- "speed": 1,- "at": 0- },- {- "speed": 1,- "at": 3- },- {- "speed": 0.2,- "at": 4.7- }- ],- "smooth": true- }- },- {- "name": "set_speed_curve",- "round": 1,- "atMs": 3617.7700000000186,- "execMs": 0.0035829999833367765,- "args": {- "nodes": [- {- "at": 0,- "speed": 1- },- {- "speed": 1,- "at": 3- },- {- "speed": 0.2,- "at": 4.7- }- ],- "clipId": "c2",- "smooth": true- }- },- {- "name": "set_speed_curve",- "round": 2,- "atMs": 4972.047542000015,- "execMs": 0.001959000015631318,- "args": {- "smooth": true,- "nodes": [- {- "at": 0,- "speed": 1- },- {- "speed": 1,- "at": 3- },- {- "at": 4.7,- "speed": 0.2- }- ],- "clipId": "c2"- }- },- {- "name": "set_speed_curve",- "round": 3,- "atMs": 6293.7252920000465,- "execMs": 0.002249999961350113,- "args": {- "smooth": true,- "clipId": "c2",- "nodes": [- {- "at": 0,- "speed": 1- },- {- "at": 3.5,- "speed": 1- },- {- "speed": 0.2,- "at": 4.7- }- ]- }- },- {- "name": "set_speed_curve",- "round": 4,- "atMs": 8030.042667000031,- "execMs": 0.003916999965440482,- "args": {- "nodes": [- {- "at": 0,- "speed": 1- },- {- "speed": 1,- "at": 3.5- },- {- "speed": 0.2,- "at": 4.7- }- ],- "smooth": true,- "clipId": "c2"- }- }- ],- "timings": {- "gateMs": 282.7062079999596,- "gateSkipped": false,- "ttftMs": 9251.170000000042,- "modelMs": 9186.10312600003,- "toolMs": 0.23675099993124604,- "totalMs": 9477.76929200004,- "judgeMs": 456.96745799999917,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1641.5506250000326,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.22504200000548735- }- ]- },- {- "roundMs": 1687.6142080000136,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.0035829999833367765- }- ]- },- {- "roundMs": 1353.634625000006,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.001959000015631318- }- ]- },- {- "roundMs": 1321.1547089999658,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.002249999961350113- }- ]- },- {- "roundMs": 1735.5676669999957,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.003916999965440482- }- ]- },- {- "roundMs": 1446.5812920000171,- "firstDeltaMs": 1220.2594169999938,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "transitions",- "top_bar"- ],- "declaredTools": 66,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_speed_curve",- "round": 0,- "atMs": 1823.3552090000012,- "execMs": 0.005791999981738627,- "args": {- "smooth": true,- "clipId": "c2",- "preset": "rampOut"- }- },- {- "name": "get_state",- "round": 1,- "atMs": 3159.632209000003,- "execMs": 0.026083000004291534,- "args": {}- }- ],- "timings": {- "gateMs": 258.54383300000336,- "gateSkipped": false,- "ttftMs": 4348.372625000018,- "modelMs": 4238.458873999887,- "toolMs": 0.03187499998603016,- "totalMs": 4501.532874999975,- "judgeMs": 609.0904170001158,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1562.485415999894,- "firstDeltaMs": null,- "calls": [- {- "name": "set_speed_curve",- "execMs": 0.005791999981738627- }- ]- },- {- "roundMs": 1335.548375000013,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.026083000004291534- }- ]- },- {- "roundMs": 1340.42508299998,- "firstDeltaMs": 1187.7447500000708,+ "roundMs": 2550.990208000003,+ "firstDeltaMs": 2096.4137080000073,"calls": []}]@@ -6374,219 +3482,33 @@]},{- "name": "remove-grey-ones-scope",+ "name": "parked-debris-cleanup-after-retime","bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 236,- "p95": 251,- "mean": 243,- "n": 2- },- "ttftMs": {- "p50": 2585,- "p95": 2889,- "mean": 2737,- "n": 2- },- "totalMs": {- "p50": 2739,- "p95": 3056,- "mean": 2897,- "n": 2- },- "modelMs": {- "p50": 2501,- "p95": 2802,- "mean": 2651,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "animation",- "transitions",- "editor"- ],- "declaredTools": 68,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "remove_transition",- "round": 0,- "atMs": 1734.201457999996,- "execMs": 0.10962499992456287,- "args": {- "transitionId": "tr-8"- }- },- {- "name": "remove_transition",- "round": 0,- "atMs": 1734.420832999982,- "execMs": 0.0014170000795274973,- "args": {- "transitionId": "tr-9"- }- }- ],- "timings": {- "gateMs": 250.64108299999498,- "gateSkipped": false,- "ttftMs": 2888.526375000016,- "modelMs": 2802.1011250000447,- "toolMs": 0.11104200000409037,- "totalMs": 3055.6086249999935,- "judgeMs": 231.79179200006183,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1481.803791000042,- "firstDeltaMs": null,- "calls": [- {- "name": "remove_transition",- "execMs": 0.10962499992456287- },- {- "name": "remove_transition",- "execMs": 0.0014170000795274973- }- ]- },- {- "roundMs": 1320.2973340000026,- "firstDeltaMs": 1153.4305419999873,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "animation",- "transitions",- "editor"- ],- "declaredTools": 68,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "remove_transition",- "round": 0,- "atMs": 1677.0437500000698,- "execMs": 0.011959000024944544,- "args": {- "transitionId": "tr-8"- }- },- {- "name": "remove_transition",- "round": 0,- "atMs": 1677.1779589999933,- "execMs": 0.0012080000014975667,- "args": {- "transitionId": "tr-9"- }- }- ],- "timings": {- "gateMs": 235.84699999995064,- "gateSkipped": false,- "ttftMs": 2585.261625000043,- "modelMs": 2500.598875000025,- "toolMs": 0.01316700002644211,- "totalMs": 2738.548375000013,- "judgeMs": 225.56137500004843,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1440.2314999999944,- "firstDeltaMs": null,- "calls": [- {- "name": "remove_transition",- "execMs": 0.011959000024944544- },- {- "name": "remove_transition",- "execMs": 0.0012080000014975667- }- ]- },- {- "roundMs": 1060.3673750000307,- "firstDeltaMs": 907.1628329999512,- "calls": []- }- ]- }- }- ]- },- {- "name": "errored-call-not-claimed-done",- "bucket": "single-tool",- "passRate": 0.5,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 275,- "p95": 275,- "mean": 275,+ "p50": 580,+ "p95": 580,+ "mean": 580,"n": 1},"ttftMs": {- "p50": 2879,- "p95": 2879,- "mean": 2879,+ "p50": 6965,+ "p95": 6965,+ "mean": 6965,"n": 1},"totalMs": {- "p50": 3117,- "p95": 3117,- "mean": 3117,+ "p50": 7433,+ "p95": 7433,+ "mean": 7433,"n": 1},"modelMs": {- "p50": 2839,- "p95": 2839,- "mean": 2839,+ "p50": 6842,+ "p95": 6842,+ "mean": 6842,"n": 1},"toolMs": {@@ -6596,8 +3518,8 @@"n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 3}},"runs": [@@ -6610,111 +3532,70 @@"timeline","inspector","effects",- "transitions"- ],- "declaredTools": 62,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_transition",- "round": 0,- "atMs": 1721.6676659999648,- "execMs": 0.12704099994152784,- "args": {- "clipId": "c2",- "style": "crossfade",- "seconds": 0.5- }- }- ],- "timings": {- "gateMs": 275.1437089999672,- "gateSkipped": false,- "ttftMs": 2879.027124999906,- "modelMs": 2839.1242499999935,- "toolMs": 0.12704099994152784,- "totalMs": 3117.322832999984,- "judgeMs": 230.7743339999579,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1444.8355410000077,- "firstDeltaMs": null,- "calls": [- {- "name": "set_transition",- "execMs": 0.12704099994152784- }- ]- },- {- "roundMs": 1394.2887089999858,- "firstDeltaMs": 1156.2570840000408,- "calls": []- }- ]- }- },- {- "pass": false,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "transitions",- "audio"+ "top_bar",+ "editor"],- "declaredTools": 66,+ "declaredTools": 71,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite",- "notes": [- "reply did not match /last clip|nothing after|no clip after|can('|no)t|couldn't|didn('|o)t/i"- ],+ "notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "set_speed","round": 0,- "atMs": 1536.4541250000475,- "execMs": 0.01158299995586276,+ "atMs": 4214.487958999991,+ "execMs": 0.11812500000814907,"args": {- "seconds": 0.5,- "style": "crossfade",- "clipId": "c2"+ "clipId": "c2",+ "speed": 2+ }+ },+ {+ "name": "remove_transition",+ "round": 1,+ "atMs": 5808.418999999994,+ "execMs": 0.006374999997206032,+ "args": {+ "transitionId": "tr-2"}}],"timings": {- "gateMs": 233.51983399991877,+ "gateMs": 579.9407080000092,"gateSkipped": false,- "ttftMs": 2636.6545840000035,- "modelMs": 2626.2835409999825,- "toolMs": 0.01158299995586276,- "totalMs": 2862.516834000009,- "judgeMs": 252.18566600000486,+ "ttftMs": 6965.281958999985,+ "modelMs": 6842.032166999998,+ "toolMs": 0.1245000000053551,+ "totalMs": 7433.07362499999,+ "judgeMs": 607.7482500000042,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1300.7304579999764,+ "roundMs": 3629.110416999989,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.01158299995586276+ "name": "set_speed",+ "execMs": 0.11812500000814907+ }+ ]+ },+ {+ "roundMs": 1590.9830829999992,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "remove_transition",+ "execMs": 0.006374999997206032}]},{- "roundMs": 1325.5530830000062,- "firstDeltaMs": 1099.8327500000596,+ "roundMs": 1621.9386670000094,+ "firstDeltaMs": 1155.1537500000268,"calls": []}]@@ -6723,44 +3604,44 @@]},{- "name": "auto-continue-no-handoff",- "bucket": "multi-tool",+ "name": "speed-ramp-from-description",+ "bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 301,- "p95": 358,- "mean": 329,- "n": 2+ "p50": 620,+ "p95": 620,+ "mean": 620,+ "n": 1},"ttftMs": {- "p50": 4262,- "p95": 4426,- "mean": 4344,- "n": 2+ "p50": 19592,+ "p95": 19592,+ "mean": 19592,+ "n": 1},"totalMs": {- "p50": 4531,- "p95": 4728,- "mean": 4629,- "n": 2+ "p50": 19752,+ "p95": 19752,+ "mean": 19752,+ "n": 1},"modelMs": {- "p50": 4167,- "p95": 4425,- "mean": 4296,- "n": 2+ "p50": 19103,+ "p95": 19103,+ "mean": 19103,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 3,- "max": 3+ "mean": 13,+ "max": 13}},"runs": [@@ -6772,307 +3653,440 @@"preview","timeline","inspector",- "subtitles",- "library",+ "effects",+ "transitions","editor"],- "declaredTools": 85,+ "declaredTools": 67,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "update_cue",+ "name": "set_speed_curve","round": 0,- "atMs": 2072.292832999956,- "execMs": 0.1140420000301674,+ "atMs": 2013.8375000000233,+ "execMs": 0.0559159999829717,"args": {- "text": "So today we're",- "id": "cue1"+ "smooth": true,+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "at": 3,+ "speed": 1+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "clipId": "c2"+ }+ },+ {+ "name": "set_speed_curve",+ "round": 1,+ "atMs": 3492.2657090000284,+ "execMs": 0.004875000013271347,+ "args": {+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "at": 4.7,+ "speed": 0.2+ }+ ],+ "clipId": "c2",+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 2,+ "atMs": 4814.276334000024,+ "execMs": 0.003958999994210899,+ "args": {+ "smooth": true,+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "speed": 1,+ "at": 3+ },+ {+ "at": 4.7,+ "speed": 0.3+ }+ ],+ "clipId": "c2"+ }+ },+ {+ "name": "set_speed_curve",+ "round": 3,+ "atMs": 6173.668750000012,+ "execMs": 0.005207999987760559,+ "args": {+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "clipId": "c2",+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 4,+ "atMs": 7739.629417000018,+ "execMs": 0.006083999993279576,+ "args": {+ "clipId": "c2",+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "speed": 1,+ "at": 3.5+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 5,+ "atMs": 9213.063459000026,+ "execMs": 0.004165999998804182,+ "args": {+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "at": 4.7,+ "speed": 0.2+ }+ ],+ "clipId": "c2",+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 6,+ "atMs": 11171.220250000013,+ "execMs": 0.007291999994777143,+ "args": {+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "speed": 1,+ "at": 3.5+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "clipId": "c2",+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 7,+ "atMs": 12676.369292000018,+ "execMs": 0.01187499999650754,+ "args": {+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "clipId": "c2",+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 8,+ "atMs": 14145.417709000001,+ "execMs": 0.006999999983236194,+ "args": {+ "clipId": "c2",+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "smooth": true+ }+ },+ {+ "name": "set_speed_curve",+ "round": 9,+ "atMs": 15399.416917000024,+ "execMs": 0.0019580000080168247,+ "args": {+ "smooth": true,+ "clipId": "c2",+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "speed": 1,+ "at": 3.5+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ]}},{- "name": "update_cue",- "round": 0,- "atMs": 2072.5324580000015,- "execMs": 0.012875000014901161,+ "name": "set_speed_curve",+ "round": 10,+ "atMs": 16870.353625000018,+ "execMs": 0.30225000000791624,"args": {- "id": "cue3",- "text": "and they..."+ "nodes": [+ {+ "speed": 1,+ "at": 0+ },+ {+ "at": 3.5,+ "speed": 1+ },+ {+ "speed": 0.2,+ "at": 4.7+ }+ ],+ "clipId": "c2",+ "smooth": true}},{- "name": "align_to_audio",- "round": 1,- "atMs": 3336.3753750000615,- "execMs": 0.005250000045634806,+ "name": "set_speed_curve",+ "round": 11,+ "atMs": 18442.462625000015,+ "execMs": 0.007959000009577721,"args": {- "track": 0,- "target": "captions"+ "clipId": "c2",+ "smooth": true,+ "nodes": [+ {+ "at": 0,+ "speed": 1+ },+ {+ "at": 3,+ "speed": 1+ },+ {+ "at": 4.7,+ "speed": 0.2+ }+ ]}}],"timings": {- "gateMs": 357.8361249999143,+ "gateMs": 620.2392079999845,"gateSkipped": false,- "ttftMs": 4261.617958000046,- "modelMs": 4167.3184990000445,- "toolMs": 0.13216700009070337,- "totalMs": 4530.7826659999555,- "judgeMs": 579.5385419999948,+ "ttftMs": 19591.776000000013,+ "modelMs": 19102.94004200006,+ "toolMs": 0.4185419999703299,+ "totalMs": 19751.77587500002,+ "judgeMs": 557.0688329999975,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1711.5742910000263,+ "roundMs": 1386.8733749999956,"firstDeltaMs": null,"calls": [{- "name": "update_cue",- "execMs": 0.1140420000301674- },+ "name": "set_speed_curve",+ "execMs": 0.0559159999829717+ }+ ]+ },+ {+ "roundMs": 1476.84733400002,+ "firstDeltaMs": null,+ "calls": [{- "name": "update_cue",- "execMs": 0.012875000014901161+ "name": "set_speed_curve",+ "execMs": 0.004875000013271347}]},{- "roundMs": 1262.4753750000382,+ "roundMs": 1320.5820830000157,"firstDeltaMs": null,"calls": [{- "name": "align_to_audio",- "execMs": 0.005250000045634806+ "name": "set_speed_curve",+ "execMs": 0.003958999994210899}]},{- "roundMs": 1193.26883299998,- "firstDeltaMs": 924.1681249999674,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "subtitles",- "library",- "editor"- ],- "declaredTools": 73,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "update_cue",- "round": 0,- "atMs": 1986.8677500000922,- "execMs": 0.010458000004291534,- "args": {- "text": "so today we're",- "id": "cue1",- "end": 1.8,- "start": 0.45- }- },- {- "name": "update_cue",- "round": 0,- "atMs": 1986.9452080000192,- "execMs": 0.004791999934241176,- "args": {- "id": "cue3",- "end": 5.6,- "start": 4.2,- "text": "and"- }- },- {- "name": "align_to_audio",- "round": 1,- "atMs": 3433.6504160000477,- "execMs": 0.0033340000081807375,- "args": {- "track": 0,- "target": "captions"- }- }- ],- "timings": {- "gateMs": 300.56587499997113,- "gateSkipped": false,- "ttftMs": 4426.42291600001,- "modelMs": 4425.475332999951,- "toolMs": 0.018583999946713448,- "totalMs": 4727.584375000093,- "judgeMs": 621.2392080000136,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 1358.1209589999926,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.005207999987760559+ }+ ]+ },{- "roundMs": 1685.6646249999758,+ "roundMs": 1564.153707999998,"firstDeltaMs": null,"calls": [{- "name": "update_cue",- "execMs": 0.010458000004291534- },+ "name": "set_speed_curve",+ "execMs": 0.006083999993279576+ }+ ]+ },+ {+ "roundMs": 1471.576457999996,+ "firstDeltaMs": null,+ "calls": [{- "name": "update_cue",- "execMs": 0.004791999934241176+ "name": "set_speed_curve",+ "execMs": 0.004165999998804182}]},{- "roundMs": 1446.298999999999,+ "roundMs": 1956.6746250000142,"firstDeltaMs": null,"calls": [{- "name": "align_to_audio",- "execMs": 0.0033340000081807375+ "name": "set_speed_curve",+ "execMs": 0.007291999994777143}]},{- "roundMs": 1293.5117079999764,- "firstDeltaMs": 992.4179999999469,- "calls": []- }- ]- }- }- ]- },- {- "name": "question-when-ambiguous",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 215,- "p95": 326,- "mean": 270,- "n": 2- },- "ttftMs": {- "p50": 6411,- "p95": 24262,- "mean": 15337,- "n": 2- },- "totalMs": {- "p50": 6511,- "p95": 24513,- "mean": 15512,- "n": 2- },- "modelMs": {- "p50": 6182,- "p95": 24297,- "mean": 15239,- "n": 2- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "skill": "timeline-editing",- "areas": [- "preview",- "timeline",- "inspector",- "library",- "editor"- ],- "declaredTools": 72,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 214.98616699990816,- "gateSkipped": false,- "ttftMs": 24262.332709000097,- "modelMs": 24296.85908299999,- "toolMs": 0,- "totalMs": 24513.341042000102,- "judgeMs": 538.3621669999557,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 1502.4437920000055,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.01187499999650754+ }+ ]+ },{- "roundMs": 1383.499833000009,+ "roundMs": 1466.7542919999978,"firstDeltaMs": null,- "calls": []+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.006999999983236194+ }+ ]},{- "roundMs": 22913.35924999998,- "firstDeltaMs": 22662.75754099991,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "timeline-editing",- "areas": [- "preview",- "timeline",- "inspector",- "library",- "top_bar",- "editor"- ],- "declaredTools": 76,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 325.60108399996534,- "gateSkipped": false,- "ttftMs": 6411.490166999982,- "modelMs": 6182.048249999993,- "toolMs": 0,- "totalMs": 6511.39374999993,- "judgeMs": 560.9472079999978,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 1252.5277500000084,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.0019580000080168247+ }+ ]+ },{- "roundMs": 4285.07583300001,+ "roundMs": 1469.813124999986,"firstDeltaMs": null,- "calls": []+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.30225000000791624+ }+ ]+ },+ {+ "roundMs": 1569.556750000018,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "set_speed_curve",+ "execMs": 0.007959000009577721+ }+ ]},{- "roundMs": 1896.9724169999827,- "firstDeltaMs": 1797.1950840000063,+ "roundMs": 1307.0157910000125,+ "firstDeltaMs": 1147.6045000000158,"calls": []}]@@ -7081,40 +4095,40 @@]},{- "name": "orphaned-overlay-debris",+ "name": "remove-grey-ones-scope","bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 243,- "p95": 279,- "mean": 261,- "n": 2+ "p50": 524,+ "p95": 524,+ "mean": 524,+ "n": 1},"ttftMs": {- "p50": 2588,- "p95": 2766,- "mean": 2677,- "n": 2+ "p50": 3189,+ "p95": 3189,+ "mean": 3189,+ "n": 1},"totalMs": {- "p50": 2817,- "p95": 3017,- "mean": 2917,- "n": 2+ "p50": 3377,+ "p95": 3377,+ "mean": 3377,+ "n": 1},"modelMs": {- "p50": 2570,- "p95": 2735,- "mean": 2652,- "n": 2+ "p50": 2848,+ "p95": 2848,+ "mean": 2848,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {"mean": 2,@@ -7129,105 +4143,183 @@"areas": ["preview","timeline",- "library",- "top_bar"+ "inspector",+ "effects",+ "animation",+ "transitions",+ "side_panel",+ "editor"],- "declaredTools": 60,+ "declaredTools": 76,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "remove_transition","round": 0,- "atMs": 1449.8045830000192,- "execMs": 0.16641599999275059,+ "atMs": 2008.9373750000086,+ "execMs": 0.19495900001493283,"args": {- "clipId": "c1",- "out": 10+ "transitionId": "tr-8"+ }+ },+ {+ "name": "remove_transition",+ "round": 0,+ "atMs": 2009.395292000001,+ "execMs": 0.002749999985098839,+ "args": {+ "transitionId": "tr-9"}}],"timings": {- "gateMs": 242.78445799998008,+ "gateMs": 524.1194590000086,"gateSkipped": false,- "ttftMs": 2587.984250000096,- "modelMs": 2569.9313339999644,- "toolMs": 0.16641599999275059,- "totalMs": 2817.488333000103,- "judgeMs": 316.0043750000186,+ "ttftMs": 3189.453917000006,+ "modelMs": 2848.478959,+ "toolMs": 0.19770900000003166,+ "totalMs": 3376.977083000005,+ "judgeMs": 275.72816699999385,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1203.449874999933,+ "roundMs": 1482.5566249999974,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.16641599999275059+ "name": "remove_transition",+ "execMs": 0.19495900001493283+ },+ {+ "name": "remove_transition",+ "execMs": 0.002749999985098839}]},{- "roundMs": 1366.4814590000315,- "firstDeltaMs": 1137.108750000014,+ "roundMs": 1365.9223340000026,+ "firstDeltaMs": 1178.8429999999935,"calls": []}]}- },+ }+ ]+ },+ {+ "name": "errored-call-not-claimed-done",+ "bucket": "single-tool",+ "passRate": 0,+ "budgetBreaches": [],+ "latency": null,+ "runs": [{- "pass": true,- "intent": "simple",- "skill": null,+ "pass": false,+ "intent": "complex",+ "skill": "transitions-and-fades","areas": ["preview","timeline",- "top_bar"+ "inspector",+ "effects",+ "animation",+ "transitions"],- "declaredTools": 53,+ "declaredTools": 65,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],+ "roundModel": "gemini-3.8-flash",+ "notes": [+ "mutating tools called: set_animation, set_clip_keyframes"+ ],+ "instant": "set_transition","judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "set_transition","round": 0,- "atMs": 1654.0348329999251,- "execMs": 0.003874999936670065,+ "atMs": 660.7137079999957,+ "execMs": 0.22049999999580905,"args": {- "clipId": "c1",- "out": 10+ "clipId": "c2",+ "style": "crossfade",+ "seconds": 0.2+ }+ },+ {+ "name": "set_animation",+ "round": 0,+ "atMs": 6211.169124999986,+ "execMs": 0.01041699998313561,+ "args": {+ "clipId": "c2",+ "seconds": 0.5,+ "style": "fade",+ "which": "out"+ }+ },+ {+ "name": "set_clip_keyframes",+ "round": 1,+ "atMs": 8729.472875000007,+ "execMs": 0.027833000000100583,+ "args": {+ "keys": [+ {+ "opacity": 1,+ "t": 0+ },+ {+ "opacity": 1,+ "t": 6+ },+ {+ "opacity": 0,+ "t": 6.5+ }+ ],+ "clipId": "c2"}}],"timings": {- "gateMs": 279.2076250000391,+ "gateMs": 659.8214159999916,"gateSkipped": false,- "ttftMs": 2766.3834999999963,- "modelMs": 2734.90945799998,- "toolMs": 0.003874999936670065,- "totalMs": 3016.5371250000317,- "judgeMs": 257.99887500004843,+ "ttftMs": 12099.956416,+ "modelMs": 11944.831166999938,+ "toolMs": 0.25874999997904524,+ "totalMs": 12619.98879100001,+ "judgeMs": 733.2519999999786,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1373.5055830000201,+ "roundMs": 5547.543499999971,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.003874999936670065+ "name": "set_animation",+ "execMs": 0.01041699998313561+ }+ ]+ },+ {+ "roundMs": 2510.194334,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "set_clip_keyframes",+ "execMs": 0.027833000000100583}]},{- "roundMs": 1361.40387499996,- "firstDeltaMs": 1111.5779999999795,+ "roundMs": 3887.093332999968,+ "firstDeltaMs": 3367.6148329999996,"calls": []}]@@ -7236,44 +4328,44 @@]},{- "name": "legacy-thread-no-leak",- "bucket": "chat",+ "name": "auto-continue-no-handoff",+ "bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 266,- "p95": 326,- "mean": 296,- "n": 2+ "p50": 522,+ "p95": 522,+ "mean": 522,+ "n": 1},"ttftMs": {- "p50": 1521,- "p95": 1609,- "mean": 1565,- "n": 2+ "p50": 5673,+ "p95": 5673,+ "mean": 5673,+ "n": 1},"totalMs": {- "p50": 1664,- "p95": 1779,- "mean": 1722,- "n": 2+ "p50": 5895,+ "p95": 5895,+ "mean": 5895,+ "n": 1},"modelMs": {- "p50": 1397,- "p95": 1451,- "mean": 1424,- "n": 2+ "p50": 5361,+ "p95": 5361,+ "mean": 5361,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 1,- "max": 1+ "mean": 4,+ "max": 4}},"runs": [@@ -7285,68 +4377,108 @@"preview","timeline","inspector",- "effects",- "animation",- "transitions"+ "subtitles",+ "library",+ "editor"],- "declaredTools": 64,+ "declaredTools": 86,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,- "trace": [],- "timings": {- "gateMs": 265.52754099993035,- "gateSkipped": false,- "ttftMs": 1520.9474170000758,- "modelMs": 1396.7702079999726,- "toolMs": 0,- "totalMs": 1664.1880420000525,- "judgeMs": 235.89158299996052,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1396.7702079999726,- "firstDeltaMs": 1253.8447079999605,- "calls": []+ "trace": [+ {+ "name": "find_filler",+ "round": 0,+ "atMs": 1959.092416999978,+ "execMs": 0.1156250000349246,+ "args": {+ "track": 0,+ "clip_id": "c1"}- ]- }- },- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "timeline",- "inspector",- "effects",- "animation",- "transitions",- "library"+ },+ {+ "name": "update_cue",+ "round": 1,+ "atMs": 3253.112124999985,+ "execMs": 0.0372499999939464,+ "args": {+ "text": "So today we're",+ "id": "cue1",+ "start": 0.45+ }+ },+ {+ "name": "update_cue",+ "round": 1,+ "atMs": 3253.4155419999734,+ "execMs": 0.0047919999924488366,+ "args": {+ "id": "cue3",+ "start": 4.2,+ "text": "And"+ }+ },+ {+ "name": "align_to_audio",+ "round": 2,+ "atMs": 4449.227374999959,+ "execMs": 0.018041000002995133,+ "args": {+ "track": 0,+ "target": "captions"+ }+ }],- "declaredTools": 71,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],"timings": {- "gateMs": 325.848791000084,+ "gateMs": 522.4588329999824,"gateSkipped": false,- "ttftMs": 1608.6598339999327,- "modelMs": 1451.1174590000883,- "toolMs": 0,- "totalMs": 1779.079625000013,- "judgeMs": 256.4152920000488,+ "ttftMs": 5672.658832999994,+ "modelMs": 5360.827416999906,+ "toolMs": 0.17570800002431497,+ "totalMs": 5894.63937499997,+ "judgeMs": 499.80562499997905,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1451.1174590000883,- "firstDeltaMs": 1280.7577499999898,+ "roundMs": 1433.821249999979,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "find_filler",+ "execMs": 0.1156250000349246+ }+ ]+ },+ {+ "roundMs": 1290.9186669999617,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "update_cue",+ "execMs": 0.0372499999939464+ },+ {+ "name": "update_cue",+ "execMs": 0.0047919999924488366+ }+ ]+ },+ {+ "roundMs": 1193.7374999999884,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "align_to_audio",+ "execMs": 0.018041000002995133+ }+ ]+ },+ {+ "roundMs": 1442.3499999999767,+ "firstDeltaMs": 1221.8870419999585,"calls": []}]@@ -7355,158 +4487,79 @@]},{- "name": "cinematic-ask-picks-preset",- "bucket": "single-tool",+ "name": "question-when-ambiguous",+ "bucket": "chat","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 272,- "p95": 441,- "mean": 356,- "n": 2+ "p50": 685,+ "p95": 685,+ "mean": 685,+ "n": 1},"ttftMs": {- "p50": 2899,- "p95": 4912,- "mean": 3906,- "n": 2+ "p50": 2690,+ "p95": 2690,+ "mean": 2690,+ "n": 1},"totalMs": {- "p50": 3216,- "p95": 5461,- "mean": 4338,- "n": 2+ "p50": 2778,+ "p95": 2778,+ "mean": 2778,+ "n": 1},"modelMs": {- "p50": 2943,- "p95": 5051,- "mean": 3997,- "n": 2+ "p50": 2090,+ "p95": 2090,+ "mean": 2090,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 2,- "max": 2+ "mean": 1,+ "max": 1}},- "runs": [- {- "pass": true,- "intent": "simple",- "skill": null,- "areas": [- "preview",- "inspector",- "color",- "effects",- "side_panel",- "top_bar"- ],- "declaredTools": 52,- "routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_color_preset",- "round": 0,- "atMs": 2445.184041999979,- "execMs": 0.21729099994990975,- "args": {- "preset": "teal-orange",- "clipId": "c1"- }- }- ],- "timings": {- "gateMs": 440.5892499999609,- "gateSkipped": false,- "ttftMs": 4912.3126250000205,- "modelMs": 5050.79520900012,- "toolMs": 0.21729099994990975,- "totalMs": 5460.582999999984,- "judgeMs": 555.5982920000097,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2036.9795840000734,- "firstDeltaMs": null,- "calls": [- {- "name": "set_color_preset",- "execMs": 0.21729099994990975- }- ]- },- {- "roundMs": 3013.8156250000466,- "firstDeltaMs": 2465.772250000038,- "calls": []- }- ]- }- },+ "runs": [{"pass": true,- "intent": "simple",- "skill": null,+ "intent": "complex",+ "skill": "timeline-editing","areas": ["preview",+ "timeline","inspector",- "color",- "effects",- "side_panel",- "top_bar"+ "library",+ "editor"],- "declaredTools": 52,+ "declaredTools": 73,"routingMisses": [],- "roundModel": "gemini-3.1-flash-lite",+ "roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,- "trace": [- {- "name": "set_color_preset",- "round": 0,- "atMs": 1707.7004170001019,- "execMs": 0.013374999980442226,- "args": {- "preset": "teal-orange",- "clipId": "c1"- }- }- ],+ "trace": [],"timings": {- "gateMs": 271.7244589999318,+ "gateMs": 684.9555839999812,"gateSkipped": false,- "ttftMs": 2898.7929590001004,- "modelMs": 2943.199459000025,- "toolMs": 0.013374999980442226,- "totalMs": 3216.1091250000754,- "judgeMs": 284.2845420000376,+ "ttftMs": 2690.4507919999887,+ "modelMs": 2090.2573329999577,+ "toolMs": 0,+ "totalMs": 2777.9478339999914,+ "judgeMs": 259.8042499999865,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1435.3386669999454,- "firstDeltaMs": null,- "calls": [- {- "name": "set_color_preset",- "execMs": 0.013374999980442226- }- ]- },- {- "roundMs": 1507.8607920000795,- "firstDeltaMs": 1190.7736250000307,+ "roundMs": 2090.2573329999577,+ "firstDeltaMs": 2003.0821659999783,"calls": []}]@@ -7515,44 +4568,44 @@]},{- "name": "warmer-crushed-blacks-in-range",- "bucket": "single-tool",+ "name": "orphaned-overlay-debris",+ "bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 245,- "p95": 265,- "mean": 255,- "n": 2+ "p50": 529,+ "p95": 529,+ "mean": 529,+ "n": 1},"ttftMs": {- "p50": 3704,- "p95": 5166,- "mean": 4435,- "n": 2+ "p50": 2989,+ "p95": 2989,+ "mean": 2989,+ "n": 1},"totalMs": {- "p50": 4010,- "p95": 5351,- "mean": 4680,- "n": 2+ "p50": 3194,+ "p95": 3194,+ "mean": 3194,+ "n": 1},"modelMs": {- "p50": 3761,- "p95": 5080,- "mean": 4420,- "n": 2+ "p50": 2661,+ "p95": 2661,+ "mean": 2661,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 3.5,- "max": 4+ "mean": 2,+ "max": 2}},"runs": [@@ -7562,152 +4615,133 @@"skill": null,"areas": ["preview",- "inspector",- "color",- "effects"+ "timeline",+ "inspector"],- "declaredTools": 41,+ "declaredTools": 62,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "set_color_grade",- "round": 1,- "atMs": 2790.560208999901,- "execMs": 0.07479099999181926,- "args": {- "clipId": "c1",- "temperature": 20- }- },- {- "name": "set_color_curves",- "round": 1,- "atMs": 2790.9987089999486,- "execMs": 0.0023750000400468707,- "args": {- "clipId": "c1",- "curve_contrast": 10- }- },- {- "name": "set_color_grade",- "round": 2,- "atMs": 4113.332999999984,- "execMs": 0.010834000073373318,+ "name": "trim_clip",+ "round": 0,+ "atMs": 1882.4166250000126,+ "execMs": 0.1556659999769181,"args": {"clipId": "c1",- "shadows": -10+ "out": 10}}],"timings": {- "gateMs": 264.95016599993687,+ "gateMs": 529.1502079999773,"gateSkipped": false,- "ttftMs": 5166.28583399998,- "modelMs": 5079.554542000056,- "toolMs": 0.08800000010523945,- "totalMs": 5350.7168749999255,- "judgeMs": 637.7544169999892,+ "ttftMs": 2989.3134169999976,+ "modelMs": 2661.2188339999993,+ "toolMs": 0.1556659999769181,+ "totalMs": 3193.9387500000303,+ "judgeMs": 545.0367499999702,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1224.7247500000522,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 1297.2725830000127,- "firstDeltaMs": null,- "calls": [- {- "name": "set_color_grade",- "execMs": 0.07479099999181926- },- {- "name": "set_color_curves",- "execMs": 0.0023750000400468707- }- ]- },- {- "roundMs": 1321.6398340000305,+ "roundMs": 1351.2681250000023,"firstDeltaMs": null,"calls": [{- "name": "set_color_grade",- "execMs": 0.010834000073373318+ "name": "trim_clip",+ "execMs": 0.1556659999769181}]},{- "roundMs": 1235.9173749999609,- "firstDeltaMs": 1051.7169170000125,+ "roundMs": 1309.950708999997,+ "firstDeltaMs": 1105.6849999999977,"calls": []}]}+ }+ ]+ },+ {+ "name": "legacy-thread-no-leak",+ "bucket": "chat",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 330,+ "p95": 330,+ "mean": 330,+ "n": 1+ },+ "ttftMs": {+ "p50": 1788,+ "p95": 1788,+ "mean": 1788,+ "n": 1+ },+ "totalMs": {+ "p50": 2028,+ "p95": 2028,+ "mean": 2028,+ "n": 1+ },+ "modelMs": {+ "p50": 1696,+ "p95": 1696,+ "mean": 1696,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 1,+ "max": 1+ }+ },+ "runs": [{"pass": true,"intent": "simple","skill": null,"areas": ["preview",+ "timeline","inspector",- "color",- "effects"+ "effects",+ "animation",+ "transitions"],- "declaredTools": 41,+ "declaredTools": 65,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,- "trace": [- {- "name": "set_color_grade",- "round": 1,- "atMs": 2705.120666999952,- "execMs": 0.0036250000121071935,- "args": {- "clipId": "c1",- "shadows": -10,- "temperature": 20- }- }- ],+ "trace": [],"timings": {- "gateMs": 245.48908299999312,+ "gateMs": 330.1852910000016,"gateSkipped": false,- "ttftMs": 3704.3957499999087,- "modelMs": 3761.363999999943,- "toolMs": 0.0036250000121071935,- "totalMs": 4010.2012499999255,- "judgeMs": 214.22733400005382,+ "ttftMs": 1787.689040999976,+ "modelMs": 1695.9717499999679,+ "toolMs": 0,+ "totalMs": 2027.8691659999895,+ "judgeMs": 270.4465419999906,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1324.3317919999827,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 1132.6171669999603,- "firstDeltaMs": null,- "calls": [- {- "name": "set_color_grade",- "execMs": 0.0036250000121071935- }- ]- },- {- "roundMs": 1304.4150410000002,- "firstDeltaMs": 998.7812080000294,+ "roundMs": 1695.9717499999679,+ "firstDeltaMs": 1456.407917000004,"calls": []}]@@ -7716,44 +4750,44 @@]},{- "name": "sky-bluer-single-band",+ "name": "cinematic-ask-picks-preset","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 241,- "p95": 300,- "mean": 271,- "n": 2+ "p50": 280,+ "p95": 280,+ "mean": 280,+ "n": 1},"ttftMs": {- "p50": 6287,- "p95": 6714,- "mean": 6501,- "n": 2+ "p50": 282,+ "p95": 282,+ "mean": 282,+ "n": 1},"totalMs": {- "p50": 6726,- "p95": 7301,- "mean": 7013,- "n": 2+ "p50": 283,+ "p95": 283,+ "mean": 283,+ "n": 1},"modelMs": {- "p50": 6419,- "p95": 7051,- "mean": 6735,- "n": 2- },- "toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1+ },+ "toolMs": {+ "p50": 1,+ "p95": 1,+ "mean": 1,+ "n": 1},"rounds": {- "mean": 4,- "max": 4+ "mean": 0,+ "max": 0}},"runs": [@@ -7766,80 +4800,85 @@"inspector","color","effects",- "image_gen"+ "side_panel",+ "top_bar"],- "declaredTools": 42,+ "declaredTools": 52,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": "set_color_preset","judgeNote": null,"trace": [{- "name": "seek",- "round": 1,- "atMs": 3337.565500000026,- "execMs": 0.23183299996890128,- "args": {- "t": 0- }- },- {- "name": "set_color_hsl",- "round": 2,- "atMs": 5158.769417000003,- "execMs": 0.013999999966472387,+ "name": "set_color_preset",+ "round": 0,+ "atMs": 281.4969999999739,+ "execMs": 0.5242079999879934,"args": {"clipId": "c1",- "band": "blue",- "sat": 25,- "hue": 15+ "preset": "teal-orange",+ "amount": 0.69}}],"timings": {- "gateMs": 241.17504200001713,+ "gateMs": 280.0892500000191,"gateSkipped": false,- "ttftMs": 6713.8770830000285,- "modelMs": 7050.863166999887,- "toolMs": 0.24583299993537366,- "totalMs": 7300.524457999971,- "judgeMs": 584.8700420000823,+ "ttftMs": 282.4989999999525,+ "modelMs": 0,+ "toolMs": 0.5242079999879934,+ "totalMs": 282.5238749999553,+ "judgeMs": 247.4059160000179,"retryMs": 0,"retries": 0,- "rounds": [- {- "roundMs": 1213.1531249999534,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 1878.9036659999983,- "firstDeltaMs": null,- "calls": [- {- "name": "seek",- "execMs": 0.23183299996890128- }- ]- },- {- "roundMs": 1818.3687919999938,- "firstDeltaMs": null,- "calls": [- {- "name": "set_color_hsl",- "execMs": 0.013999999966472387- }- ]- },- {- "roundMs": 2140.437583999941,- "firstDeltaMs": 1554.0435419999994,- "calls": []- }- ]+ "rounds": []}+ }+ ]+ },+ {+ "name": "warmer-crushed-blacks-in-range",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 395,+ "p95": 395,+ "mean": 395,+ "n": 1+ },+ "ttftMs": {+ "p50": 6540,+ "p95": 6540,+ "mean": 6540,+ "n": 1+ },+ "totalMs": {+ "p50": 6710,+ "p95": 6710,+ "mean": 6710,+ "n": 1},+ "modelMs": {+ "p50": 6302,+ "p95": 6302,+ "mean": 6302,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1+ },+ "rounds": {+ "mean": 3,+ "max": 3+ }+ },+ "runs": [{"pass": true,"intent": "simple",@@ -7848,76 +4887,56 @@"preview","inspector","color",- "effects",- "image_gen"+ "effects"],- "declaredTools": 42,+ "declaredTools": 41,"routingMisses": [],"roundModel": "gemini-3.1-flash-lite","notes": [],+ "instant": null,"judgeNote": null,"trace": [{- "name": "seek",+ "name": "set_color_grade","round": 1,- "atMs": 3156.8108749999665,- "execMs": 0.014333999948576093,- "args": {- "t": 0- }- },- {- "name": "set_color_hsl",- "round": 2,- "atMs": 5049.040666999994,- "execMs": 0.01362500002142042,+ "atMs": 5570.3494170000195,+ "execMs": 0.3076249999576248,"args": {- "hue": 20,- "sat": 25,- "band": "blue",- "clipId": "c1"+ "temperature": 20,+ "clipId": "c1",+ "shadows": -15}}],"timings": {- "gateMs": 300.2105000000447,+ "gateMs": 395.3481669999892,"gateSkipped": false,- "ttftMs": 6287.196249999921,- "modelMs": 6418.930500999908,- "toolMs": 0.027958999969996512,- "totalMs": 6725.693250000011,- "judgeMs": 541.2030830000294,+ "ttftMs": 6540.034542000038,+ "modelMs": 6301.72504099994,+ "toolMs": 0.3076249999576248,+ "totalMs": 6710.445959000033,+ "judgeMs": 576.8358749999898,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1158.9759589999449,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 1693.9996249999385,+ "roundMs": 3856.6763330000103,"firstDeltaMs": null,- "calls": [- {- "name": "seek",- "execMs": 0.014333999948576093- }- ]+ "calls": []},{- "roundMs": 1890.6579589999747,+ "roundMs": 1307.2124159999657,"firstDeltaMs": null,"calls": [{- "name": "set_color_hsl",- "execMs": 0.01362500002142042+ "name": "set_color_grade",+ "execMs": 0.3076249999576248}]},{- "roundMs": 1675.29695800005,- "firstDeltaMs": 1236.9017080001067,+ "roundMs": 1137.8362919999636,+ "firstDeltaMs": 967.8776249999646,"calls": []}]@@ -7926,124 +4945,135 @@]},{- "name": "reference-photo-matches-grade",+ "name": "sky-bluer-single-band","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 255,- "p95": 264,- "mean": 259,- "n": 2+ "p50": 451,+ "p95": 451,+ "mean": 451,+ "n": 1},"ttftMs": {- "p50": 10077,- "p95": 20151,- "mean": 15114,- "n": 2+ "p50": 452,+ "p95": 452,+ "mean": 452,+ "n": 1},"totalMs": {- "p50": 10240,- "p95": 20436,- "mean": 15338,- "n": 2+ "p50": 452,+ "p95": 452,+ "mean": 452,+ "n": 1},"modelMs": {- "p50": 9969,- "p95": 20176,- "mean": 15072,- "n": 2+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 1},"rounds": {- "mean": 3,- "max": 3+ "mean": 0,+ "max": 0}},"runs": [{"pass": true,"intent": "complex",- "skill": null,+ "skill": "color-grading","areas": ["preview","inspector","color","effects",- "side_panel",- "top_bar"+ "image_gen",+ "editor"],- "declaredTools": 52,+ "declaredTools": 46,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": "set_color_hsl","judgeNote": null,"trace": [{- "name": "match_color_grade",+ "name": "set_color_hsl","round": 0,- "atMs": 5722.50245800009,- "execMs": 0.23270799999590963,- "args": {- "ref_asset_id": "a-i1",- "clipId": "c1"- }- },- {- "name": "read_color_stats",- "round": 1,- "atMs": 8448.422292000032,- "execMs": 0.053958000033162534,+ "atMs": 451.54308299999684,+ "execMs": 0.17145800002617761,"args": {- "clipId": "c1"+ "clipId": "c1",+ "band": "blue",+ "sat": 29}}],"timings": {- "gateMs": 263.6963749999413,- "gateSkipped": true,- "ttftMs": 10076.608708000043,- "modelMs": 9968.938291000086,- "toolMs": 0.28666600002907217,- "totalMs": 10239.683917000075,- "judgeMs": 499.6667080000043,+ "gateMs": 450.66599999996834,+ "gateSkipped": false,+ "ttftMs": 451.9269579999964,+ "modelMs": 0,+ "toolMs": 0.17145800002617761,+ "totalMs": 451.9367499999935,+ "judgeMs": 232.18725000001723,"retryMs": 0,"retries": 0,- "rounds": [- {- "roundMs": 5455.68979199999,- "firstDeltaMs": null,- "calls": [- {- "name": "match_color_grade",- "execMs": 0.23270799999590963- }- ]- },- {- "roundMs": 2723.8698330000043,- "firstDeltaMs": null,- "calls": [- {- "name": "read_color_stats",- "execMs": 0.053958000033162534- }- ]- },- {- "roundMs": 1789.3786660000915,- "firstDeltaMs": 1626.7485410000663,- "calls": []- }- ]+ "rounds": []}+ }+ ]+ },+ {+ "name": "reference-photo-matches-grade",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 322,+ "p95": 322,+ "mean": 322,+ "n": 1+ },+ "ttftMs": {+ "p50": 4720,+ "p95": 4720,+ "mean": 4720,+ "n": 1+ },+ "totalMs": {+ "p50": 4898,+ "p95": 4898,+ "mean": 4898,+ "n": 1+ },+ "modelMs": {+ "p50": 4569,+ "p95": 4569,+ "mean": 4569,+ "n": 1+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 1},+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,"intent": "complex",@@ -8054,68 +5084,51 @@"color","effects","side_panel",- "top_bar"+ "top_bar",+ "editor"],- "declaredTools": 52,+ "declaredTools": 56,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],+ "instant": null,"judgeNote": null,"trace": [{"name": "match_color_grade","round": 0,- "atMs": 16618.874959000037,- "execMs": 0.015000000013969839,+ "atMs": 2269.434332999983,+ "execMs": 0.15854199999012053,"args": {"ref_asset_id": "a-i1","clipId": "c1"}- },- {- "name": "read_color_stats",- "round": 1,- "atMs": 18550.126249999972,- "execMs": 0.008917000028304756,- "args": {- "clipId": "c1"- }}],"timings": {- "gateMs": 255.05629099998623,+ "gateMs": 322.3672089999891,"gateSkipped": true,- "ttftMs": 20151.05587499996,- "modelMs": 20175.66770799982,- "toolMs": 0.023917000042274594,- "totalMs": 20436.48333399999,- "judgeMs": 523.6447499999776,+ "ttftMs": 4719.961791999987,+ "modelMs": 4569.165667000052,+ "toolMs": 0.15854199999012053,+ "totalMs": 4897.899708000012,+ "judgeMs": 598.573040999996,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 16361.533707999974,+ "roundMs": 1942.8682090000366,"firstDeltaMs": null,"calls": [{"name": "match_color_grade",- "execMs": 0.015000000013969839- }- ]- },- {- "roundMs": 1929.4221249999246,- "firstDeltaMs": null,- "calls": [- {- "name": "read_color_stats",- "execMs": 0.008917000028304756+ "execMs": 0.15854199999012053}]},{- "roundMs": 1884.7118749999208,- "firstDeltaMs": 1599.4977499999804,+ "roundMs": 2626.2974580000155,+ "firstDeltaMs": 2448.595250000013,"calls": []}]@@ -8130,38 +5143,38 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 256,- "p95": 275,- "mean": 265,- "n": 2+ "p50": 476,+ "p95": 476,+ "mean": 476,+ "n": 1},"ttftMs": {- "p50": 13186,- "p95": 13852,- "mean": 13519,- "n": 2+ "p50": 8314,+ "p95": 8314,+ "mean": 8314,+ "n": 1},"totalMs": {- "p50": 13680,- "p95": 14212,- "mean": 13946,- "n": 2+ "p50": 8798,+ "p95": 8798,+ "mean": 8798,+ "n": 1},"modelMs": {- "p50": 13386,- "p95": 13935,- "mean": 13660,- "n": 2+ "p50": 8311,+ "p95": 8311,+ "mean": 8311,+ "n": 1},"toolMs": {"p50": 0,- "p95": 1,- "mean": 1,- "n": 2+ "p95": 0,+ "mean": 0,+ "n": 1},"rounds": {- "mean": 5.5,- "max": 6+ "mean": 3,+ "max": 3}},"runs": [@@ -8174,7 +5187,6 @@"timeline","inspector","color",- "removal","effects","animation","transitions",@@ -8187,248 +5199,76 @@"top_bar","editor"],- "declaredTools": 121,+ "declaredTools": 119,"routingMisses": [],"roundModel": "gemini-3.8-flash","notes": [],- "judgeNote": "upsell 0.54",+ "instant": null,+ "judgeNote": "upsell 0.57","trace": [{"name": "read_project",- "round": 1,- "atMs": 7373.890207999968,- "execMs": 0.689790999982506,+ "round": 0,+ "atMs": 3115.3913750000065,+ "execMs": 0.20216700003948063,"args": {"link": "https://donkeycut.com/app/p/ref-123"}},{"name": "replicate_project",- "round": 2,- "atMs": 9963.025707999943,- "execMs": 0.1279169999761507,+ "round": 1,+ "atMs": 6177.915750000044,+ "execMs": 0.1061250000493601,"args": {"link": "https://donkeycut.com/app/p/ref-123","media": [{- "source_asset_id": "r-v1",- "asset_id": "a-v1"- },- {- "asset_id": "a-v2",- "source_asset_id": "r-v2"- }- ]- }- },- {- "name": "update_overlay",- "round": 3,- "atMs": 11836.166540999897,- "execMs": 0.03116699995007366,- "args": {- "id": "t-n1",- "text": "MY DESK SETUP"- }- },- {- "name": "subtitles_generate",- "round": 3,- "atMs": 11836.586374999955,- "execMs": 0.03554099996108562,- "args": {}- }- ],- "timings": {- "gateMs": 255.5660420000786,- "gateSkipped": false,- "ttftMs": 13851.970332999947,- "modelMs": 13934.569001000025,- "toolMs": 0.884415999869816,- "totalMs": 14211.674124999903,- "judgeMs": 628.9623330000322,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4637.172499999986,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2474.225375000038,- "firstDeltaMs": null,- "calls": [- {- "name": "read_project",- "execMs": 0.689790999982506- }- ]- },- {- "roundMs": 2583.4949999999953,- "firstDeltaMs": null,- "calls": [- {- "name": "replicate_project",- "execMs": 0.1279169999761507- }- ]- },- {- "roundMs": 1867.8827089999104,- "firstDeltaMs": null,- "calls": [- {- "name": "update_overlay",- "execMs": 0.03116699995007366+ "asset_id": "a-v1",+ "source_asset_id": "r-v1"},{- "name": "subtitles_generate",- "execMs": 0.03554099996108562+ "source_asset_id": "r-v2",+ "asset_id": "a-v2"}]- },- {- "roundMs": 2371.7934170000954,- "firstDeltaMs": 2013.3481250000186,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "skill": "replicating-a-project",- "areas": [- "preview",- "timeline",- "inspector",- "color",- "effects",- "animation",- "transitions",- "subtitles",- "audio",- "video_gen",- "scene",- "library",- "side_panel",- "top_bar",- "editor"- ],- "declaredTools": 118,- "routingMisses": [],- "roundModel": "gemini-3.8-flash",- "notes": [],- "judgeNote": "upsell 0.57",- "trace": [- {- "name": "read_project",- "round": 1,- "atMs": 4354.051875000005,- "execMs": 0.0339169999351725,- "args": {- "link": "https://donkeycut.com/app/p/ref-123"- }- },- {- "name": "replicate_project",- "round": 2,- "atMs": 7220.381125000073,- "execMs": 0.11149999999906868,- "args": {- "media": [- {- "source_asset_id": "r-v1",- "asset_id": "a-v1"- },- {- "asset_id": "a-v2",- "source_asset_id": "r-v2"- }- ],- "link": "https://donkeycut.com/app/p/ref-123"- }- },- {- "name": "update_overlay",- "round": 3,- "atMs": 9275.000750000007,- "execMs": 0.033500000019557774,- "args": {- "text": "MY DESK SETUP",- "id": "t-n1"}- },- {- "name": "subtitles_generate",- "round": 4,- "atMs": 11213.23350000009,- "execMs": 0.03175000008195639,- "args": {}}],"timings": {- "gateMs": 275.12804199999664,+ "gateMs": 475.94483300001593,"gateSkipped": false,- "ttftMs": 13185.857250000001,- "modelMs": 13385.525876000058,- "toolMs": 0.21066700003575534,- "totalMs": 13680.230250000022,- "judgeMs": 590.8087079999968,+ "ttftMs": 8313.829958000046,+ "modelMs": 8311.124376000022,+ "toolMs": 0.3082920000888407,+ "totalMs": 8798.196000000054,+ "judgeMs": 562.6585830000113,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1735.5736670000479,- "firstDeltaMs": null,- "calls": []- },- {- "roundMs": 2335.4039169999305,+ "roundMs": 2634.9706250000163,"firstDeltaMs": null,"calls": [{"name": "read_project",- "execMs": 0.0339169999351725+ "execMs": 0.20216700003948063}]},{- "roundMs": 2863.430708000087,+ "roundMs": 3058.2170840000035,"firstDeltaMs": null,"calls": [{"name": "replicate_project",- "execMs": 0.11149999999906868- }- ]- },- {- "roundMs": 2051.527374999947,- "firstDeltaMs": null,- "calls": [- {- "name": "update_overlay",- "execMs": 0.033500000019557774- }- ]- },- {- "roundMs": 1935.511792000034,- "firstDeltaMs": null,- "calls": [- {- "name": "subtitles_generate",- "execMs": 0.03175000008195639+ "execMs": 0.1061250000493601}]},{- "roundMs": 2464.0784170000115,- "firstDeltaMs": 1970.1614590000827,+ "roundMs": 2617.9366670000018,+ "firstDeltaMs": 2133.771374999953,"calls": []}]diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.tsindex 4641bc79..2b462399 100644--- a/site/scripts/eval-cut-chat.ts+++ b/site/scripts/eval-cut-chat.ts@@ -140,6 +140,7 @@ async function runConfig(` ${report.passRate === 1 ? "ok " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` +` pass ${pct(report.passRate)} total p50 ${ms(l?.totalMs.p50)} ttft p50 ${ms(l?.ttftMs?.p50)}` +` rounds ${l ? l.rounds.mean : "—"}` ++ (runs.find((r) => r.instant) ? ` instant ${runs.find((r) => r.instant)?.instant}` : "") +(report.budgetBreaches.length > 0 ? ` BUDGET: ${report.budgetBreaches.join("; ")}` : ""));for (const r of runs.filter((r) => !r.pass)) {diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.tsindex a3e52f39..3fc54bc0 100644--- a/site/scripts/lib/cut-eval/cases.ts+++ b/site/scripts/lib/cut-eval/cases.ts@@ -4,6 +4,7 @@* project-mutating tool calls, edit requests still reach for tools.** Every case carries a latency bucket:+ * instant — the judgment settles the whole turn: no model round at all* chat — the fast path: gated greetings and question turns* single-tool — one decisive call reaches the right tool* multi-tool — composed edits over several calls@@ -37,7 +38,7 @@ import {userTurn,} from "./fixtures";-export type Bucket = "chat" | "single-tool" | "multi-tool";+export type Bucket = "instant" | "chat" | "single-tool" | "multi-tool";export interface EvalCase {name: string;@@ -53,6 +54,10 @@ export interface EvalCase {/** Latency guard: the turn fails if the trace exceeds this many calls. A* simple ask must not detour through skill reads or state polls. */maxToolCalls?: number;+ /** The registry action the instant path must settle this turn to, with no+ * model round at all. `null` pins the opposite: the turn must reach the+ * model. Left out, either is allowed. */+ instant?: string | null;/** The gate side the turn must classify to: "chat" turns run with no tool* declarations (tool calls become impossible), "work" covers both routing* verdicts (simple and complex). */@@ -95,6 +100,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// subtitles_generate): a bare greeting asks for nothing, so the turn// must classify "chat" and run with every tool declaration withheld.name: "greeting-is-gated",+ instant: null,bucket: "chat",input: () => [userTurn("hi")],reply: /help|what would you/i,@@ -105,6 +111,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase{// Same gate mid-conversation: thanks after landed work requests nothing.name: "thanks-is-gated",+ instant: null,bucket: "chat",input: () => [plainUserTurn("trim the first clip down to 5 seconds"),@@ -185,6 +192,9 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCasestubs: {set_transition: { ok: true },add_title: { ok: true },+ // "Short" is part of the ask, and a photo lands at the default still+ // length, so trimming the pair down is the movie being made.+ trim_clip: { in: 0, out: 3.5, len: 3.5 },},},{@@ -261,6 +271,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// to "make room" is wrong — the trap word "instead"/"version" is not a// delete instruction.name: "iterate-keeps-existing-clip",+ instant: null,bucket: "single-tool",input: () => [userTurn("the beach clip feels off — make me a better version, more of a golden-hour sunset"),@@ -296,6 +307,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// then narrates that as the answer. The snapshot names every clip and// its muted flag, so the mute needs no reads either.name: "mute-all-does-not-redo-prior-turn",+ instant: null,bucket: "single-tool",input: () => [plainUserTurn(@@ -410,6 +422,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase{// Fast path: a direct edit is one trim_clip on the clip in the snapshot.name: "trim-ask-single-tool",+ instant: null,bucket: "single-tool",input: () => [userTurn("trim the first clip down to 5 seconds")],reply: /trim|5|second/i,@@ -546,6 +559,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// "Pop" exists only as an animation, so the routing is unambiguous// (a wipe ask could legitimately become a wipe *transition* instead).name: "animation-overrides-transition",+ instant: null,bucket: "single-tool",input: () => [userTurn("get rid of the crossfade — make the second clip pop in instead", {@@ -774,6 +788,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// A described ramp is a curve, laid in one call: the shape the user// asked for lands as nodes over the clip's footage.name: "speed-ramp-from-description",+ instant: null,bucket: "single-tool",// A clean timeline: the debris cases own the parked bars, and a turn// that can see them is right to tidy them.@@ -927,6 +942,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// blacks" pushes shadows down (or deepens the low end of the master// curve) — the interceptor fails any move in the wrong direction.name: "warmer-crushed-blacks-in-range",+ instant: null,bucket: "single-tool",input: () => [userTurn("make the beach clip warmer and crush the blacks a little")],reply: /warm|black|shadow|grade/i,@@ -957,6 +973,7 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase// set_color_hsl on aqua or blue — a whole-frame saturation push leaks// onto everything else and fails.name: "sky-bluer-single-band",+ instant: "set_color_hsl",bucket: "single-tool",input: () => [userTurn("make the sky bluer without changing anything else")],reply: /sky|blue/i,diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.tsindex 98140477..3783afe4 100644--- a/site/scripts/lib/cut-eval/fixtures.ts+++ b/site/scripts/lib/cut-eval/fixtures.ts@@ -36,6 +36,9 @@ export const SAFE_TOOLS = new Set(["list_voices","seek","select",+ // Finding the items a description covers reads the timeline; the change+ // that follows is what edits.+ "select_items","set_playing","set_view",]);@@ -746,7 +749,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {}/** Serve a read-only tool from the fixture snapshot. */-export function serveSafeTool(name: string, state: unknown): unknown {+export function serveSafeTool(name: string, state: unknown, args: Record<string, unknown> = {}): unknown {if (name === "get_state") {// The fixture snapshot is frozen — it can't reflect this turn's stubbed// edits. Say so, or the model sees its adds "missing" and re-adds in a@@ -763,5 +766,21 @@ export function serveSafeTool(name: string, state: unknown): unknown {if (name === "find_filler") return { fillers: FILLER_WORDS, count: FILLER_WORDS.length, track: 0 };if (name === "measure_level") return { targetId: "", clips: [] };if (name === "detect_beats") return { bpm: 0, beats: [] };+ if (name === "select_items") {+ // The find half of a sweep, served off the fixture: every item of that+ // kind matches, so a case asserting the apply half gets the whole row.+ const s = (state ?? {}) as Record<string, unknown>;+ const kind = String(args.kind ?? "clip");+ const rows =+ kind === "audio" ? s.soundtrack+ : kind === "overlay" ? s.overlays+ : kind === "cue" ? ((s.subtitles ?? {}) as Record<string, unknown>).cues+ : kind === "transition" ? s.transitions+ : s.videoTrack;+ const ids = (Array.isArray(rows) ? rows : [])+ .map((r) => (r as { id?: unknown }).id)+ .filter((id): id is string => typeof id === "string");+ return { kind, searched: ids.length, ids, matched: ids.map((id) => ({ id })) };+ }return { ok: true };}diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.tsindex 6740b7c5..243c4cfa 100644--- a/site/scripts/lib/cut-eval/harness.ts+++ b/site/scripts/lib/cut-eval/harness.ts@@ -100,6 +100,8 @@ export interface CaseResult {routingMisses: string[];/** The model the rounds ran on — differs per turn under a router config. */roundModel: string;+ /** The action the instant path carried the turn out with, if any. */+ instant: string | null;timings: RunTimings;}@@ -125,6 +127,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>let reply = "";let streamError: string | null = null;let extensions = 0;+ let instant: string | null = null;const devPost =(path: string) =>@@ -168,7 +171,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>throw new Error(String((stub as { __error: unknown }).__error));return stub;}- if (SAFE_TOOLS.has(name)) return serveSafeTool(name, c.state ?? EDITOR_STATE);+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, c.state ?? EDITOR_STATE, args);violations.push(name);return { error: "eval: this tool is disabled for this turn — do not retry it" };} finally {@@ -187,6 +190,9 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>gateSkipped = skipped;route = { skill: r.skill, areas: r.areas, declaredTools: r.declaredTools };},+ onInstant: (action) => {+ instant = action?.id ?? null;+ },onRequestTools: (areas) => routingMisses.push(...areas),onRound: (ms, firstDeltaMs) => {modelMs += ms;@@ -237,6 +243,12 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>// "work" expectation.const gateSide = intent === "chat" ? "chat" : "work";if (c.gate && gateSide !== c.gate) notes.push(`gate said ${intent}, expected ${c.gate}`);+ // An instant turn is the whole point of the path: the judgment settled it+ // and no model round ran. A case that pins one asserts both.+ if (c.instant !== undefined) {+ if (instant !== c.instant) notes.push(`instant was ${instant ?? "none"}, expected ${c.instant ?? "none"}`);+ if (c.instant !== null && rounds.length > 0) notes.push(`instant turn still spent ${rounds.length} model round(s)`);+ }if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);for (const t of c.requiredTools ?? []) {if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);@@ -299,6 +311,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>areas: route?.areas ?? [],declaredTools: route?.declaredTools ?? 0,routingMisses,+ instant,roundModel: resolveGeminiModel(intent === "simple" ? cfg.simpleModel : cfg.complexModel),timings,};diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.tsindex 5ab5b724..68fe26b2 100644--- a/site/scripts/lib/cut-eval/report.ts+++ b/site/scripts/lib/cut-eval/report.ts@@ -26,6 +26,8 @@ export interface RunReport {/** The model the rounds ran on — differs per turn under a router config. */roundModel?: string;notes: string[];+ /** The registry action the turn ran with no model round, if any. */+ instant?: string | null;/** The voice/taste judge's complaint. Report-only: never part of pass. */judgeNote?: string | null;trace: TraceEntry[];@@ -107,6 +109,7 @@ export function toRunReport(result: CaseResult | null, error?: string): RunReporroutingMisses: result.routingMisses,roundModel: result.roundModel,notes: result.notes,+ instant: result.instant,judgeNote: result.judgeNote,trace: result.trace,timings: result.timings,