Open source CapCut alternative.
TypeScript
feat(site): chat models run on gemini 3.7 flash [rebuild]
David Lec4c22aa7dcd4
diff --git a/docs/guides/backend-apis.md b/docs/guides/backend-apis.mdindex 75191d6d..617ae784 100644--- a/docs/guides/backend-apis.md+++ b/docs/guides/backend-apis.md@@ -109,7 +109,7 @@ Model choice is code (see Handler Rules on `process.env`):| Call | Model ||---|---|| Gate-judged simple chat turns, fast structured decisions | `gemini-3.1-flash-lite` |-| General chat, non-decision structured calls | `gemini-3.5-flash` |+| General chat, non-decision structured calls | `gemini-3.7-flash` |Structured requests normalize their JSON schema for Gemini and retry without aprovider-enforced schema when Vertex rejects the schema parameters.diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.jsonindex 9e4b3267..7520b4f5 100644--- a/evals/cut-chat.latest-report.json+++ b/evals/cut-chat.latest-report.json@@ -1,82 +1,82 @@{"schema": "cut-chat-eval/v1",- "generatedAt": "2026-08-11T13:51:03.802Z",+ "generatedAt": "2026-08-14T11:00:02.814Z","base": "http://localhost:3000","runsPerCase": 3,"configs": [{"label": "baseline",- "chatModel": "gemini-3.1-flash-lite | complex\u2192gemini-3.5-flash",+ "chatModel": "gemini-3.1-flash-lite | complex\u2192gemini-3.7-flash","gateModel": "gemini-3.1-flash-lite","buckets": {- "single-tool": {- "cases": 16,- "passRate": 0.92,+ "chat": {+ "cases": 9,+ "passRate": 0.96,"gateMs": {- "p50": 1168,- "p95": 1961,- "mean": 1236,- "n": 16+ "p50": 1088,+ "p95": 1794,+ "mean": 1031,+ "n": 9},"ttftMs": {- "p50": 5130,- "p95": 14917,- "mean": 7137,- "n": 16+ "p50": 3796,+ "p95": 5928,+ "mean": 3622,+ "n": 9},"totalMs": {- "p50": 5873,- "p95": 15188,- "mean": 7559,- "n": 16+ "p50": 4139,+ "p95": 6189,+ "mean": 3807,+ "n": 9},- "meanRounds": 2.0+ "meanRounds": 1.7},"multi-tool": {"cases": 8,- "passRate": 0.92,+ "passRate": 1,"gateMs": {- "p50": 1018,- "p95": 1907,- "mean": 1062,+ "p50": 1061,+ "p95": 2014,+ "mean": 941,"n": 8},"ttftMs": {- "p50": 7329,- "p95": 66488,- "mean": 19330,+ "p50": 5380,+ "p95": 53553,+ "mean": 15775,"n": 8},"totalMs": {- "p50": 8045,- "p95": 67010,- "mean": 19724,+ "p50": 5767,+ "p95": 53903,+ "mean": 16060,"n": 8},- "meanRounds": 5.9+ "meanRounds": 5.2},- "chat": {- "cases": 9,- "passRate": 1.0,+ "single-tool": {+ "cases": 16,+ "passRate": 1,"gateMs": {- "p50": 1266,- "p95": 1500,- "mean": 993,- "n": 9+ "p50": 1125,+ "p95": 1881,+ "mean": 1174,+ "n": 16},"ttftMs": {- "p50": 4385,- "p95": 8949,- "mean": 4740,- "n": 9+ "p50": 3291,+ "p95": 16261,+ "mean": 6018,+ "n": 16},"totalMs": {- "p50": 4390,- "p95": 9794,- "mean": 5030,- "n": 9+ "p50": 4081,+ "p95": 16492,+ "mean": 6354,+ "n": 16},- "meanRounds": 1.0+ "meanRounds": 2.5}},"cases": [@@ -93,21 +93,21 @@"n": 3},"ttftMs": {- "p50": 4577,- "p95": 9515,- "mean": 6084,+ "p50": 3545,+ "p95": 5295,+ "mean": 4055,"n": 3},"totalMs": {- "p50": 4675,- "p95": 9683,- "mean": 6235,+ "p50": 3661,+ "p95": 5391,+ "mean": 4221,"n": 3},"modelMs": {- "p50": 4675,- "p95": 9682,- "mean": 6235,+ "p50": 3658,+ "p95": 5383,+ "mean": 4216,"n": 3},"toolMs": {@@ -125,24 +125,24 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 0.6382089999999607,+ "gateMs": 0.6801250000000891,"gateSkipped": true,- "ttftMs": 9515.249,- "modelMs": 9682.239625,+ "ttftMs": 5295.283375,+ "modelMs": 5382.8430419999995,"toolMs": 0,- "totalMs": 9682.877834,- "judgeMs": 2070.2317079999993,+ "totalMs": 5391.090625,+ "judgeMs": 1283.3167080000003,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 9682.239625,- "firstDeltaMs": 9509.159167,+ "roundMs": 5382.8430419999995,+ "firstDeltaMs": 5287.6332919999995,"calls": []}]@@ -151,24 +151,24 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 0.16216699999858974,+ "gateMs": 0.3914169999998194,"gateSkipped": true,- "ttftMs": 4577.185791999998,- "modelMs": 4674.673832999999,+ "ttftMs": 3544.728167000001,+ "modelMs": 3657.7012909999994,"toolMs": 0,- "totalMs": 4674.8359999999975,- "judgeMs": 1124.0844169999982,+ "totalMs": 3661.4497919999994,+ "judgeMs": 925.4969999999994,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4674.673832999999,- "firstDeltaMs": 4575.544040999999,+ "roundMs": 3657.7012909999994,+ "firstDeltaMs": 3541.2325409999994,"calls": []}]@@ -177,24 +177,24 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 0.21391600000060862,+ "gateMs": 0.17783300000155577,"gateSkipped": true,- "ttftMs": 4160.707750000001,- "modelMs": 4347.5510829999985,+ "ttftMs": 3325.0255830000006,+ "modelMs": 3607.0003749999996,"toolMs": 0,- "totalMs": 4347.764998999999,- "judgeMs": 1307.9249579999996,+ "totalMs": 3609.242291999999,+ "judgeMs": 1483.7433330000003,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4347.5510829999985,- "firstDeltaMs": 4158.627499999999,+ "roundMs": 3607.0003749999996,+ "firstDeltaMs": 3323.1571669999994,"calls": []}]@@ -209,27 +209,27 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1315,- "p95": 1584,- "mean": 1358,+ "p50": 1246,+ "p95": 1734,+ "mean": 1331,"n": 3},"ttftMs": {- "p50": 2894,- "p95": 3383,- "mean": 3053,+ "p50": 3572,+ "p95": 3682,+ "mean": 3463,"n": 3},"totalMs": {- "p50": 2910,- "p95": 3398,- "mean": 3063,+ "p50": 3572,+ "p95": 3782,+ "mean": 3500,"n": 3},"modelMs": {- "p50": 1706,- "p95": 1814,- "mean": 1705,+ "p50": 3569,+ "p95": 3773,+ "mean": 3495,"n": 3},"toolMs": {@@ -239,32 +239,37 @@"n": 3},"rounds": {- "mean": 1,- "max": 1+ "mean": 2,+ "max": 2}},"runs": [{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 1315.435542000003,+ "gateMs": 1733.6028330000008,"gateSkipped": false,- "ttftMs": 2893.8250000000007,- "modelMs": 1594.2496249999967,+ "ttftMs": 3681.6503749999974,+ "modelMs": 3773.088208000001,"toolMs": 0,- "totalMs": 2909.6851669999996,- "judgeMs": 2071.457875,+ "totalMs": 3782.118624999999,+ "judgeMs": 1090.0546250000007,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1594.2496249999967,- "firstDeltaMs": 1574.7519999999968,+ "roundMs": 1731.358916000001,+ "firstDeltaMs": 1675.356208000001,+ "calls": []+ },+ {+ "roundMs": 2041.729292,+ "firstDeltaMs": 1941.7217500000006,"calls": []}]@@ -273,24 +278,29 @@{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 1583.569958,+ "gateMs": 1246.2363330000007,"gateSkipped": false,- "ttftMs": 3382.8357499999984,- "modelMs": 1814.494584,+ "ttftMs": 3571.9032499999994,+ "modelMs": 3569.397124999996,"toolMs": 0,- "totalMs": 3398.064542,- "judgeMs": 1060.0287499999977,+ "totalMs": 3572.304959000001,+ "judgeMs": 949.5440000000017,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1814.494584,- "firstDeltaMs": 1797.901167,+ "roundMs": 1245.7104169999984,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 2323.6867079999975,+ "firstDeltaMs": 2323.3776249999974,"calls": []}]@@ -299,24 +309,29 @@{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 1174.5241660000029,+ "gateMs": 1013.8146249999991,"gateSkipped": false,- "ttftMs": 2880.972833,- "modelMs": 1705.5139169999966,+ "ttftMs": 3134.808499999999,+ "modelMs": 3141.4495419999985,"toolMs": 0,- "totalMs": 2880.0380829999995,- "judgeMs": 1119.4047910000008,+ "totalMs": 3144.746292,+ "judgeMs": 1055.9203339999985,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1705.5139169999966,- "firstDeltaMs": 1705.1563339999993,+ "roundMs": 1012.1462920000013,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 2129.303249999997,+ "firstDeltaMs": 2119.4951249999976,"calls": []}]@@ -331,27 +346,27 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 982,- "p95": 1612,- "mean": 1170,+ "p50": 1625,+ "p95": 1813,+ "mean": 1497,"n": 3},"ttftMs": {- "p50": 2839,- "p95": 3329,- "mean": 2905,+ "p50": 4437,+ "p95": 6880,+ "mean": 4935,"n": 3},"totalMs": {- "p50": 2838,- "p95": 3615,- "mean": 3006,+ "p50": 4439,+ "p95": 7008,+ "mean": 4985,"n": 3},"modelMs": {- "p50": 1922,- "p95": 2003,- "mean": 1836,+ "p50": 4003,+ "p95": 6851,+ "mean": 4786,"n": 3},"toolMs": {@@ -361,32 +376,42 @@"n": 3},"rounds": {- "mean": 1,- "max": 1+ "mean": 2.7,+ "max": 3}},"runs": [{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 981.7013339999976,+ "gateMs": 1625.3346669999992,"gateSkipped": false,- "ttftMs": 2548.5990419999944,- "modelMs": 1582.0289170000033,+ "ttftMs": 6879.611375,+ "modelMs": 6850.584874999997,"toolMs": 0,- "totalMs": 2563.730251000001,- "judgeMs": 1223.5903749999998,+ "totalMs": 7008.350666999999,+ "judgeMs": 1155.4799579999963,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1582.0289170000033,- "firstDeltaMs": 1563.7441250000047,+ "roundMs": 1470.3922920000005,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 0.019791999999142718,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 5380.172790999997,+ "firstDeltaMs": 5251.712541000001,"calls": []}]@@ -395,24 +420,29 @@{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],- "judgeNote": "overly robotic response to a social pleasantry",+ "judgeNote": null,"trace": [],"timings": {- "gateMs": 1612.1080409999995,+ "gateMs": 1051.4393750000017,"gateSkipped": false,- "ttftMs": 3329.329791999997,- "modelMs": 2002.5194590000028,+ "ttftMs": 3488.972542000003,+ "modelMs": 3505.183582999991,"toolMs": 0,- "totalMs": 3614.6275000000023,- "judgeMs": 2260.5377089999965,+ "totalMs": 3508.2672500000044,+ "judgeMs": 972.8683750000055,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2002.5194590000028,- "firstDeltaMs": 1715.645709000004,+ "roundMs": 1050.3509579999954,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 2454.8326249999955,+ "firstDeltaMs": 2435.768707999996,"calls": []}]@@ -421,24 +451,34 @@{"pass": true,"intent": "chat",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 916.338499999998,+ "gateMs": 1812.8156249999956,"gateSkipped": false,- "ttftMs": 2838.568833000005,- "modelMs": 1922.1279589999976,+ "ttftMs": 4436.651000000005,+ "modelMs": 4002.911376000011,"toolMs": 0,- "totalMs": 2838.4664589999957,- "judgeMs": 1691.0353329999998,+ "totalMs": 4439.214458000002,+ "judgeMs": 750.3692499999961,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1922.1279589999976,- "firstDeltaMs": 1921.8951249999955,+ "roundMs": 1380.1530420000054,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 0.03908400000364054,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 2622.719250000002,+ "firstDeltaMs": 2620.609209000002,"calls": []}]@@ -447,44 +487,44 @@]},{- "name": "question-answers-in-chat",- "bucket": "chat",+ "name": "follow-up-do-it-works",+ "bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1500,- "p95": 1584,- "mean": 1435,+ "p50": 1006,+ "p95": 1111,+ "mean": 1017,"n": 3},"ttftMs": {- "p50": 3643,- "p95": 6706,- "mean": 4280,+ "p50": 3064,+ "p95": 6813,+ "mean": 4249,"n": 3},"totalMs": {- "p50": 4262,- "p95": 6907,- "mean": 4653,+ "p50": 3341,+ "p95": 7105,+ "mean": 4529,"n": 3},"modelMs": {- "p50": 2678,- "p95": 5407,- "mean": 3217,+ "p50": 3337,+ "p95": 7089,+ "mean": 4520,"n": 3},"toolMs": {"p50": 0,- "p95": 0,+ "p95": 1,"mean": 0,"n": 3},"rounds": {- "mean": 1,- "max": 1+ "mean": 2,+ "max": 2}},"runs": [@@ -494,21 +534,68 @@"roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,- "trace": [],+ "trace": [+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 5375.111749999996,+ "execMs": 0.7929579999981797+ },+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 5377.615957999995,+ "execMs": 0.01750000000174623+ },+ {+ "name": "delete_cue",+ "round": 0,+ "atMs": 5378.9735,+ "execMs": 0.05762499999400461+ },+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 5379.2072499999995,+ "execMs": 0.007291000001714565+ }+ ],"timings": {- "gateMs": 1499.9039579999953,+ "gateMs": 1006.0852919999961,"gateSkipped": false,- "ttftMs": 6705.881500000003,- "modelMs": 5407.145083999996,- "toolMs": 0,- "totalMs": 6907.0490419999915,- "judgeMs": 2238.7736670000013,+ "ttftMs": 6813.174874999997,+ "modelMs": 7088.589292000004,+ "toolMs": 0.8753739999956451,+ "totalMs": 7104.922332999995,+ "judgeMs": 1164.2447089999987,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5407.145083999996,- "firstDeltaMs": 5204.016041999996,+ "roundMs": 5365.451042000001,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "update_cue",+ "execMs": 0.7929579999981797+ },+ {+ "name": "update_cue",+ "execMs": 0.01750000000174623+ },+ {+ "name": "delete_cue",+ "execMs": 0.05762499999400461+ },+ {+ "name": "update_cue",+ "execMs": 0.007291000001714565+ }+ ]+ },+ {+ "roundMs": 1723.1382500000036,+ "firstDeltaMs": 1432.1299169999984,"calls": []}]@@ -520,21 +607,48 @@"roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,- "trace": [],+ "trace": [+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 1973.907124999998,+ "execMs": 0.011249999995925464+ },+ {+ "name": "delete_cue",+ "round": 0,+ "atMs": 1974.1132500000022,+ "execMs": 0.017166999998153187+ }+ ],"timings": {- "gateMs": 1221.9570000000022,+ "gateMs": 933.0407500000001,"gateSkipped": false,- "ttftMs": 2490.9581670000043,- "modelMs": 1566.588917000001,- "toolMs": 0,- "totalMs": 2788.545917000003,- "judgeMs": 1711.0802499999918,+ "ttftMs": 3063.574959000005,+ "modelMs": 3336.548625000003,+ "toolMs": 0.02841699999407865,+ "totalMs": 3340.5978749999995,+ "judgeMs": 1044.6940419999955,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1566.588917000001,- "firstDeltaMs": 1267.5192090000055,+ "roundMs": 1971.3986250000016,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "update_cue",+ "execMs": 0.011249999995925464+ },+ {+ "name": "delete_cue",+ "execMs": 0.017166999998153187+ }+ ]+ },+ {+ "roundMs": 1365.1500000000015,+ "firstDeltaMs": 1088.4904580000002,"calls": []}]@@ -546,21 +660,68 @@"roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,- "trace": [],+ "trace": [+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 1788.0434170000008,+ "execMs": 0.017584000001079403+ },+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 1788.5724580000024,+ "execMs": 0.018167000001994893+ },+ {+ "name": "delete_cue",+ "round": 0,+ "atMs": 1788.833292000003,+ "execMs": 0.0171249999984866+ },+ {+ "name": "update_cue",+ "round": 0,+ "atMs": 1789.0872080000045,+ "execMs": 0.016290999999910127+ }+ ],"timings": {- "gateMs": 1584.239333999998,+ "gateMs": 1111.4699170000022,"gateSkipped": false,- "ttftMs": 3642.686916999999,- "modelMs": 2678.1887089999946,- "toolMs": 0,- "totalMs": 4262.428042999993,- "judgeMs": 1256.590584000005,+ "ttftMs": 2871.0125830000034,+ "modelMs": 3133.879873999991,+ "toolMs": 0.06916700000147102,+ "totalMs": 3140.0463750000054,+ "judgeMs": 2061.4119169999976,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2678.1887089999946,- "firstDeltaMs": 2056.337625,+ "roundMs": 1784.9065409999967,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "update_cue",+ "execMs": 0.017584000001079403+ },+ {+ "name": "update_cue",+ "execMs": 0.018167000001994893+ },+ {+ "name": "delete_cue",+ "execMs": 0.0171249999984866+ },+ {+ "name": "update_cue",+ "execMs": 0.016290999999910127+ }+ ]+ },+ {+ "roundMs": 1348.9733329999945,+ "firstDeltaMs": 1080.4344159999964,"calls": []}]@@ -569,33 +730,33 @@]},{- "name": "photos-question-stays-in-chat",+ "name": "question-answers-in-chat","bucket": "chat","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 0,- "p95": 1,- "mean": 0,+ "p50": 1088,+ "p95": 1306,+ "mean": 1102,"n": 3},"ttftMs": {- "p50": 6838,- "p95": 10478,- "mean": 7730,+ "p50": 1261,+ "p95": 1421,+ "mean": 1286,"n": 3},"totalMs": {- "p50": 7204,- "p95": 11692,- "mean": 8649,+ "p50": 1385,+ "p95": 1828,+ "mean": 1514,"n": 3},"modelMs": {- "p50": 7204,- "p95": 11692,- "mean": 8649,+ "p50": 1383,+ "p95": 1827,+ "mean": 1512,"n": 3},"toolMs": {@@ -605,32 +766,32 @@"n": 3},"rounds": {- "mean": 1.3,- "max": 2+ "mean": 1,+ "max": 1}},"runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": "excessive filler, use of bullet points, and marketing-style suggestions instead of being concise.",+ "judgeNote": null,"trace": [],"timings": {- "gateMs": 0.701375000004191,- "gateSkipped": true,- "ttftMs": 5874.439958999996,- "modelMs": 7050.758457999997,+ "gateMs": 1306.2425829999993,+ "gateSkipped": false,+ "ttftMs": 1420.9730419999978,+ "modelMs": 1826.5942499999946,"toolMs": 0,- "totalMs": 7051.459833000001,- "judgeMs": 1445.1213750000024,+ "totalMs": 1827.8963749999966,+ "judgeMs": 1049.1808750000055,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 7050.758457999997,- "firstDeltaMs": 5871.849875,+ "roundMs": 1826.5942499999946,+ "firstDeltaMs": 1420.0688749999972,"calls": []}]@@ -638,25 +799,25 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [],"timings": {- "gateMs": 0.08950000000186265,- "gateSkipped": true,- "ttftMs": 6838.067708000002,- "modelMs": 7203.645833000002,+ "gateMs": 1088.3530829999945,+ "gateSkipped": false,+ "ttftMs": 1176.4475419999944,+ "modelMs": 1326.5217919999996,"toolMs": 0,- "totalMs": 7203.735333000004,- "judgeMs": 1705.49229200001,+ "totalMs": 1329.1205419999897,+ "judgeMs": 1653.0858749999898,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 7203.645833000002,- "firstDeltaMs": 6837.30133300001,+ "roundMs": 1326.5217919999996,+ "firstDeltaMs": 1174.1965420000051,"calls": []}]@@ -664,52 +825,25 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": "contains upsell (music) and uses filler formatting.",- "trace": [- {- "name": "watch_video",- "round": 0,- "atMs": 6834.3042499999865,- "execMs": 0.030624999999417923- },- {- "name": "watch_video",- "round": 0,- "atMs": 6834.906041999988,- "execMs": 0.001416999992216006- }- ],+ "judgeNote": null,+ "trace": [],"timings": {- "gateMs": 0.30220800000824966,- "gateSkipped": true,- "ttftMs": 10478.143666999997,- "modelMs": 11691.9525,- "toolMs": 0.03204199999163393,- "totalMs": 11692.28675,- "judgeMs": 1305.2327920000098,+ "gateMs": 910.9697080000042,+ "gateSkipped": false,+ "ttftMs": 1261.1302080000023,+ "modelMs": 1382.7398749999993,+ "toolMs": 0,+ "totalMs": 1385.3621660000063,+ "judgeMs": 2083.4081660000084,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6822.634416999994,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.030624999999417923- },- {- "name": "watch_video",- "execMs": 0.001416999992216006- }- ]- },- {- "roundMs": 4869.318083000006,- "firstDeltaMs": 3642.571458000006,+ "roundMs": 1382.7398749999993,+ "firstDeltaMs": 1258.820000000007,"calls": []}]@@ -718,33 +852,33 @@]},{- "name": "clutter-complaint-deletes-nothing",- "bucket": "chat",+ "name": "photos-request-places-clips",+ "bucket": "multi-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1266,- "p95": 1926,- "mean": 1480,+ "p50": 0,+ "p95": 0,+ "mean": 0,"n": 3},"ttftMs": {- "p50": 5197,- "p95": 9159,- "mean": 6068,+ "p50": 25756,+ "p95": 26357,+ "mean": 25544,"n": 3},"totalMs": {- "p50": 5455,- "p95": 9389,- "mean": 6289,+ "p50": 25800,+ "p95": 26524,+ "mean": 25615,"n": 3},"modelMs": {- "p50": 4189,- "p95": 7463,- "mean": 4808,+ "p50": 25787,+ "p95": 26507,+ "mean": 25600,"n": 3},"toolMs": {@@ -754,966 +888,256 @@"n": 3},"rounds": {- "mean": 1,- "max": 1+ "mean": 6.7,+ "max": 7}},"runs": [{"pass": true,- "intent": "chat",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1249.0194999999949,- "gateSkipped": false,- "ttftMs": 3847.3752499999973,- "modelMs": 2772.835208000004,- "toolMs": 0,- "totalMs": 4021.854707999999,- "judgeMs": 1100.5716249999969,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2772.835208000004,- "firstDeltaMs": 2595.9299579999933,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "chat",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1265.765665999992,- "gateSkipped": false,- "ttftMs": 5196.987957999998,- "modelMs": 4188.830541000003,- "toolMs": 0,- "totalMs": 5454.596206999995,- "judgeMs": 891.1059590000077,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 4188.830541000003,- "firstDeltaMs": 3929.7065000000002,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "chat",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1925.9489170000015,- "gateSkipped": false,- "ttftMs": 9158.980416000006,- "modelMs": 7463.317750000002,- "toolMs": 0,- "totalMs": 9389.266667000004,- "judgeMs": 1119.002416999996,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 7463.317750000002,- "firstDeltaMs": 7231.192625000011,- "calls": []- }- ]- }- }- ]- },- {- "name": "cartoon-question-stays-in-chat",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1340,- "p95": 1594,- "mean": 1289,- "n": 3- },- "ttftMs": {- "p50": 8949,- "p95": 11302,- "mean": 9415,- "n": 3- },- "totalMs": {- "p50": 9794,- "p95": 11667,- "mean": 10024,- "n": 3- },- "modelMs": {- "p50": 8200,- "p95": 10734,- "mean": 8735,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 1,- "max": 1- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1339.6451249999955,- "gateSkipped": false,- "ttftMs": 7994.283500000005,- "modelMs": 7271.1564160000125,- "toolMs": 0,- "totalMs": 8610.801541000008,- "judgeMs": 1219.9470420000143,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 7271.1564160000125,- "firstDeltaMs": 6654.061916000006,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 933.1749160000181,- "gateSkipped": false,- "ttftMs": 11301.68291599999,- "modelMs": 10733.860083000007,- "toolMs": 0,- "totalMs": 11667.034999000025,- "judgeMs": 1743.929375000007,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 10733.860083000007,- "firstDeltaMs": 10367.081417000009,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1594.2282500000147,- "gateSkipped": false,- "ttftMs": 8949.262208,- "modelMs": 8199.745999999985,- "toolMs": 0,- "totalMs": 9793.97425,- "judgeMs": 1565.0725410000014,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 8199.745999999985,- "firstDeltaMs": 7353.404749999987,- "calls": []- }- ]- }- }- ]- },- {- "name": "question-when-ambiguous",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1229,- "p95": 1616,- "mean": 1301,- "n": 3- },- "ttftMs": {- "p50": 4385,- "p95": 4694,- "mean": 4322,- "n": 3- },- "totalMs": {- "p50": 4390,- "p95": 4979,- "mean": 4497,- "n": 3- },- "modelMs": {- "p50": 3161,- "p95": 3363,- "mean": 3196,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 1,- "max": 1- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1229.3556250000256,- "gateSkipped": false,- "ttftMs": 4384.771666999994,- "modelMs": 3160.650291000027,- "toolMs": 0,- "totalMs": 4390.005916000053,- "judgeMs": 1045.922499999986,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3160.650291000027,- "firstDeltaMs": 3152.9439160000184,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1058.3992919999873,- "gateSkipped": false,- "ttftMs": 3886.625792000006,- "modelMs": 3065.247707999981,- "toolMs": 0,- "totalMs": 4123.646999999968,- "judgeMs": 2137.7861250000133,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3065.247707999981,- "firstDeltaMs": 2826.716374999989,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1615.6097080000036,- "gateSkipped": false,- "ttftMs": 4694.452707999997,- "modelMs": 3363.0867499999877,- "toolMs": 0,- "totalMs": 4978.696457999991,- "judgeMs": 1115.4267080000136,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3363.0867499999877,- "firstDeltaMs": 3076.8358339999977,- "calls": []- }- ]- }- }- ]- },- {- "name": "legacy-thread-no-leak",- "bucket": "chat",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1307,- "p95": 1821,- "mean": 1462,- "n": 3- },- "ttftMs": {- "p50": 3339,- "p95": 5836,- "mean": 3940,- "n": 3- },- "totalMs": {- "p50": 3744,- "p95": 6461,- "mean": 4499,- "n": 3- },- "modelMs": {- "p50": 2033,- "p95": 5154,- "mean": 3037,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 1,- "max": 1- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1306.5508339999942,- "gateSkipped": false,- "ttftMs": 5835.870916999993,- "modelMs": 5154.352416999987,- "toolMs": 0,- "totalMs": 6460.9032509999815,- "judgeMs": 1403.1368749999965,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 5154.352416999987,- "firstDeltaMs": 4526.572374999989,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1259.9091250000056,- "gateSkipped": false,- "ttftMs": 2646.8406670000113,- "modelMs": 2032.8448750000098,- "toolMs": 0,- "totalMs": 3292.7540000000154,- "judgeMs": 1056.1719589999993,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2032.8448750000098,- "firstDeltaMs": 1384.6364580000227,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [],- "timings": {- "gateMs": 1820.6741250000196,- "gateSkipped": false,- "ttftMs": 3338.5416250000126,- "modelMs": 1923.3888330000045,- "toolMs": 0,- "totalMs": 3744.062958000024,- "judgeMs": 1033.1144170000043,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1923.3888330000045,- "firstDeltaMs": 1516.050166000001,- "calls": []- }- ]- }- }- ]- },- {- "name": "video-ask-single-generate",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1693,- "p95": 2189,- "mean": 1692,- "n": 3- },- "ttftMs": {- "p50": 5332,- "p95": 6072,- "mean": 5033,- "n": 3- },- "totalMs": {- "p50": 6053,- "p95": 6328,- "mean": 5466,- "n": 3- },- "modelMs": {- "p50": 4139,- "p95": 4361,- "mean": 3774,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "generate_video",+ "name": "add_clip","round": 0,- "atMs": 4639.7805,- "execMs": 0.004582999999911408- }- ],- "timings": {- "gateMs": 2188.5847080000003,- "gateSkipped": false,- "ttftMs": 6072.40675,- "modelMs": 4139.222292,- "toolMs": 0.004582999999911408,- "totalMs": 6327.811583000001,- "judgeMs": 910.561917,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 2439.952584,- "firstDeltaMs": null,- "calls": [- {- "name": "generate_video",- "execMs": 0.004582999999911408- }- ]- },- {- "roundMs": 1699.2697079999998,- "firstDeltaMs": 1431.038458,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [+ "atMs": 5606.476874999993,+ "execMs": 0.009625000006053597+ },+ {+ "name": "add_clip",+ "round": 0,+ "atMs": 5606.878375,+ "execMs": 0.0018330000020796433+ },+ {+ "name": "get_state",+ "round": 1,+ "atMs": 8406.270332999993,+ "execMs": 0.10975000000325963+ },+ {+ "name": "add_clip",+ "round": 2,+ "atMs": 11743.913375000004,+ "execMs": 0.004458000010345131+ },{- "name": "generate_video",- "round": 0,- "atMs": 3784.281,- "execMs": 0.0013330000001587905+ "name": "get_state",+ "round": 3,+ "atMs": 15570.833249999996,+ "execMs": 0.02425000000221189+ },+ {+ "name": "add_clip",+ "round": 4,+ "atMs": 17930.058791999996,+ "execMs": 0.003790999995544553}],"timings": {- "gateMs": 1692.5868330000003,- "gateSkipped": false,- "ttftMs": 5331.679416,- "modelMs": 4360.817916,- "toolMs": 0.0013330000001587905,- "totalMs": 6053.406082,- "judgeMs": 1652.759583000001,+ "gateMs": 0.20495800000207964,+ "gateSkipped": true,+ "ttftMs": 25756.029542000004,+ "modelMs": 25786.876457999984,+ "toolMs": 0.15370700001949444,+ "totalMs": 25800.268375,+ "judgeMs": 1172.552582999997,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2088.7971660000003,+ "roundMs": 5602.695707999999,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.0013330000001587905+ "name": "add_clip",+ "execMs": 0.009625000006053597+ },+ {+ "name": "add_clip",+ "execMs": 0.0018330000020796433}]},{- "roundMs": 2272.0207499999997,- "firstDeltaMs": 1547.0587919999998,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": "missing approval gate for credit-consuming action",- "trace": [- {- "name": "generate_video",- "round": 0,- "atMs": 2503.4117910000004,- "execMs": 0.005292000001645647- }- ],- "timings": {- "gateMs": 1194.3320829999993,- "gateSkipped": false,- "ttftMs": 3695.109290999999,- "modelMs": 2822.2551239999993,- "toolMs": 0.005292000001645647,- "totalMs": 4016.5924990000003,- "judgeMs": 1282.6564169999983,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 1303.0563330000004,+ "roundMs": 2796.879833999992,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.005292000001645647+ "name": "get_state",+ "execMs": 0.10975000000325963}]},{- "roundMs": 1519.198790999999,- "firstDeltaMs": 1190.5308750000004,- "calls": []- }- ]- }- }- ]- },- {- "name": "iterate-keeps-existing-clip",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1404,- "p95": 1557,- "mean": 1347,- "n": 3- },- "ttftMs": {- "p50": 9729,- "p95": 12070,- "mean": 10089,- "n": 3- },- "totalMs": {- "p50": 9887,- "p95": 12344,- "mean": 10372,- "n": 3- },- "modelMs": {- "p50": 8330,- "p95": 10940,- "mean": 9025,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": "did not state the approval gate for credit expenditure.",- "trace": [- {- "name": "generate_video",- "round": 0,- "atMs": 9959.234,- "execMs": 0.017166999999972177- }- ],- "timings": {- "gateMs": 1404.417875,- "gateSkipped": false,- "ttftMs": 12069.571916,- "modelMs": 10939.925082999998,- "toolMs": 0.017166999999972177,- "totalMs": 12344.360124999997,- "judgeMs": 2059.193125,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 8541.640167,+ "roundMs": 3335.8724999999977,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.017166999999972177+ "name": "add_clip",+ "execMs": 0.004458000010345131}]},{- "roundMs": 2398.2849159999987,- "firstDeltaMs": 2109.3704579999994,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": "missing mandatory approval gate for spending credits/rendering changes.",- "trace": [- {- "name": "generate_video",- "round": 0,- "atMs": 7588.552333000001,- "execMs": 0.0023330000003625173- }- ],- "timings": {- "gateMs": 1556.5739580000009,- "gateSkipped": false,- "ttftMs": 9729.262499999999,- "modelMs": 8329.984250000001,- "toolMs": 0.0023330000003625173,- "totalMs": 9886.560541000003,- "judgeMs": 1033.645124999999,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 6031.121999999999,+ "roundMs": 3825.2406249999913,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.0023330000003625173+ "name": "get_state",+ "execMs": 0.02425000000221189}]},{- "roundMs": 2298.862250000002,- "firstDeltaMs": 2140.067041000002,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "generate_video",- "round": 0,- "atMs": 6088.003375,- "execMs": 0.00429200000144192- }- ],- "timings": {- "gateMs": 1078.6309169999986,- "gateSkipped": false,- "ttftMs": 8467.358457999999,- "modelMs": 7805.817917,- "toolMs": 0.00429200000144192,- "totalMs": 8884.453126,- "judgeMs": 1249.2672919999968,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 5007.172917,+ "roundMs": 2357.2911659999954,"firstDeltaMs": null,"calls": [{- "name": "generate_video",- "execMs": 0.00429200000144192+ "name": "add_clip",+ "execMs": 0.003790999995544553}]},{- "roundMs": 2798.6450000000004,- "firstDeltaMs": 2378.7929170000025,+ "roundMs": 7868.8966250000085,+ "firstDeltaMs": 7825.063083000001,"calls": []}]}- }- ]- },- {- "name": "mute-all-does-not-redo-prior-turn",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1164,- "p95": 1172,- "mean": 1145,- "n": 3- },- "ttftMs": {- "p50": 5479,- "p95": 7867,- "mean": 5892,- "n": 3- },- "totalMs": {- "p50": 6275,- "p95": 8043,- "mean": 6334,- "n": 3},- "modelMs": {- "p50": 5111,- "p95": 6871,- "mean": 5189,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],- "judgeNote": "inaccuracy (the assistant claimed to mute \"city clips\" when the user requested \"all clips\")",+ "judgeNote": null,"trace": [{- "name": "set_clip_muted",+ "name": "add_clip","round": 0,- "atMs": 6596.596708999998,- "execMs": 0.08583299999736482+ "atMs": 5480.511957999988,+ "execMs": 0.004040999992867},{- "name": "set_clip_muted",+ "name": "add_clip","round": 0,- "atMs": 6597.089333999997,- "execMs": 0.0033750000002328306+ "atMs": 5480.807499999995,+ "execMs": 0.0018330000020796433},{- "name": "set_clip_muted",- "round": 0,- "atMs": 6597.287292000001,- "execMs": 0.0027919999993173406+ "name": "get_state",+ "round": 1,+ "atMs": 9978.928874999998,+ "execMs": 0.0342499999969732+ },+ {+ "name": "add_clip",+ "round": 2,+ "atMs": 13573.690332999991,+ "execMs": 0.005917000002227724+ },+ {+ "name": "add_clip",+ "round": 3,+ "atMs": 16911.97591699999,+ "execMs": 0.0052500000019790605+ },+ {+ "name": "get_state",+ "round": 4,+ "atMs": 20580.118208,+ "execMs": 0.012166999993496574+ },+ {+ "name": "add_clip",+ "round": 5,+ "atMs": 24268.81745799999,+ "execMs": 0.005374999993364327}],"timings": {- "gateMs": 1171.8670409999977,- "gateSkipped": false,- "ttftMs": 7866.677334,- "modelMs": 6871.4575419999965,- "toolMs": 0.091999999996915,- "totalMs": 8043.416582999991,- "judgeMs": 1091.9065000000046,+ "gateMs": 0.1699159999989206,+ "gateSkipped": true,+ "ttftMs": 26357.05308299999,+ "modelMs": 26507.328874000013,+ "toolMs": 0.06883299998298753,+ "totalMs": 26523.933332999994,+ "judgeMs": 1062.2252500000031,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5417.422166999997,+ "roundMs": 5478.818249999997,"firstDeltaMs": null,"calls": [{- "name": "set_clip_muted",- "execMs": 0.08583299999736482+ "name": "add_clip",+ "execMs": 0.004040999992867},{- "name": "set_clip_muted",- "execMs": 0.0033750000002328306- },+ "name": "add_clip",+ "execMs": 0.0018330000020796433+ }+ ]+ },+ {+ "roundMs": 4496.467917000002,+ "firstDeltaMs": null,+ "calls": [{- "name": "set_clip_muted",- "execMs": 0.0027919999993173406+ "name": "get_state",+ "execMs": 0.0342499999969732}]},{- "roundMs": 1454.0353749999995,- "firstDeltaMs": 1267.5579579999976,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "set_clip_muted",- "round": 0,- "atMs": 3916.6915829999925,- "execMs": 0.008084000000962988- },- {- "name": "set_clip_muted",- "round": 0,- "atMs": 3917.0253329999978,- "execMs": 0.0025830000085989013- },- {- "name": "set_clip_muted",- "round": 0,- "atMs": 3917.165540999995,- "execMs": 0.0014159999991534278- }- ],- "timings": {- "gateMs": 1163.6107920000068,- "gateSkipped": false,- "ttftMs": 5478.921665999995,- "modelMs": 5111.251833999995,- "toolMs": 0.012083000008715317,- "totalMs": 6274.874709000011,- "judgeMs": 1545.0772089999955,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 3592.3726250000036,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "add_clip",+ "execMs": 0.005917000002227724+ }+ ]+ },{- "roundMs": 2751.0811669999966,+ "roundMs": 3335.5647080000053,"firstDeltaMs": null,"calls": [{- "name": "set_clip_muted",- "execMs": 0.008084000000962988- },+ "name": "add_clip",+ "execMs": 0.0052500000019790605+ }+ ]+ },+ {+ "roundMs": 3666.1145830000023,+ "firstDeltaMs": null,+ "calls": [{- "name": "set_clip_muted",- "execMs": 0.0025830000085989013- },+ "name": "get_state",+ "execMs": 0.012166999993496574+ }+ ]+ },+ {+ "roundMs": 3685.969916000002,+ "firstDeltaMs": null,+ "calls": [{- "name": "set_clip_muted",- "execMs": 0.0014159999991534278+ "name": "add_clip",+ "execMs": 0.005374999993364327}]},{- "roundMs": 2360.1706669999985,- "firstDeltaMs": 1557.9960419999989,+ "roundMs": 2252.020875000002,+ "firstDeltaMs": 2085.45404099999,"calls": []}]@@ -1721,235 +1145,142 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_clip_muted",+ "name": "add_clip","round": 0,- "atMs": 3206.6980840000033,- "execMs": 0.004207999998470768+ "atMs": 4524.402624999988,+ "execMs": 0.004541000002063811},{- "name": "set_clip_muted",+ "name": "add_clip","round": 0,- "atMs": 3206.8253750000003,- "execMs": 0.0007079999923007563+ "atMs": 4524.829999999987,+ "execMs": 0.0023750000109430403},{- "name": "set_clip_muted",- "round": 0,- "atMs": 3206.866209,- "execMs": 0.000375000003259629+ "name": "get_state",+ "round": 1,+ "atMs": 7152.849041999987,+ "execMs": 0.014457999990554526+ },+ {+ "name": "add_clip",+ "round": 2,+ "atMs": 11383.374249999993,+ "execMs": 0.0038330000243149698+ },+ {+ "name": "add_clip",+ "round": 2,+ "atMs": 11383.536499999987,+ "execMs": 0.001416999992216006+ },+ {+ "name": "get_state",+ "round": 3,+ "atMs": 13699.658916999993,+ "execMs": 0.02104200000758283+ },+ {+ "name": "add_clip",+ "round": 4,+ "atMs": 16223.84941699999,+ "execMs": 0.008166999992681667+ },+ {+ "name": "get_state",+ "round": 5,+ "atMs": 21366.135584000003,+ "execMs": 0.019665999978315085}],"timings": {- "gateMs": 1099.5708330000052,- "gateSkipped": false,- "ttftMs": 4331.471541999999,- "modelMs": 3583.189293000003,- "toolMs": 0.005290999994031154,- "totalMs": 4682.765417000002,- "judgeMs": 1056.2835420000047,+ "gateMs": 0.4308749999909196,+ "gateSkipped": true,+ "ttftMs": 24519.073459000007,+ "modelMs": 24505.098748999953,+ "toolMs": 0.07549899999867193,+ "totalMs": 24521.54041700001,+ "judgeMs": 1139.9844999999914,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2104.1892090000038,+ "roundMs": 4519.740416000001,"firstDeltaMs": null,"calls": [{- "name": "set_clip_muted",- "execMs": 0.004207999998470768- },- {- "name": "set_clip_muted",- "execMs": 0.0007079999923007563+ "name": "add_clip",+ "execMs": 0.004541000002063811},{- "name": "set_clip_muted",- "execMs": 0.000375000003259629+ "name": "add_clip",+ "execMs": 0.0023750000109430403}]},{- "roundMs": 1479.0000839999993,- "firstDeltaMs": 1124.1863750000048,- "calls": []- }- ]- }- }- ]- },- {- "name": "imported-photo-edit-references-photo",- "bucket": "single-tool",- "passRate": 1,- "budgetBreaches": [],- "latency": {- "gateMs": {- "p50": 1357,- "p95": 1760,- "mean": 1406,- "n": 3- },- "ttftMs": {- "p50": 9075,- "p95": 11338,- "mean": 9588,- "n": 3- },- "totalMs": {- "p50": 9316,- "p95": 11646,- "mean": 9922,- "n": 3- },- "modelMs": {- "p50": 8214,- "p95": 10290,- "mean": 8516,- "n": 3- },- "toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 3- },- "rounds": {- "mean": 2,- "max": 2- }- },- "runs": [- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "generate_image",- "round": 0,- "atMs": 4581.180332999997,- "execMs": 0.1023330000025453- }- ],- "timings": {- "gateMs": 1101.4423340000067,- "gateSkipped": false,- "ttftMs": 9074.625415999995,- "modelMs": 8214.064999999988,- "toolMs": 0.1023330000025453,- "totalMs": 9315.609666999997,- "judgeMs": 1006.6284580000065,- "retryMs": 0,- "retries": 0,- "rounds": [- {- "roundMs": 3470.6438749999943,+ "roundMs": 2625.8867079999764,"firstDeltaMs": null,"calls": [{- "name": "generate_image",- "execMs": 0.1023330000025453+ "name": "get_state",+ "execMs": 0.014457999990554526}]},{- "roundMs": 4743.4211249999935,- "firstDeltaMs": 4492.238375000001,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "generate_image",- "round": 0,- "atMs": 5732.386499999993,- "execMs": 0.01900000000023283- }- ],- "timings": {- "gateMs": 1356.507958000002,- "gateSkipped": false,- "ttftMs": 11338.172582999992,- "modelMs": 10289.911624999993,- "toolMs": 0.01900000000023283,- "totalMs": 11646.438582999996,- "judgeMs": 2002.9989999999962,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 4228.620708000002,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "add_clip",+ "execMs": 0.0038330000243149698+ },+ {+ "name": "add_clip",+ "execMs": 0.001416999992216006+ }+ ]+ },{- "roundMs": 4373.390625,+ "roundMs": 2315.2736670000013,"firstDeltaMs": null,"calls": [{- "name": "generate_image",- "execMs": 0.01900000000023283+ "name": "get_state",+ "execMs": 0.02104200000758283}]},{- "roundMs": 5916.520999999993,- "firstDeltaMs": 5604.620624999996,- "calls": []- }- ]- }- },- {- "pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": "does not explicitly state what changed in the project or timeline as required for an editing turn.",- "trace": [- {- "name": "generate_image",- "round": 0,- "atMs": 6737.4905,- "execMs": 0.014083000001846813- }- ],- "timings": {- "gateMs": 1760.3349580000067,- "gateSkipped": false,- "ttftMs": 8350.598333999995,- "modelMs": 7042.557709000001,- "toolMs": 0.014083000001846813,- "totalMs": 8802.90675000001,- "judgeMs": 1622.8013329999958,- "retryMs": 0,- "retries": 0,- "rounds": [+ "roundMs": 2522.8675829999847,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "add_clip",+ "execMs": 0.008166999992681667+ }+ ]+ },{- "roundMs": 4973.934666999994,+ "roundMs": 5139.648833999992,"firstDeltaMs": null,"calls": [{- "name": "generate_image",- "execMs": 0.014083000001846813+ "name": "get_state",+ "execMs": 0.019665999978315085}]},{- "roundMs": 2068.6230420000065,- "firstDeltaMs": 1611.5269580000022,+ "roundMs": 3153.060832999996,+ "firstDeltaMs": 3151.3847499999974,"calls": []}]@@ -1958,88 +1289,68 @@]},{- "name": "audio-ask-single-generate",- "bucket": "single-tool",- "passRate": 0.6666666666666666,+ "name": "photos-question-stays-in-chat",+ "bucket": "chat",+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1961,- "p95": 2175,- "mean": 2068,- "n": 2+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3},"ttftMs": {- "p50": 4869,- "p95": 7609,- "mean": 6239,- "n": 2+ "p50": 3796,+ "p95": 9432,+ "mean": 5531,+ "n": 3},"totalMs": {- "p50": 5299,- "p95": 8043,- "mean": 6671,- "n": 2+ "p50": 4585,+ "p95": 10114,+ "mean": 6327,+ "n": 3},"modelMs": {- "p50": 3338,- "p95": 5867,- "mean": 4603,- "n": 2+ "p50": 4582,+ "p95": 10112,+ "mean": 6325,+ "n": 3},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 1,+ "max": 1}},"runs": [{- "pass": false,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [- "banned phrasing: would you like me to",- "reply ends with a question after completed work"- ],- "judgeNote": "failed to state what changed in concrete terms and included an unnecessary question.",- "trace": [- {- "name": "voiceover_generate",- "round": 0,- "atMs": 3728.9431669999904,- "execMs": 0.008792000007815659- }- ],+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": "includes filler list and bolding, contrary to concise and factual requirements.",+ "trace": [],"timings": {- "gateMs": 1699.8382920000004,- "gateSkipped": false,- "ttftMs": 5259.466499999995,- "modelMs": 4228.192292000007,- "toolMs": 0.008792000007815659,- "totalMs": 5928.039376000015,- "judgeMs": 1274.4507079999894,+ "gateMs": 0.29920800001127645,+ "gateSkipped": true,+ "ttftMs": 9432.159249999997,+ "modelMs": 10111.655667000014,+ "toolMs": 0,+ "totalMs": 10114.286250000005,+ "judgeMs": 1545.4791250000126,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2024.2993750000023,- "firstDeltaMs": null,- "calls": [- {- "name": "voiceover_generate",- "execMs": 0.008792000007815659- }- ]- },- {- "roundMs": 2203.8929170000047,- "firstDeltaMs": 1529.2497920000023,+ "roundMs": 10111.655667000014,+ "firstDeltaMs": 9429.893333999993,"calls": []}]@@ -2047,85 +1358,51 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],- "judgeNote": "the assistant did not perform the edit as requested and failed to state the action taken.",- "trace": [- {- "name": "voiceover_generate",- "round": 0,- "atMs": 6426.81679099999,- "execMs": 0.011124999989988282- }- ],+ "judgeNote": "includes filler and list structure which contradicts the concise requirement.",+ "trace": [],"timings": {- "gateMs": 2175.430333000011,- "gateSkipped": false,- "ttftMs": 7609.286749999999,- "modelMs": 5867.479916000011,- "toolMs": 0.011124999989988282,- "totalMs": 8042.921374000012,- "judgeMs": 1223.5306249999994,+ "gateMs": 0.3065000000060536,+ "gateSkipped": true,+ "ttftMs": 3795.5814580000006,+ "modelMs": 4581.852749999991,+ "toolMs": 0,+ "totalMs": 4584.550958000007,+ "judgeMs": 1185.659667,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4249.87879100001,- "firstDeltaMs": null,- "calls": [- {- "name": "voiceover_generate",- "execMs": 0.011124999989988282- }- ]- },- {- "roundMs": 1617.601125000001,- "firstDeltaMs": 1180.6729170000035,+ "roundMs": 4581.852749999991,+ "firstDeltaMs": 3793.1336249999877,"calls": []}]}},{- "pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],- "judgeNote": "editing turn failed to state what changed in concrete terms or offer placement/approval.",- "trace": [- {- "name": "voiceover_generate",- "round": 0,- "atMs": 3922.9099590000114,- "execMs": 0.007874999981140718- }- ],+ "judgeNote": "contains self-promotional list formatting and filler instead of being concise and direct.",+ "trace": [],"timings": {- "gateMs": 1960.9494589999958,- "gateSkipped": false,- "ttftMs": 4869.437583999999,- "modelMs": 3338.3740000000107,- "toolMs": 0.007874999981140718,- "totalMs": 5299.331333999988,- "judgeMs": 1118.1096250000119,+ "gateMs": 0.16795900001307018,+ "gateSkipped": true,+ "ttftMs": 3365.5846249999886,+ "modelMs": 4280.409333000018,+ "toolMs": 0,+ "totalMs": 4282.13499999998,+ "judgeMs": 1223.6891659999965,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1958.6070419999887,- "firstDeltaMs": null,- "calls": [- {- "name": "voiceover_generate",- "execMs": 0.007874999981140718- }- ]- },- {- "roundMs": 1379.766958000022,- "firstDeltaMs": 943.9545830000134,+ "roundMs": 4280.409333000018,+ "firstDeltaMs": 3364.0880830000096,"calls": []}]@@ -2134,33 +1411,33 @@]},{- "name": "music-ask-single-generate",- "bucket": "single-tool",+ "name": "clutter-complaint-deletes-nothing",+ "bucket": "chat","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1168,- "p95": 1922,- "mean": 1419,+ "p50": 999,+ "p95": 1019,+ "mean": 968,"n": 3},"ttftMs": {- "p50": 4174,- "p95": 5350,- "mean": 4484,+ "p50": 3976,+ "p95": 4023,+ "mean": 3807,"n": 3},"totalMs": {- "p50": 4891,- "p95": 5623,- "mean": 4911,+ "p50": 4139,+ "p95": 4160,+ "mean": 3975,"n": 3},"modelMs": {- "p50": 3701,- "p95": 3724,- "mean": 3492,+ "p50": 4137,+ "p95": 4157,+ "mean": 3972,"n": 3},"toolMs": {@@ -2177,42 +1454,30 @@"runs": [{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "chat",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,- "trace": [- {- "name": "generate_music",- "round": 0,- "atMs": 2744.4827499999956,- "execMs": 0.05770900001516566- }- ],+ "trace": [],"timings": {- "gateMs": 1166.6052499999932,+ "gateMs": 1018.6645830000052,"gateSkipped": false,- "ttftMs": 4173.906666999974,- "modelMs": 3724.2767920000188,- "toolMs": 0.05770900001516566,- "totalMs": 4890.939751000027,- "judgeMs": 1558.3744589999842,+ "ttftMs": 4023.348457999993,+ "modelMs": 4156.531707999995,+ "toolMs": 0,+ "totalMs": 4159.913292000012,+ "judgeMs": 1288.322165999998,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1571.5885420000122,+ "roundMs": 1017.4890829999931,"firstDeltaMs": null,- "calls": [- {- "name": "generate_music",- "execMs": 0.05770900001516566- }- ]+ "calls": []},{- "roundMs": 2152.6882500000065,- "firstDeltaMs": 1427.686875000014,+ "roundMs": 3139.042625000002,+ "firstDeltaMs": 3002.773125000007,"calls": []}]@@ -2220,42 +1485,30 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "chat",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,- "trace": [- {- "name": "generate_music",- "round": 0,- "atMs": 2664.921207999985,- "execMs": 0.005541999998968095- }- ],+ "trace": [],"timings": {- "gateMs": 1168.3784589999996,+ "gateMs": 999.4806249999965,"gateSkipped": false,- "ttftMs": 3927.6446659999783,- "modelMs": 3050.835167000012,- "toolMs": 0.005541999998968095,- "totalMs": 4219.2191680000105,- "judgeMs": 1183.1158340000256,+ "ttftMs": 3422.882499999978,+ "modelMs": 3623.404459000012,+ "toolMs": 0,+ "totalMs": 3626.0681669999904,+ "judgeMs": 1067.8349999999919,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1494.159249999997,+ "roundMs": 998.0565420000057,"firstDeltaMs": null,- "calls": [- {- "name": "generate_music",- "execMs": 0.005541999998968095- }- ]+ "calls": []},{- "roundMs": 1556.675917000015,- "firstDeltaMs": 1261.3280000000086,+ "roundMs": 2625.3479170000064,+ "firstDeltaMs": 2422.437249999988,"calls": []}]@@ -2263,42 +1516,30 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "chat",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,- "trace": [- {- "name": "generate_music",- "round": 0,- "atMs": 4043.8962919999904,- "execMs": 0.0009169999975711107- }- ],+ "trace": [],"timings": {- "gateMs": 1922.238458000007,+ "gateMs": 886.5164169999771,"gateSkipped": false,- "ttftMs": 5350.096457999985,- "modelMs": 3701.044875999971,- "toolMs": 0.0009169999975711107,- "totalMs": 5623.2842509999755,- "judgeMs": 980.3183330000029,+ "ttftMs": 3975.787666999997,+ "modelMs": 4136.723333000002,+ "toolMs": 0,+ "totalMs": 4138.84941699999,+ "judgeMs": 995.8177499999874,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2119.050833999994,+ "roundMs": 885.9137080000073,"firstDeltaMs": null,- "calls": [- {- "name": "generate_music",- "execMs": 0.0009169999975711107- }- ]+ "calls": []},{- "roundMs": 1581.9940419999766,- "firstDeltaMs": 1305.8496249999735,+ "roundMs": 3250.8096249999944,+ "firstDeltaMs": 3087.9022500000137,"calls": []}]@@ -2307,33 +1548,33 @@]},{- "name": "trim-ask-single-tool",+ "name": "video-ask-single-generate","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1034,- "p95": 1941,- "mean": 1289,+ "p50": 1044,+ "p95": 2039,+ "mean": 1367,"n": 3},"ttftMs": {- "p50": 4870,- "p95": 4978,- "mean": 4550,+ "p50": 2605,+ "p95": 3613,+ "mean": 2892,"n": 3},"totalMs": {- "p50": 5142,- "p95": 5160,- "mean": 4771,+ "p50": 2937,+ "p95": 4451,+ "mean": 3367,"n": 3},"modelMs": {- "p50": 3200,- "p95": 4126,- "mean": 3482,+ "p50": 2928,+ "p95": 3915,+ "mean": 3184,"n": 3},"toolMs": {@@ -2356,36 +1597,36 @@"judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "generate_video","round": 0,- "atMs": 2341.239499999996,- "execMs": 0.011041999998269603+ "atMs": 1299.2413330000127,+ "execMs": 0.0105829999956768}],"timings": {- "gateMs": 893.2194580000069,+ "gateMs": 1017.9502500000235,"gateSkipped": false,- "ttftMs": 3800.93733299998,- "modelMs": 3118.1342920000025,- "toolMs": 0.011041999998269603,- "totalMs": 4011.3647920000076,- "judgeMs": 1534.2747080000117,+ "ttftMs": 2604.6957500000135,+ "modelMs": 2928.2186659999716,+ "toolMs": 0.0105829999956768,+ "totalMs": 2936.8464169999934,+ "judgeMs": 1152.3697089999914,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1443.4872920000053,+ "roundMs": 1291.8809159999946,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.011041999998269603+ "name": "generate_video",+ "execMs": 0.0105829999956768}]},{- "roundMs": 1674.6469999999972,- "firstDeltaMs": 1458.2413749999832,+ "roundMs": 1636.337749999977,+ "firstDeltaMs": 1304.3766659999965,"calls": []}]@@ -2399,36 +1640,36 @@"judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "generate_video","round": 0,- "atMs": 3497.3065420000057,- "execMs": 0.0071659999957773834+ "atMs": 1285.855958,+ "execMs": 0.005707999982405454}],"timings": {- "gateMs": 1941.2230410000193,+ "gateMs": 1044.4083330000285,"gateSkipped": false,- "ttftMs": 4978.370083999995,- "modelMs": 3200.454000000027,- "toolMs": 0.0071659999957773834,- "totalMs": 5141.684207000042,- "judgeMs": 1043.0578340000066,+ "ttftMs": 2457.7862920000043,+ "modelMs": 2709.105833999987,+ "toolMs": 0.005707999982405454,+ "totalMs": 2712.599499999982,+ "judgeMs": 1007.8175420000043,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1553.9718330000178,+ "roundMs": 1283.4550419999869,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.0071659999957773834+ "name": "generate_video",+ "execMs": 0.005707999982405454}]},{- "roundMs": 1646.4821670000092,- "firstDeltaMs": 1479.5018339999951,+ "roundMs": 1425.6507920000004,+ "firstDeltaMs": 1171.2049999999872,"calls": []}]@@ -2442,36 +1683,36 @@"judgeNote": null,"trace": [{- "name": "trim_clip",+ "name": "generate_video","round": 0,- "atMs": 3469.6101250000065,- "execMs": 0.009249999973690137+ "atMs": 2039.3513749999984,+ "execMs": 0.008792000007815659}],"timings": {- "gateMs": 1033.6389999999956,+ "gateMs": 2039.0194999999949,"gateSkipped": false,- "ttftMs": 4869.926124999998,- "modelMs": 4126.357707999967,- "toolMs": 0.009249999973690137,- "totalMs": 5160.005957999936,- "judgeMs": 665.0173749999958,+ "ttftMs": 3612.8035839999793,+ "modelMs": 3915.338792000024,+ "toolMs": 0.008792000007815659,+ "totalMs": 4450.538084,+ "judgeMs": 1063.8595839999907,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2433.7336659999855,+ "roundMs": 1505.5185420000053,"firstDeltaMs": null,"calls": [{- "name": "trim_clip",- "execMs": 0.009249999973690137+ "name": "generate_video",+ "execMs": 0.008792000007815659}]},{- "roundMs": 1692.6240419999813,- "firstDeltaMs": 1397.010083000001,+ "roundMs": 2409.820250000019,+ "firstDeltaMs": 1572.3080410000111,"calls": []}]@@ -2480,33 +1721,33 @@]},{- "name": "transition-ask-styled",+ "name": "iterate-keeps-existing-clip","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1049,- "p95": 1080,- "mean": 1039,+ "p50": 1125,+ "p95": 1589,+ "mean": 1274,"n": 3},"ttftMs": {- "p50": 4667,- "p95": 6599,- "mean": 5162,+ "p50": 8666,+ "p95": 12466,+ "mean": 9679,"n": 3},"totalMs": {- "p50": 5358,- "p95": 6721,- "mean": 5517,+ "p50": 8887,+ "p95": 12692,+ "mean": 9894,"n": 3},"modelMs": {- "p50": 4370,- "p95": 5641,- "mean": 4477,+ "p50": 8880,+ "p95": 12688,+ "mean": 9887,"n": 3},"toolMs": {@@ -2516,49 +1757,54 @@"n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 3}},"runs": [{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",- "round": 0,- "atMs": 2871.2307920000167,- "execMs": 0.10854200000176206+ "name": "generate_video",+ "round": 1,+ "atMs": 5808.098457999993,+ "execMs": 0.4249580000177957}],"timings": {- "gateMs": 987.7601660000219,+ "gateMs": 1106.5414170000004,"gateSkipped": false,- "ttftMs": 4667.087041999999,- "modelMs": 4369.901126000012,- "toolMs": 0.10854200000176206,- "totalMs": 5357.769834000035,- "judgeMs": 1705.3454590000038,+ "ttftMs": 8666.234582999983,+ "modelMs": 8879.914042999997,+ "toolMs": 0.4249580000177957,+ "totalMs": 8886.599541999982,+ "judgeMs": 1024.8275840000133,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1878.7014170000039,+ "roundMs": 1105.986917000002,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4698.145709000004,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.10854200000176206+ "name": "generate_video",+ "execMs": 0.4249580000177957}]},{- "roundMs": 2491.1997090000077,- "firstDeltaMs": 1794.2004590000142,+ "roundMs": 3075.781416999991,+ "firstDeltaMs": 2855.6023749999877,"calls": []}]@@ -2566,42 +1812,47 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",- "round": 0,- "atMs": 2488.6279159999976,- "execMs": 0.015625+ "name": "generate_video",+ "round": 1,+ "atMs": 9028.588958000008,+ "execMs": 0.021500000002561137}],"timings": {- "gateMs": 1049.4243330000027,+ "gateMs": 1589.2670409999846,"gateSkipped": false,- "ttftMs": 4218.372957999993,- "modelMs": 3421.43187499998,- "toolMs": 0.015625,- "totalMs": 4470.871832999983,- "judgeMs": 982.6664579999924,+ "ttftMs": 12466.115083000012,+ "modelMs": 12687.800541999983,+ "toolMs": 0.021500000002561137,+ "totalMs": 12692.396750000014,+ "judgeMs": 1988.3183750000026,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1436.1326249999984,+ "roundMs": 1587.6192079999892,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 7437.672583999985,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.015625+ "name": "generate_video",+ "execMs": 0.021500000002561137}]},{- "roundMs": 1985.2992499999818,- "firstDeltaMs": 1728.2534999999916,+ "roundMs": 3662.508750000008,+ "firstDeltaMs": 3436.438374999998,"calls": []}]@@ -2609,42 +1860,47 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",- "round": 0,- "atMs": 5147.19754199998,- "execMs": 0.00979199999710545+ "name": "generate_video",+ "round": 1,+ "atMs": 5275.527792000008,+ "execMs": 0.013042000005953014}],"timings": {- "gateMs": 1080.3517920000013,+ "gateMs": 1125.2969999999914,"gateSkipped": false,- "ttftMs": 6599.162707999989,- "modelMs": 5640.641291999986,- "toolMs": 0.00979199999710545,- "totalMs": 6721.002875999984,- "judgeMs": 1306.8071660000132,+ "ttftMs": 7905.0895420000015,+ "modelMs": 8093.165415999974,+ "toolMs": 0.013042000005953014,+ "totalMs": 8102.304999999993,+ "judgeMs": 848.6339169999992,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4064.380083999975,+ "roundMs": 1122.2649580000143,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4146.219790999981,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.00979199999710545+ "name": "generate_video",+ "execMs": 0.013042000005953014}]},{- "roundMs": 1576.2612080000108,- "firstDeltaMs": 1450.1052499999932,+ "roundMs": 2824.6806669999787,+ "firstDeltaMs": 2628.1128749999916,"calls": []}]@@ -2653,33 +1909,33 @@]},{- "name": "slide-in-direction-correct",+ "name": "mute-all-does-not-redo-prior-turn","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1101,- "p95": 2229,- "mean": 1473,+ "p50": 1517,+ "p95": 1969,+ "mean": 1514,"n": 3},"ttftMs": {- "p50": 5118,- "p95": 11277,- "mean": 6710,+ "p50": 8590,+ "p95": 10157,+ "mean": 7317,"n": 3},"totalMs": {- "p50": 5347,- "p95": 11920,- "mean": 7069,+ "p50": 8591,+ "p95": 10179,+ "mean": 7364,"n": 3},"modelMs": {- "p50": 3118,- "p95": 10819,- "mean": 5596,+ "p50": 8581,+ "p95": 10169,+ "mean": 7357,"n": 3},"toolMs": {@@ -2689,49 +1945,74 @@"n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 4}},"runs": [{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_animation",- "round": 0,- "atMs": 3810.6886670000004,- "execMs": 0.04279199999928096+ "name": "set_clip_muted",+ "round": 1,+ "atMs": 5964.722125,+ "execMs": 0.08354200000758283+ },+ {+ "name": "set_clip_muted",+ "round": 1,+ "atMs": 5965.411999999982,+ "execMs": 0.015583000000333413+ },+ {+ "name": "set_clip_muted",+ "round": 1,+ "atMs": 5965.584124999994,+ "execMs": 0.0012089999800082296}],"timings": {- "gateMs": 2229.0063330000003,+ "gateMs": 1056.3727910000016,"gateSkipped": false,- "ttftMs": 5118.017333,- "modelMs": 3117.9767079999992,- "toolMs": 0.04279199999928096,- "totalMs": 5347.025832999999,- "judgeMs": 1844.4930829999994,+ "ttftMs": 8589.884625000006,+ "modelMs": 8581.354957999982,+ "toolMs": 0.10033399998792447,+ "totalMs": 8590.777457999997,+ "judgeMs": 896.9985420000157,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1565.4869579999995,+ "roundMs": 1054.28125,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4903.310165999981,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.04279199999928096+ "name": "set_clip_muted",+ "execMs": 0.08354200000758283+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.015583000000333413+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.0012089999800082296}]},{- "roundMs": 1552.4897499999997,- "firstDeltaMs": 1305.6425829999998,+ "roundMs": 2623.7635420000006,+ "firstDeltaMs": 2623.061625000002,"calls": []}]@@ -2739,42 +2020,72 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "set_animation",- "round": 0,- "atMs": 5614.405542,- "execMs": 0.013249999999970896+ "name": "set_clip_muted",+ "round": 2,+ "atMs": 7247.186042000016,+ "execMs": 0.006792000000132248+ },+ {+ "name": "set_clip_muted",+ "round": 2,+ "atMs": 7247.410292000015,+ "execMs": 0.0009589999972376972+ },+ {+ "name": "set_clip_muted",+ "round": 2,+ "atMs": 7247.5032920000085,+ "execMs": 0.0005830000154674053}],"timings": {- "gateMs": 1101.1039579999997,+ "gateMs": 1968.711707999988,"gateSkipped": false,- "ttftMs": 11277.413499999999,- "modelMs": 10819.189166999999,- "toolMs": 0.013249999999970896,- "totalMs": 11920.306374999998,- "judgeMs": 2039.3310840000013,+ "ttftMs": 10157.192375000013,+ "modelMs": 10169.06408499999,+ "toolMs": 0.00833400001283735,+ "totalMs": 10178.960374999995,+ "judgeMs": 1064.9554160000116,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4509.250167,+ "roundMs": 1963.3322089999856,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 0.012667000002693385,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 5275.5905839999905,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.013249999999970896+ "name": "set_clip_muted",+ "execMs": 0.006792000000132248+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.0009589999972376972+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.0005830000154674053}]},{- "roundMs": 6309.9389999999985,- "firstDeltaMs": 5660.954999999998,+ "roundMs": 2930.128625000012,+ "firstDeltaMs": 2908.5801670000073,"calls": []}]@@ -2788,36 +2099,56 @@"judgeNote": null,"trace": [{- "name": "set_animation",+ "name": "set_clip_muted",+ "round": 0,+ "atMs": 2015.978792000038,+ "execMs": 0.006124999956227839+ },+ {+ "name": "set_clip_muted",+ "round": 0,+ "atMs": 2016.2934590000077,+ "execMs": 0.002957999997306615+ },+ {+ "name": "set_clip_muted","round": 0,- "atMs": 2437.9349580000016,- "execMs": 0.019499999998515705+ "atMs": 2016.3934590000426,+ "execMs": 0.0012500000302679837}],"timings": {- "gateMs": 1087.6784170000028,+ "gateMs": 1516.8079170000274,"gateSkipped": false,- "ttftMs": 3733.237792,- "modelMs": 2852.1087919999954,- "toolMs": 0.019499999998515705,- "totalMs": 3939.8067089999968,- "judgeMs": 1550.5267920000006,+ "ttftMs": 3204.917375000019,+ "modelMs": 3319.433290999965,+ "toolMs": 0.010332999983802438,+ "totalMs": 3323.5751670000027,+ "judgeMs": 1101.993542000011,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1346.726083999998,+ "roundMs": 2013.8921249999548,"firstDeltaMs": null,"calls": [{- "name": "set_animation",- "execMs": 0.019499999998515705+ "name": "set_clip_muted",+ "execMs": 0.006124999956227839+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.002957999997306615+ },+ {+ "name": "set_clip_muted",+ "execMs": 0.0012500000302679837}]},{- "roundMs": 1505.3827079999974,- "firstDeltaMs": 1293.4027499999975,+ "roundMs": 1305.54116600001,+ "firstDeltaMs": 1187.149040999997,"calls": []}]@@ -2826,33 +2157,33 @@]},{- "name": "look-ask-single-tool",+ "name": "imported-photo-edit-references-photo","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1449,- "p95": 1844,- "mean": 1474,+ "p50": 1053,+ "p95": 1071,+ "mean": 1049,"n": 3},"ttftMs": {- "p50": 4601,- "p95": 5168,- "mean": 4615,+ "p50": 8969,+ "p95": 10235,+ "mean": 9316,"n": 3},"totalMs": {- "p50": 4824,- "p95": 5416,- "mean": 4963,+ "p50": 9153,+ "p95": 10456,+ "mean": 9504,"n": 3},"modelMs": {- "p50": 3520,- "p95": 3571,- "mean": 3489,+ "p50": 9146,+ "p95": 10451,+ "mean": 9498,"n": 3},"toolMs": {@@ -2862,49 +2193,54 @@"n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 3}},"runs": [{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "add_effect",- "round": 0,- "atMs": 3993.3052499999976,- "execMs": 0.03299999999944703+ "name": "generate_image",+ "round": 1,+ "atMs": 5451.5324580000015,+ "execMs": 0.07512499997392297}],"timings": {- "gateMs": 1844.3342919999996,+ "gateMs": 1071.4392920000246,"gateSkipped": false,- "ttftMs": 5168.402291000002,- "modelMs": 3571.335750000002,- "toolMs": 0.03299999999944703,- "totalMs": 5415.703042000001,- "judgeMs": 1173.9096250000002,+ "ttftMs": 8968.940958000021,+ "modelMs": 9145.912624999997,+ "toolMs": 0.07512499997392297,+ "totalMs": 9153.212958000018,+ "judgeMs": 906.2155830000411,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2142.2010420000006,+ "roundMs": 1070.613874999981,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4374.9537080000155,"firstDeltaMs": null,"calls": [{- "name": "add_effect",- "execMs": 0.03299999999944703+ "name": "generate_image",+ "execMs": 0.07512499997392297}]},{- "roundMs": 1429.1347080000014,- "firstDeltaMs": 1174.6121660000026,+ "roundMs": 3700.345042000001,+ "firstDeltaMs": 3516.6372089999495,"calls": []}]@@ -2912,42 +2248,47 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "add_effect",- "round": 0,- "atMs": 3543.826458000003,- "execMs": 0.0020839999997406267+ "name": "generate_image",+ "round": 1,+ "atMs": 5470.562291000038,+ "execMs": 0.026165999996010214}],"timings": {- "gateMs": 1448.5215000000026,+ "gateMs": 1053.2412089999998,"gateSkipped": false,- "ttftMs": 4600.839458000002,- "modelMs": 3375.511541999993,- "toolMs": 0.0020839999997406267,- "totalMs": 4824.035125999995,- "judgeMs": 1144.0252089999994,+ "ttftMs": 10235.048708000046,+ "modelMs": 10450.810709000041,+ "toolMs": 0.026165999996010214,+ "totalMs": 10456.242166000011,+ "judgeMs": 1732.5662500000326,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2093.5255419999958,+ "roundMs": 1051.8834170000046,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4414.898333000019,"firstDeltaMs": null,"calls": [{- "name": "add_effect",- "execMs": 0.0020839999997406267+ "name": "generate_image",+ "execMs": 0.026165999996010214}]},{- "roundMs": 1281.9859999999971,- "firstDeltaMs": 1056.633332999998,+ "roundMs": 4984.0289590000175,+ "firstDeltaMs": 4763.069749999966,"calls": []}]@@ -2955,42 +2296,47 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "add_effect",- "round": 0,- "atMs": 2715.1397920000018,- "execMs": 0.011459000001195818+ "name": "generate_image",+ "round": 1,+ "atMs": 4711.853166999994,+ "execMs": 0.014374999969732016}],"timings": {- "gateMs": 1130.4469160000008,+ "gateMs": 1023.6219169999822,"gateSkipped": false,- "ttftMs": 4077.1619580000042,- "modelMs": 3519.7674999999945,- "toolMs": 0.011459000001195818,- "totalMs": 4650.225874999996,- "judgeMs": 1834.577416,+ "ttftMs": 8743.302208999987,+ "modelMs": 8895.867667000042,+ "toolMs": 0.014374999969732016,+ "totalMs": 8901.882041999954,+ "judgeMs": 907.602125000034,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1581.8143749999945,+ "roundMs": 1021.7506670000148,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3686.4589590000105,"firstDeltaMs": null,"calls": [{- "name": "add_effect",- "execMs": 0.011459000001195818+ "name": "generate_image",+ "execMs": 0.014374999969732016}]},{- "roundMs": 1937.953125,- "firstDeltaMs": 1360.5665420000005,+ "roundMs": 4187.658041000017,+ "firstDeltaMs": 4029.4950410000165,"calls": []}]@@ -2999,33 +2345,33 @@]},{- "name": "animation-overrides-transition",+ "name": "audio-ask-single-generate","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1784,- "p95": 1842,- "mean": 1766,+ "p50": 1533,+ "p95": 1838,+ "mean": 1633,"n": 3},"ttftMs": {- "p50": 8626,- "p95": 10678,- "mean": 9140,+ "p50": 3057,+ "p95": 5651,+ "mean": 3761,"n": 3},"totalMs": {- "p50": 8788,- "p95": 10994,- "mean": 9387,+ "p50": 3350,+ "p95": 5793,+ "mean": 3954,"n": 3},"modelMs": {- "p50": 7118,- "p95": 9151,- "mean": 7621,+ "p50": 3342,+ "p95": 5789,+ "mean": 3947,"n": 3},"toolMs": {@@ -3042,52 +2388,42 @@"runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",- "round": 0,- "atMs": 8791.210708999999,- "execMs": 0.078125- },- {- "name": "set_animation",+ "name": "voiceover_generate","round": 0,- "atMs": 8791.778624999999,- "execMs": 0.003540999998222105+ "atMs": 1700.0957499999786,+ "execMs": 0.03291700000409037}],"timings": {- "gateMs": 1842.1546250000029,+ "gateMs": 1532.810999999987,"gateSkipped": false,- "ttftMs": 10677.501459,- "modelMs": 9151.498375000003,- "toolMs": 0.0816659999982221,- "totalMs": 10993.734666000004,- "judgeMs": 955.9993340000001,+ "ttftMs": 3057.1424579999875,+ "modelMs": 3342.3412499999977,+ "toolMs": 0.03291700000409037,+ "totalMs": 3350.1117920000106,+ "judgeMs": 1886.6240830000024,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6940.993625000003,+ "roundMs": 1694.7835000000196,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.078125- },- {- "name": "set_animation",- "execMs": 0.003540999998222105+ "name": "voiceover_generate",+ "execMs": 0.03291700000409037}]},{- "roundMs": 2210.50475,- "firstDeltaMs": 1884.5993750000052,+ "roundMs": 1647.5577499999781,+ "firstDeltaMs": 1354.9294169999775,"calls": []}]@@ -3095,52 +2431,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": null,+ "judgeNote": "failed to report the specific tool action and did not place the clip despite having the capability to do so.","trace": [{- "name": "set_transition",- "round": 0,- "atMs": 6783.634334000002,- "execMs": 0.0032910000008996576- },- {- "name": "set_animation",+ "name": "voiceover_generate","round": 0,- "atMs": 6783.917834,- "execMs": 0.0020000000004074536+ "atMs": 4378.363666000019,+ "execMs": 0.0047500000218860805}],"timings": {- "gateMs": 1670.439624999999,+ "gateMs": 1837.8528330000117,"gateSkipped": false,- "ttftMs": 8626.219708999997,- "modelMs": 7117.609916999994,- "toolMs": 0.005291000001307111,- "totalMs": 8788.054832999995,- "judgeMs": 1796.8855410000106,+ "ttftMs": 5651.093791000021,+ "modelMs": 5789.496667000058,+ "toolMs": 0.0047500000218860805,+ "totalMs": 5792.956333000038,+ "judgeMs": 1206.63391699997,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5109.921416999998,+ "roundMs": 4376.3344170000055,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.0032910000008996576- },- {- "name": "set_animation",- "execMs": 0.0020000000004074536+ "name": "voiceover_generate",+ "execMs": 0.0047500000218860805}]},{- "roundMs": 2007.6884999999966,- "firstDeltaMs": 1840.9062919999924,+ "roundMs": 1413.1622500000522,+ "firstDeltaMs": 1271.5903750000289,"calls": []}]@@ -3148,52 +2474,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",- "round": 0,- "atMs": 6382.127334000004,- "execMs": 0.0022499999904539436- },- {- "name": "set_animation",+ "name": "voiceover_generate","round": 0,- "atMs": 6382.372000000003,- "execMs": 0.0015839999978197739+ "atMs": 1529.0014160000137,+ "execMs": 0.0039579999865964055}],"timings": {- "gateMs": 1784.381583999988,+ "gateMs": 1528.721084000019,"gateSkipped": false,- "ttftMs": 8117.488959000009,- "modelMs": 6594.456666999991,- "toolMs": 0.0038339999882737175,- "totalMs": 8378.842084999967,- "judgeMs": 1047.1532080000034,+ "ttftMs": 2574.5761250000214,+ "modelMs": 2710.5195420000236,+ "toolMs": 0.0039579999865964055,+ "totalMs": 2718.1632910000044,+ "judgeMs": 1101.9649579999968,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4595.841667000001,+ "roundMs": 1522.7087080000201,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.0022499999904539436- },- {- "name": "set_animation",- "execMs": 0.0015839999978197739+ "name": "voiceover_generate",+ "execMs": 0.0039579999865964055}]},{- "roundMs": 1998.6149999999907,- "firstDeltaMs": 1734.5484169999982,+ "roundMs": 1187.8108340000035,+ "firstDeltaMs": 1044.459124999994,"calls": []}]@@ -3202,40 +2518,40 @@]},{- "name": "edit-request-still-uses-tools",+ "name": "music-ask-single-generate","bucket": "single-tool",- "passRate": 0.6666666666666666,+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1080,- "p95": 1617,- "mean": 1348,- "n": 2+ "p50": 1881,+ "p95": 2644,+ "mean": 1873,+ "n": 3},"ttftMs": {- "p50": 4367,- "p95": 5239,- "mean": 4803,- "n": 2+ "p50": 2972,+ "p95": 3824,+ "mean": 3137,+ "n": 3},"totalMs": {- "p50": 4546,- "p95": 6010,- "mean": 5278,- "n": 2+ "p50": 3279,+ "p95": 4266,+ "mean": 3505,+ "n": 3},"modelMs": {- "p50": 2929,- "p95": 4931,- "mean": 3930,- "n": 2+ "p50": 2965,+ "p95": 3066,+ "mean": 2986,+ "n": 3},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 3},"rounds": {"mean": 2,@@ -3244,46 +2560,43 @@},"runs": [{- "pass": false,+ "pass": true,"intent": "simple","roundModel": "gemini-3.1-flash-lite",- "notes": [- "banned phrasing: would you like me to",- "reply ends with a question after completed work"- ],- "judgeNote": "includes an unnecessary question instead of being decisive.",+ "notes": [],+ "judgeNote": null,"trace": [{- "name": "subtitles_generate",+ "name": "generate_music","round": 0,- "atMs": 6295.389165999994,- "execMs": 0.008540999988326803+ "atMs": 2644.8367919999873,+ "execMs": 0.3267500000074506}],"timings": {- "gateMs": 1236.3779579999973,+ "gateMs": 2644.0936249999795,"gateSkipped": false,- "ttftMs": 7565.974208,- "modelMs": 6619.3318750000035,- "toolMs": 0.008540999988326803,- "totalMs": 7855.718373999989,- "judgeMs": 1052.966916999998,+ "ttftMs": 3823.8966670000227,+ "modelMs": 3065.8347509999876,+ "toolMs": 0.3267500000074506,+ "totalMs": 4266.2813750000205,+ "judgeMs": 993.905792000005,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5054.065875,+ "roundMs": 1446.8140840000124,"firstDeltaMs": null,"calls": [{- "name": "subtitles_generate",- "execMs": 0.008540999988326803+ "name": "generate_music",+ "execMs": 0.3267500000074506}]},{- "roundMs": 1565.2660000000033,- "firstDeltaMs": 1268.640167000005,+ "roundMs": 1619.0206669999752,+ "firstDeltaMs": 1176.9354999999632,"calls": []}]@@ -3297,36 +2610,36 @@"judgeNote": null,"trace": [{- "name": "subtitles_generate",+ "name": "generate_music","round": 0,- "atMs": 3854.8450410000078,- "execMs": 0.007041999997454695+ "atMs": 1881.1422080000048,+ "execMs": 0.011832999996840954}],"timings": {- "gateMs": 1079.525833000007,+ "gateMs": 1880.875457999995,"gateSkipped": false,- "ttftMs": 5238.876791000002,- "modelMs": 4930.8359170000185,- "toolMs": 0.007041999997454695,- "totalMs": 6010.368792000023,- "judgeMs": 1846.1411249999946,+ "ttftMs": 2972.1590419999557,+ "modelMs": 2926.754917000013,+ "toolMs": 0.011832999996840954,+ "totalMs": 3279.4256249999744,+ "judgeMs": 2055.154957999999,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2772.699375000011,+ "roundMs": 1530.9637499999953,"firstDeltaMs": null,"calls": [{- "name": "subtitles_generate",- "execMs": 0.007041999997454695+ "name": "generate_music",+ "execMs": 0.011832999996840954}]},{- "roundMs": 2158.1365420000075,- "firstDeltaMs": 1382.3067500000034,+ "roundMs": 1395.7911670000176,+ "firstDeltaMs": 1088.8744580000057,"calls": []}]@@ -3337,39 +2650,39 @@"intent": "simple","roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": "failed to state what changed in concrete terms (e.g., specifying the duration or style of captions).",+ "judgeNote": null,"trace": [{- "name": "subtitles_generate",+ "name": "generate_music","round": 0,- "atMs": 3141.281541999997,- "execMs": 0.008874999999534339+ "atMs": 1478.7429580000462,+ "execMs": 0.007624999969266355}],"timings": {- "gateMs": 1617.3688330000004,+ "gateMs": 1092.6510830000043,"gateSkipped": false,- "ttftMs": 4366.647708999997,- "modelMs": 2928.960375000024,- "toolMs": 0.008874999999534339,- "totalMs": 4546.338083000024,- "judgeMs": 2031.9173750000045,+ "ttftMs": 2616.122250000015,+ "modelMs": 2965.4589169999235,+ "toolMs": 0.007624999969266355,+ "totalMs": 2969.112542000017,+ "judgeMs": 916.1081659999909,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1521.2808330000116,+ "roundMs": 1477.0544999999693,"firstDeltaMs": null,"calls": [{- "name": "subtitles_generate",- "execMs": 0.008874999999534339+ "name": "generate_music",+ "execMs": 0.007624999969266355}]},{- "roundMs": 1407.6795420000126,- "firstDeltaMs": 1223.2073750000127,+ "roundMs": 1488.4044169999543,+ "firstDeltaMs": 1135.6620419999817,"calls": []}]@@ -3378,33 +2691,33 @@]},{- "name": "cartoon-from-audio-plans-scene",+ "name": "trim-ask-single-tool","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1185,- "p95": 1690,- "mean": 1303,+ "p50": 1141,+ "p95": 1776,+ "mean": 1321,"n": 3},"ttftMs": {- "p50": 9898,- "p95": 11150,- "mean": 9587,+ "p50": 2929,+ "p95": 3493,+ "mean": 2947,"n": 3},"totalMs": {- "p50": 10303,- "p95": 11463,- "mean": 9970,+ "p50": 3175,+ "p95": 3667,+ "mean": 3154,"n": 3},"modelMs": {- "p50": 8613,- "p95": 10279,- "mean": 8667,+ "p50": 2675,+ "p95": 3664,+ "mean": 2985,"n": 3},"toolMs": {@@ -3421,42 +2734,42 @@"runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "trim_clip","round": 0,- "atMs": 7860.8502919999955,- "execMs": 0.05570799999986775+ "atMs": 1776.7467499999912,+ "execMs": 0.055333000025711954}],"timings": {- "gateMs": 1689.9393750000017,+ "gateMs": 1776.2614590000012,"gateSkipped": false,- "ttftMs": 9897.533917,- "modelMs": 8613.239250000006,- "toolMs": 0.05570799999986775,- "totalMs": 10303.234333000008,- "judgeMs": 879.6468339999992,+ "ttftMs": 2929.383165999956,+ "modelMs": 2675.4219579999917,+ "toolMs": 0.055333000025711954,+ "totalMs": 3174.673207999964,+ "judgeMs": 1016.290083000029,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6166.519917000005,+ "roundMs": 1278.9317080000183,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.05570799999986775+ "name": "trim_clip",+ "execMs": 0.055333000025711954}]},{- "roundMs": 2446.719333000001,- "firstDeltaMs": 2035.7157080000034,+ "roundMs": 1396.4902499999735,+ "firstDeltaMs": 1151.3965409999946,"calls": []}]@@ -3464,42 +2777,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "trim_clip","round": 0,- "atMs": 6130.794541000003,- "execMs": 0.008374999997613486+ "atMs": 2103.4463749999995,+ "execMs": 0.0006670000148005784}],"timings": {- "gateMs": 1033.9590830000016,+ "gateMs": 1140.7013329999754,"gateSkipped": false,- "ttftMs": 7712.616625000002,- "modelMs": 7109.619416000009,- "toolMs": 0.008374999997613486,- "totalMs": 8143.586874000008,- "judgeMs": 984.9002080000064,+ "ttftMs": 3493.3332919999957,+ "modelMs": 3664.3994999999995,+ "toolMs": 0.0006670000148005784,+ "totalMs": 3666.8260420000297,+ "judgeMs": 1052.9281670000055,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5094.701291000005,+ "roundMs": 2101.538750000007,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.008374999997613486+ "name": "trim_clip",+ "execMs": 0.0006670000148005784}]},{- "roundMs": 2014.9181250000038,- "firstDeltaMs": 1580.2008750000023,+ "roundMs": 1562.8607499999925,+ "firstDeltaMs": 1389.563291999977,"calls": []}]@@ -3507,42 +2820,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "trim_clip","round": 0,- "atMs": 6205.330916999999,- "execMs": 0.0034999999988940544+ "atMs": 1310.7320419999887,+ "execMs": 0.0037089999532327056}],"timings": {- "gateMs": 1184.7039580000055,+ "gateMs": 1044.7667909999727,"gateSkipped": false,- "ttftMs": 11150.112333000005,- "modelMs": 10278.514374999999,- "toolMs": 0.0034999999988940544,- "totalMs": 11463.221833000003,- "judgeMs": 1042.4619999999995,+ "ttftMs": 2418.112124999985,+ "modelMs": 2616.4332499999437,+ "toolMs": 0.0037089999532327056,+ "totalMs": 2620.10958400002,+ "judgeMs": 737.3471249999711,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5019.801041999999,+ "roundMs": 1308.4917499999865,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.0034999999988940544+ "name": "trim_clip",+ "execMs": 0.0037089999532327056}]},{- "roundMs": 5258.713333,- "firstDeltaMs": 4944.258041999994,+ "roundMs": 1307.9414999999572,+ "firstDeltaMs": 1106.2012499999837,"calls": []}]@@ -3551,33 +2864,33 @@]},{- "name": "cartoon-korean-audio-passes-language",+ "name": "transition-ask-styled","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1172,- "p95": 4260,- "mean": 2120,+ "p50": 1052,+ "p95": 1067,+ "mean": 1019,"n": 3},"ttftMs": {- "p50": 14917,- "p95": 16232,- "mean": 13130,+ "p50": 2449,+ "p95": 3099,+ "mean": 2645,"n": 3},"totalMs": {- "p50": 15188,- "p95": 16421,- "mean": 13432,+ "p50": 2761,+ "p95": 3371,+ "mean": 2918,"n": 3},"modelMs": {- "p50": 10927,- "p95": 15248,- "mean": 11312,+ "p50": 2756,+ "p95": 3367,+ "mean": 2913,"n": 3},"toolMs": {@@ -3594,42 +2907,42 @@"runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_transition","round": 0,- "atMs": 12916.456167000004,- "execMs": 0.4334579999995185+ "atMs": 1379.428459000017,+ "execMs": 0.07425000000512227}],"timings": {- "gateMs": 4259.978583000004,+ "gateMs": 1067.2609999999986,"gateSkipped": false,- "ttftMs": 14917.243458000004,- "modelMs": 10927.228750000002,- "toolMs": 0.4334579999995185,- "totalMs": 15187.640791000005,- "judgeMs": 905.3641669999924,+ "ttftMs": 2448.612625000009,+ "modelMs": 2755.9288750000414,+ "toolMs": 0.07425000000512227,+ "totalMs": 2761.4503339999937,+ "judgeMs": 1047.971750000026,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 8653.378042000004,+ "roundMs": 1375.3022080000374,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.4334579999995185+ "name": "set_transition",+ "execMs": 0.07425000000512227}]},{- "roundMs": 2273.850707999998,- "firstDeltaMs": 1998.5725419999944,+ "roundMs": 1380.626667000004,+ "firstDeltaMs": 1068.0642080000252,"calls": []}]@@ -3637,42 +2950,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_transition","round": 0,- "atMs": 6591.181500000006,- "execMs": 0.026041999997687526+ "atMs": 1993.977917000011,+ "execMs": 0.01049999997485429}],"timings": {- "gateMs": 1172.1658329999918,+ "gateMs": 938.2940839999937,"gateSkipped": false,- "ttftMs": 16232.217417000007,- "modelMs": 15248.496459000002,- "toolMs": 0.026041999997687526,- "totalMs": 16420.68833399999,- "judgeMs": 1196.5186669999966,+ "ttftMs": 3098.8815420000465,+ "modelMs": 3367.135375999962,+ "toolMs": 0.01049999997485429,+ "totalMs": 3371.258167000022,+ "judgeMs": 1516.0574170000036,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5416.285833999995,+ "roundMs": 1991.8977919999743,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.026041999997687526+ "name": "set_transition",+ "execMs": 0.01049999997485429}]},{- "roundMs": 9832.210625000007,- "firstDeltaMs": 9636.693041000006,+ "roundMs": 1375.2375839999877,+ "firstDeltaMs": 1103.1051669999724,"calls": []}]@@ -3680,42 +2993,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_transition","round": 0,- "atMs": 6088.948000000004,- "execMs": 0.007417000000714324+ "atMs": 1291.093125000014,+ "execMs": 0.010458000004291534}],"timings": {- "gateMs": 926.8268339999922,+ "gateMs": 1051.9026669999585,"gateSkipped": false,- "ttftMs": 8239.541916999995,- "modelMs": 7760.191665999999,- "toolMs": 0.007417000000714324,- "totalMs": 8687.025916999992,- "judgeMs": 1205.6396669999958,+ "ttftMs": 2386.882291999995,+ "modelMs": 2616.712375000061,+ "toolMs": 0.010458000004291534,+ "totalMs": 2621.8016669999924,+ "judgeMs": 1721.3243750000256,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5159.146458000003,+ "roundMs": 1289.1664590000291,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.007417000000714324+ "name": "set_transition",+ "execMs": 0.010458000004291534}]},{- "roundMs": 2601.045207999996,- "firstDeltaMs": 2150.0598749999917,+ "roundMs": 1327.5459160000319,+ "firstDeltaMs": 1093.680457999988,"calls": []}]@@ -3724,33 +3037,33 @@]},{- "name": "cartoon-style-reference-anchors-look",+ "name": "slide-in-direction-correct","bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 0,- "p95": 1,- "mean": 0,+ "p50": 1090,+ "p95": 1244,+ "mean": 1094,"n": 3},"ttftMs": {- "p50": 13338,- "p95": 15713,- "mean": 12894,+ "p50": 2580,+ "p95": 2706,+ "mean": 2572,"n": 3},"totalMs": {- "p50": 13858,- "p95": 16221,- "mean": 13422,+ "p50": 2781,+ "p95": 2837,+ "mean": 2767,"n": 3},"modelMs": {- "p50": 13857,- "p95": 16221,- "mean": 13422,+ "p50": 2772,+ "p95": 2834,+ "mean": 2762,"n": 3},"toolMs": {@@ -3767,42 +3080,42 @@"runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_animation","round": 0,- "atMs": 8004.275708000001,- "execMs": 0.07395799999358132+ "atMs": 1557.475792000012,+ "execMs": 0.07687499996973202}],"timings": {- "gateMs": 0.6427499999990687,- "gateSkipped": true,- "ttftMs": 13337.51000000001,- "modelMs": 13856.96041700001,- "toolMs": 0.07395799999358132,- "totalMs": 13857.677125000002,- "judgeMs": 1393.2539590000088,+ "gateMs": 1090.030999999959,+ "gateSkipped": false,+ "ttftMs": 2580.360875000013,+ "modelMs": 2772.343874000013,+ "toolMs": 0.07687499996973202,+ "totalMs": 2781.289166999981,+ "judgeMs": 970.5906669999822,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 8000.024875000003,+ "roundMs": 1550.2604580000043,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.07395799999358132+ "name": "set_animation",+ "execMs": 0.07687499996973202}]},{- "roundMs": 5856.9355420000065,- "firstDeltaMs": 5331.7124170000025,+ "roundMs": 1222.0834160000086,+ "firstDeltaMs": 1021.5077910000109,"calls": []}]@@ -3810,42 +3123,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_animation","round": 0,- "atMs": 8398.676292000004,- "execMs": 0.06816599999729078+ "atMs": 1408.6141249999637,+ "execMs": 0.007375000044703484}],"timings": {- "gateMs": 0.1965829999971902,- "gateSkipped": true,- "ttftMs": 15712.965750000018,- "modelMs": 16220.727707999991,- "toolMs": 0.06816599999729078,- "totalMs": 16220.992456999986,- "judgeMs": 1314.6695829999808,+ "gateMs": 947.7063750000088,+ "gateSkipped": false,+ "ttftMs": 2428.2894589999923,+ "modelMs": 2679.0456669999985,+ "toolMs": 0.007375000044703484,+ "totalMs": 2683.5357919999515,+ "judgeMs": 1084.5970409999718,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 8395.28525,+ "roundMs": 1406.1441249999916,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.06816599999729078+ "name": "set_animation",+ "execMs": 0.007375000044703484}]},{- "roundMs": 7825.44245799999,- "firstDeltaMs": 7312.30074999998,+ "roundMs": 1272.9015420000069,+ "firstDeltaMs": 1018.1190829999978,"calls": []}]@@ -3853,42 +3166,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "generate_scene",+ "name": "set_animation","round": 0,- "atMs": 7504.355459000013,- "execMs": 0.017332999996142462+ "atMs": 1452.5358750000014,+ "execMs": 0.009458000015001744}],"timings": {- "gateMs": 0.2917500000039581,- "gateSkipped": true,- "ttftMs": 9631.887584000011,- "modelMs": 10187.376000999997,- "toolMs": 0.017332999996142462,- "totalMs": 10187.685083999997,- "judgeMs": 1441.2837499999732,+ "gateMs": 1243.919166000036,+ "gateSkipped": false,+ "ttftMs": 2705.975124999997,+ "modelMs": 2833.9052499999525,+ "toolMs": 0.009458000015001744,+ "totalMs": 2837.4290829999954,+ "judgeMs": 1071.6321250000037,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 7500.506334000005,+ "roundMs": 1450.3486249999842,"firstDeltaMs": null,"calls": [{- "name": "generate_scene",- "execMs": 0.017332999996142462+ "name": "set_animation",+ "execMs": 0.009458000015001744}]},{- "roundMs": 2686.8696669999918,- "firstDeltaMs": 2125.743208,+ "roundMs": 1383.5566249999683,+ "firstDeltaMs": 1252.307415999996,"calls": []}]@@ -3897,40 +3210,40 @@]},{- "name": "errored-call-not-claimed-done",+ "name": "look-ask-single-tool","bucket": "single-tool",- "passRate": 0.3333333333333333,+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1168,- "p95": 1168,- "mean": 1168,- "n": 1+ "p50": 1063,+ "p95": 1855,+ "mean": 1310,+ "n": 3},"ttftMs": {- "p50": 5130,- "p95": 5130,- "mean": 5130,- "n": 1+ "p50": 3661,+ "p95": 5936,+ "mean": 4186,+ "n": 3},"totalMs": {- "p50": 5873,- "p95": 5873,- "mean": 5873,- "n": 1+ "p50": 4293,+ "p95": 6529,+ "mean": 4638,+ "n": 3},"modelMs": {- "p50": 4704,- "p95": 4704,- "mean": 4704,- "n": 1+ "p50": 3884,+ "p95": 6522,+ "mean": 4498,+ "n": 3},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 1+ "n": 3},"rounds": {"mean": 2,@@ -3939,45 +3252,43 @@},"runs": [{- "pass": false,+ "pass": true,"intent": "simple","roundModel": "gemini-3.1-flash-lite",- "notes": [- "reply ends with a question after completed work"- ],+ "notes": [],"judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "add_effect","round": 0,- "atMs": 5678.251958000008,- "execMs": 2.494957999995677+ "atMs": 4470.417917000013,+ "execMs": 0.1610839999630116}],"timings": {- "gateMs": 1328.9027500000084,+ "gateMs": 1010.8807919999817,"gateSkipped": false,- "ttftMs": 7572.251167000009,- "modelMs": 7079.375583999994,- "toolMs": 2.494957999995677,- "totalMs": 8410.773291999998,- "judgeMs": 1852.2445000000007,+ "ttftMs": 5936.294500000018,+ "modelMs": 6522.251667000004,+ "toolMs": 0.1610839999630116,+ "totalMs": 6529.198625000019,+ "judgeMs": 947.3155830000178,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4343.580875,+ "roundMs": 4465.052416999999,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 2.494957999995677+ "name": "add_effect",+ "execMs": 0.1610839999630116}]},{- "roundMs": 2735.7947089999943,- "firstDeltaMs": 1889.5414170000004,+ "roundMs": 2057.199250000005,+ "firstDeltaMs": 1464.3708750000224,"calls": []}]@@ -3991,82 +3302,79 @@"judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "add_effect","round": 0,- "atMs": 3897.3800829999964,- "execMs": 0.025584000002709217+ "atMs": 1797.435708999983,+ "execMs": 0.01570799999171868}],"timings": {- "gateMs": 1168.4779580000031,+ "gateMs": 1063.0829580000136,"gateSkipped": false,- "ttftMs": 5129.927916999994,- "modelMs": 4704.481707999992,- "toolMs": 0.025584000002709217,- "totalMs": 5872.985249999998,- "judgeMs": 1095.860541999995,+ "ttftMs": 2961.2141249999986,+ "modelMs": 3087.9227079999982,+ "toolMs": 0.01570799999171868,+ "totalMs": 3092.5492919999524,+ "judgeMs": 925.5277090000454,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2726.7607500000013,+ "roundMs": 1795.6965830000117,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.025584000002709217+ "name": "add_effect",+ "execMs": 0.01570799999171868}]},{- "roundMs": 1977.7209579999908,- "firstDeltaMs": 1231.3497079999943,+ "roundMs": 1292.2261249999865,+ "firstDeltaMs": 1161.2092909999774,"calls": []}]}},{- "pass": false,+ "pass": true,"intent": "simple","roundModel": "gemini-3.1-flash-lite",- "notes": [- "banned phrasing: would you like me to",- "reply ends with a question after completed work"- ],- "judgeNote": "failing to execute or explaining a skip clearly, and asking a question instead of being decisive.",+ "notes": [],+ "judgeNote": null,"trace": [{- "name": "set_transition",+ "name": "add_effect","round": 0,- "atMs": 3271.600250000003,- "execMs": 0.007333000001381151+ "atMs": 1855.6511249999749,+ "execMs": 0.03008399996906519}],"timings": {- "gateMs": 1119.0382079999981,+ "gateMs": 1855.242166000011,"gateSkipped": false,- "ttftMs": 4688.491416999997,- "modelMs": 3876.2348330000095,- "toolMs": 0.007333000001381151,- "totalMs": 4995.280374000009,- "judgeMs": 1919.4953749999986,+ "ttftMs": 3660.7609999999986,+ "modelMs": 3884.035750000039,+ "toolMs": 0.03008399996906519,+ "totalMs": 4293.494957999967,+ "judgeMs": 939.6069170000264,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2150.9803329999995,+ "roundMs": 1448.0902080000378,"firstDeltaMs": null,"calls": [{- "name": "set_transition",- "execMs": 0.007333000001381151+ "name": "add_effect",+ "execMs": 0.03008399996906519}]},{- "roundMs": 1725.25450000001,- "firstDeltaMs": 1416.5740000000078,+ "roundMs": 2435.9455420000013,+ "firstDeltaMs": 1803.3277920000255,"calls": []}]@@ -4075,97 +3383,100 @@]},{- "name": "follow-up-do-it-works",- "bucket": "multi-tool",- "passRate": 0.6666666666666666,+ "name": "animation-overrides-transition",+ "bucket": "single-tool",+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 974,- "p95": 1565,- "mean": 1269,- "n": 2+ "p50": 1284,+ "p95": 1661,+ "mean": 1337,+ "n": 3},"ttftMs": {- "p50": 3953,- "p95": 4774,- "mean": 4364,- "n": 2+ "p50": 10267,+ "p95": 11152,+ "mean": 9916,+ "n": 3},"totalMs": {- "p50": 4285,- "p95": 5099,- "mean": 4692,- "n": 2+ "p50": 10588,+ "p95": 11325,+ "mean": 10156,+ "n": 3},"modelMs": {- "p50": 3311,- "p95": 3534,- "mean": 3423,- "n": 2+ "p50": 10581,+ "p95": 11317,+ "mean": 10149,+ "n": 3},"toolMs": {"p50": 0,"p95": 0,"mean": 0,- "n": 2+ "n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 3,+ "max": 3}},"runs": [{- "pass": false,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",- "notes": [- "reply ends with a question after completed work"- ],- "judgeNote": "contains a conversational question instead of being strictly factual about the editing actions.",+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,"trace": [{- "name": "update_cue",- "round": 0,- "atMs": 4349.902083000001,- "execMs": 0.37966599999981554+ "name": "set_transition",+ "round": 1,+ "atMs": 9012.012125000008,+ "execMs": 0.2667500000097789},{- "name": "update_cue",- "round": 0,- "atMs": 4351.348,- "execMs": 0.008750000000873115+ "name": "set_animation",+ "round": 1,+ "atMs": 9013.07925000001,+ "execMs": 0.010084000008646399}],"timings": {- "gateMs": 2033.2585840000002,+ "gateMs": 1660.529749999987,"gateSkipped": false,- "ttftMs": 5563.394166,- "modelMs": 3954.6733750000003,- "toolMs": 0.38841600000068865,- "totalMs": 5988.320375000001,- "judgeMs": 1984.9173330000003,+ "ttftMs": 11152.384875000047,+ "modelMs": 11316.946124999959,+ "toolMs": 0.2768340000184253,+ "totalMs": 11324.970958999998,+ "judgeMs": 1153.9718749999884,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2296.0387500000006,+ "roundMs": 1659.7830829999875,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 7347.547041999991,"firstDeltaMs": null,"calls": [{- "name": "update_cue",- "execMs": 0.37966599999981554+ "name": "set_transition",+ "execMs": 0.2667500000097789},{- "name": "update_cue",- "execMs": 0.008750000000873115+ "name": "set_animation",+ "execMs": 0.010084000008646399}]},{- "roundMs": 1658.6346249999997,- "firstDeltaMs": 1210.815125,+ "roundMs": 2309.61599999998,+ "firstDeltaMs": 2137.259625000006,"calls": []}]@@ -4173,72 +3484,57 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "update_cue",- "round": 0,- "atMs": 2772.4438339999997,- "execMs": 0.032540999998673215- },- {- "name": "update_cue",- "round": 0,- "atMs": 2772.8794589999998,- "execMs": 0.007665999999517226- },- {- "name": "delete_cue",- "round": 0,- "atMs": 2774.2670419999995,- "execMs": 0.07520800000020245+ "name": "set_transition",+ "round": 1,+ "atMs": 7483.321666999953,+ "execMs": 0.02066699997521937},{- "name": "update_cue",- "round": 0,- "atMs": 2774.5324170000004,- "execMs": 0.005874999998923158+ "name": "set_animation",+ "round": 1,+ "atMs": 7483.8680830000085,+ "execMs": 0.008708000008482486}],"timings": {- "gateMs": 973.5534169999992,+ "gateMs": 1065.7632500000182,"gateSkipped": false,- "ttftMs": 3952.7311250000002,- "modelMs": 3311.4413330000007,- "toolMs": 0.12128999999731604,- "totalMs": 4285.116039999997,- "judgeMs": 1936.6611670000002,+ "ttftMs": 10266.502791999956,+ "modelMs": 10581.017125000013,+ "toolMs": 0.029374999983701855,+ "totalMs": 10588.246791999962,+ "judgeMs": 940.5445000000182,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1796.5493750000005,+ "roundMs": 1064.5001249999623,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 6414.402708000038,"firstDeltaMs": null,"calls": [{- "name": "update_cue",- "execMs": 0.032540999998673215- },- {- "name": "update_cue",- "execMs": 0.007665999999517226- },- {- "name": "delete_cue",- "execMs": 0.07520800000020245+ "name": "set_transition",+ "execMs": 0.02066699997521937},{- "name": "update_cue",- "execMs": 0.005874999998923158+ "name": "set_animation",+ "execMs": 0.008708000008482486}]},{- "roundMs": 1514.8919580000002,- "firstDeltaMs": 1176.6964169999992,+ "roundMs": 3102.114292000013,+ "firstDeltaMs": 2780.987167000014,"calls": []}]@@ -4246,72 +3542,57 @@},{"pass": true,- "intent": "simple",- "roundModel": "gemini-3.1-flash-lite",+ "intent": "complex",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "update_cue",- "round": 0,- "atMs": 3660.395124999999,- "execMs": 0.0030829999996058177- },- {- "name": "update_cue",- "round": 0,- "atMs": 3660.474209,- "execMs": 0.0029999999969732016- },- {- "name": "delete_cue",- "round": 0,- "atMs": 3660.513750000002,- "execMs": 0.006833999999798834+ "name": "set_transition",+ "round": 1,+ "atMs": 6226.458625000028,+ "execMs": 0.00712500000372529},{- "name": "update_cue",- "round": 0,- "atMs": 3660.561834,- "execMs": 0.0017920000027515925+ "name": "set_animation",+ "round": 1,+ "atMs": 6226.785499999998,+ "execMs": 0.004167000006418675}],"timings": {- "gateMs": 1565.1637499999997,+ "gateMs": 1284.059917000006,"gateSkipped": false,- "ttftMs": 4774.273000000001,- "modelMs": 3533.5945820000015,- "toolMs": 0.014708999999129446,- "totalMs": 5098.773041,- "judgeMs": 1776.3302500000027,+ "ttftMs": 8328.875416000024,+ "modelMs": 8549.77874999994,+ "toolMs": 0.011292000010143965,+ "totalMs": 8555.71754100005,+ "judgeMs": 1086.4707079999498,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2092.6877910000003,+ "roundMs": 1282.7284169999766,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4939.356832999969,"firstDeltaMs": null,"calls": [{- "name": "update_cue",- "execMs": 0.0030829999996058177- },- {- "name": "update_cue",- "execMs": 0.0029999999969732016- },- {- "name": "delete_cue",- "execMs": 0.006833999999798834+ "name": "set_transition",+ "execMs": 0.00712500000372529},{- "name": "update_cue",- "execMs": 0.0017920000027515925+ "name": "set_animation",+ "execMs": 0.004167000006418675}]},{- "roundMs": 1440.9067910000012,- "firstDeltaMs": 1113.3020830000023,+ "roundMs": 2327.693499999994,+ "firstDeltaMs": 2100.9543750000303,"calls": []}]@@ -4320,529 +3601,314 @@]},{- "name": "photos-request-places-clips",+ "name": "filler-words-cut-with-editor","bucket": "multi-tool",- "passRate": 0.6666666666666666,+ "passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2+ "p50": 1061,+ "p95": 1070,+ "mean": 1046,+ "n": 3},"ttftMs": {- "p50": 15993,- "p95": 119466,- "mean": 67730,- "n": 2+ "p50": 53553,+ "p95": 65238,+ "mean": 56323,+ "n": 3},"totalMs": {- "p50": 16356,- "p95": 119706,- "mean": 68031,- "n": 2+ "p50": 53903,+ "p95": 65509,+ "mean": 56672,+ "n": 3},"modelMs": {- "p50": 16356,- "p95": 119706,- "mean": 68031,- "n": 2+ "p50": 53869,+ "p95": 65475,+ "mean": 56640,+ "n": 3},"toolMs": {- "p50": 0,- "p95": 0,- "mean": 0,- "n": 2+ "p50": 1,+ "p95": 2,+ "mean": 1,+ "n": 3},"rounds": {- "mean": 17.5,- "max": 31+ "mean": 17,+ "max": 19}},"runs": [{- "pass": false,+ "pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [- "banned phrasing: let me know if"- ],- "judgeNote": "conversational tone but editing tool task only partially completed.",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,"trace": [{- "name": "add_clip",- "round": 0,- "atMs": 9365.163709,- "execMs": 0.005541999998968095- },- {- "name": "add_clip",- "round": 0,- "atMs": 9365.842166999999,- "execMs": 0.0026670000006561168- },- {- "name": "get_state",+ "name": "list_skills","round": 1,- "atMs": 11869.745208999997,- "execMs": 0.08970800000679446+ "atMs": 4498.100333000009,+ "execMs": 0.06716600002255291},{- "name": "get_state",+ "name": "read_skill","round": 2,- "atMs": 14749.093583999998,- "execMs": 0.026583999999274965+ "atMs": 6580.541124999989,+ "execMs": 0.003874999994877726},{- "name": "add_clip",+ "name": "detect_silence","round": 3,- "atMs": 17343.954999999998,- "execMs": 0.004000000000814907+ "atMs": 8704.500292000012,+ "execMs": 0.011499999964144081},{- "name": "add_clip",+ "name": "detect_silence","round": 4,- "atMs": 19506.157834,- "execMs": 0.0015839999978197739+ "atMs": 13210.601000000024,+ "execMs": 0.017832999990787357},{- "name": "get_state",+ "name": "listen_audio","round": 5,- "atMs": 21685.913041999996,- "execMs": 0.003165999994962476+ "atMs": 15508.043082999997,+ "execMs": 0.015167000005021691},{- "name": "add_clip",+ "name": "get_state","round": 6,- "atMs": 24268.873959,- "execMs": 0.0062499999985448085+ "atMs": 18906.72658299998,+ "execMs": 0.5958330000285059},{- "name": "get_state",+ "name": "watch_video","round": 7,- "atMs": 26143.933291999998,- "execMs": 0.01583299999765586+ "atMs": 21188.934458000003,+ "execMs": 0.00849999999627471},{- "name": "add_clip",+ "name": "split_at","round": 8,- "atMs": 30329.573958999998,- "execMs": 0.005417000000306871+ "atMs": 30418.627625000023,+ "execMs": 0.6585830000112765},{- "name": "get_state",+ "name": "split_at","round": 9,- "atMs": 32705.758208999996,- "execMs": 0.027792000000772532+ "atMs": 35752.72479200002,+ "execMs": 0.08879100001649931},{- "name": "get_state",+ "name": "delete_item","round": 10,- "atMs": 35970.680834,- "execMs": 0.01916699999856064+ "atMs": 38183.098624999984,+ "execMs": 0.13295800000196323},{- "name": "add_clip",+ "name": "refine_speech_cuts","round": 11,- "atMs": 38458.996084,- "execMs": 0.007041000004392117+ "atMs": 40802.824291999976,+ "execMs": 0.0783750000409782},{- "name": "add_clip",+ "name": "get_state","round": 12,- "atMs": 41768.666417,- "execMs": 0.00516599999536993+ "atMs": 43427.77137500001,+ "execMs": 0.06870799994794652},{- "name": "get_state",+ "name": "delete_cue","round": 13,- "atMs": 44489.186,- "execMs": 0.01583299999765586+ "atMs": 46768.41295799997,+ "execMs": 0.021166999998968095},{- "name": "add_clip",+ "name": "update_cue","round": 14,- "atMs": 48395.494167,- "execMs": 0.004249999998137355+ "atMs": 48999.416333,+ "execMs": 0.009916999959386885},{- "name": "get_state",+ "name": "update_cue","round": 15,- "atMs": 50938.076084,- "execMs": 0.01245799999742303- },- {- "name": "get_state",- "round": 16,- "atMs": 53156.87449999999,- "execMs": 0.005374999993364327- },- {- "name": "get_state",- "round": 17,- "atMs": 56771.60133399999,- "execMs": 0.008916999999200925- },- {- "name": "add_clip",- "round": 18,- "atMs": 60906.756459,- "execMs": 0.0048749999987194315- },- {- "name": "get_state",- "round": 19,- "atMs": 67288.815209,- "execMs": 0.014083000001846813- },- {- "name": "add_clip",- "round": 20,- "atMs": 69495.08629199999,- "execMs": 0.0017920000100275502- },- {- "name": "get_state",- "round": 21,- "atMs": 78613.232209,- "execMs": 0.017041999992216006- },- {- "name": "get_state",- "round": 22,- "atMs": 94290.09375,- "execMs": 0.012583000003360212- },- {- "name": "get_state",- "round": 23,- "atMs": 107840.301417,- "execMs": 0.009541999999783002- },- {- "name": "add_clip",- "round": 24,- "atMs": 116847.97025,- "execMs": 0.008792000007815659- },- {- "name": "get_state",- "round": 25,- "atMs": 125978.60858399999,- "execMs": 0.01345900000887923- },- {- "name": "add_clip",- "round": 26,- "atMs": 138560.886084,- "execMs": 0.004833999992115423- },- {- "name": "add_clip",- "round": 27,- "atMs": 140968.45450000002,- "execMs": 0.007499999977881089+ "atMs": 51278.89516700001,+ "execMs": 0.008333000005222857}],"timings": {- "gateMs": 1.1879169999992882,- "gateSkipped": true,- "ttftMs": 144245.64929200002,- "modelMs": 144549.141544,- "toolMs": 0.36125199997331947,- "totalMs": 144550.690713,- "judgeMs": 1231.1373749999912,+ "gateMs": 1006.983917000005,+ "gateSkipped": false,+ "ttftMs": 53553.38274999999,+ "modelMs": 53869.068457000016,+ "toolMs": 1.786705999984406,+ "totalMs": 53903.30433299998,+ "judgeMs": 1057.4297919999808,"retryMs": 0,"retries": 0,- "rounds": [- {- "roundMs": 9354.848915999999,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.005541999998968095- },- {- "name": "add_clip",- "execMs": 0.0026670000006561168- }- ]- },- {- "roundMs": 2500.9542089999995,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.08970800000679446- }- ]- },- {- "roundMs": 2877.726499999997,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.026583999999274965- }- ]- },- {- "roundMs": 2593.068625,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.004000000000814907- }- ]- },- {- "roundMs": 2160.6204999999973,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.0015839999978197739- }- ]- },- {- "roundMs": 2179.226834000001,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.003165999994962476- }- ]- },- {- "roundMs": 2582.1612499999974,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.0062499999985448085- }- ]- },- {- "roundMs": 1873.1346660000054,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.01583299999765586- }- ]- },- {- "roundMs": 4183.652332999998,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.005417000000306871- }- ]- },- {- "roundMs": 2373.120541999997,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.027792000000772532- }- ]- },- {- "roundMs": 3261.6688749999958,- "firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.01916699999856064- }- ]- },- {- "roundMs": 2485.8957500000033,- "firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.007041000004392117- }- ]- },+ "rounds": [{- "roundMs": 3307.974583000003,+ "roundMs": 1005.463665999996,"firstDeltaMs": null,- "calls": [- {- "name": "add_clip",- "execMs": 0.00516599999536993- }- ]+ "calls": []},{- "roundMs": 2718.184208000006,+ "roundMs": 3487.1161250000005,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.01583299999765586+ "name": "list_skills",+ "execMs": 0.06716600002255291}]},{- "roundMs": 3903.3641669999924,+ "roundMs": 2080.839500000002,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.004249999998137355+ "name": "read_skill",+ "execMs": 0.003874999994877726}]},{- "roundMs": 2539.9696669999976,+ "roundMs": 2122.9190410000156,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.01245799999742303+ "name": "detect_silence",+ "execMs": 0.011499999964144081}]},{- "roundMs": 2216.2216670000053,+ "roundMs": 4505.259791999997,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.005374999993364327+ "name": "detect_silence",+ "execMs": 0.017832999990787357}]},{- "roundMs": 3613.9260000000068,+ "roundMs": 2294.869000000006,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.008916999999200925+ "name": "listen_audio",+ "execMs": 0.015167000005021691}]},{- "roundMs": 4131.619416999994,+ "roundMs": 3396.9686670000083,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0048749999987194315+ "name": "get_state",+ "execMs": 0.5958330000285059}]},{- "roundMs": 6379.290792,+ "roundMs": 2278.8691249999683,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.014083000001846813+ "name": "watch_video",+ "execMs": 0.00849999999627471}]},{- "roundMs": 2203.8260840000003,+ "roundMs": 9226.810332999972,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0017920000100275502+ "name": "split_at",+ "execMs": 0.6585830000112765}]},{- "roundMs": 9116.870875000008,+ "roundMs": 5330.670750000048,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.017041999992216006+ "name": "split_at",+ "execMs": 0.08879100001649931}]},{- "roundMs": 15674.640417000002,+ "roundMs": 2428.1704579999787,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.012583000003360212+ "name": "delete_item",+ "execMs": 0.13295800000196323}]},{- "roundMs": 13546.471165999988,+ "roundMs": 2617.4469590000226,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.009541999999783002+ "name": "refine_speech_cuts",+ "execMs": 0.0783750000409782}]},{- "roundMs": 9004.178334000011,+ "roundMs": 2623.427250000008,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.008792000007815659+ "name": "get_state",+ "execMs": 0.06870799994794652}]},{- "roundMs": 9126.41066600001,+ "roundMs": 3338.361499999999,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.01345900000887923+ "name": "delete_cue",+ "execMs": 0.021166999998968095}]},{- "roundMs": 12579.425334,+ "roundMs": 2229.974125000008,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.004833999992115423+ "name": "update_cue",+ "execMs": 0.009916999959386885}]},{- "roundMs": 2403.5741670000134,+ "roundMs": 2278.8755409999867,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.007499999977881089+ "name": "update_cue",+ "execMs": 0.008333000005222857}]},{- "roundMs": 3657.1149999999907,- "firstDeltaMs": 3274.4027500000084,+ "roundMs": 2623.0266249999986,+ "firstDeltaMs": 2273.851040999987,"calls": []}]@@ -4851,620 +3917,715 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{- "name": "add_clip",- "round": 0,- "atMs": 6347.024291000009,- "execMs": 0.010749999986728653- },- {- "name": "add_clip",- "round": 0,- "atMs": 6347.542916000006,- "execMs": 0.003542000020388514- },- {- "name": "get_state",+ "name": "read_skill","round": 1,- "atMs": 8630.411083000014,- "execMs": 0.02733400001307018+ "atMs": 4444.065416999976,+ "execMs": 0.05087500001536682},{- "name": "add_clip",+ "name": "detect_silence","round": 2,- "atMs": 12486.632083000004,- "execMs": 0.00812499999301508+ "atMs": 7394.844541999977,+ "execMs": 0.03141699999105185},{- "name": "add_clip",+ "name": "listen_audio","round": 3,- "atMs": 14495.920332999987,- "execMs": 0.0072499999951105565+ "atMs": 9648.370166999986,+ "execMs": 0.009833999967668205},{"name": "get_state","round": 4,- "atMs": 16405.490208000003,- "execMs": 0.011207999981706962+ "atMs": 13734.405291999981,+ "execMs": 0.11616600002162158},{- "name": "add_clip",+ "name": "detect_silence","round": 5,- "atMs": 19207.112708,- "execMs": 0.021417000010842457+ "atMs": 16470.391999999993,+ "execMs": 0.027000000001862645},{- "name": "get_state",+ "name": "split_at","round": 6,- "atMs": 20904.51225,- "execMs": 0.020541999983834103+ "atMs": 27509.322291999997,+ "execMs": 0.18245800002478063},{- "name": "add_clip",+ "name": "split_at","round": 7,- "atMs": 23616.670625,- "execMs": 0.018041000002995133+ "atMs": 29699.89966699999,+ "execMs": 0.028833999997004867},{- "name": "get_state",+ "name": "delete_item","round": 8,- "atMs": 26455.49966599999,- "execMs": 0.011291999981040135+ "atMs": 32497.72758299997,+ "execMs": 0.17475000000558794},{- "name": "add_clip",+ "name": "get_state","round": 9,- "atMs": 29737.228457999998,- "execMs": 0.005541999998968095+ "atMs": 34591.16470799997,+ "execMs": 0.0491660000407137},{- "name": "add_clip",+ "name": "delete_cue","round": 10,- "atMs": 32356.517083000013,- "execMs": 0.003167000017128885+ "atMs": 37737.48195799999,+ "execMs": 0.06154099997365847},{- "name": "get_state",+ "name": "update_cue","round": 11,- "atMs": 34265.730583,- "execMs": 0.009916999988490716+ "atMs": 43047.10120799998,+ "execMs": 0.02549999998882413},{- "name": "add_clip",+ "name": "update_cue","round": 12,- "atMs": 36945.80374999999,- "execMs": 0.004125000006752089+ "atMs": 45603.681667,+ "execMs": 0.053124999976716936},{"name": "get_state","round": 13,- "atMs": 39213.31774999999,- "execMs": 0.006250000005820766- },- {- "name": "add_clip",- "round": 14,- "atMs": 42105.20529099999,- "execMs": 0.0029999999969732016- },- {- "name": "get_state",- "round": 15,- "atMs": 47365.30050000001,- "execMs": 0.012291999999433756- },- {- "name": "add_clip",- "round": 16,- "atMs": 50709.069208,- "execMs": 0.0027089999930467457- },- {- "name": "add_clip",- "round": 17,- "atMs": 53436.82837500001,- "execMs": 0.004917000012937933- },- {- "name": "get_state",- "round": 18,- "atMs": 56165.44862499999,- "execMs": 0.013333999988390133- },- {- "name": "add_clip",- "round": 19,- "atMs": 61446.066374999995,- "execMs": 0.004041000007418916- },- {- "name": "add_clip",- "round": 20,- "atMs": 63683.539166,- "execMs": 0.0020409999997355044- },- {- "name": "get_state",- "round": 21,- "atMs": 65870.680208,- "execMs": 0.0034580000210553408- },- {- "name": "add_clip",- "round": 22,- "atMs": 69925.819583,- "execMs": 0.0030830000177957118- },- {- "name": "add_clip",- "round": 22,- "atMs": 69926.04358299999,- "execMs": 0.0014999999839346856- },- {- "name": "get_state",- "round": 23,- "atMs": 76665.128333,- "execMs": 0.010332999983802438- },- {- "name": "add_clip",- "round": 24,- "atMs": 84464.85920800001,- "execMs": 0.002542000001994893- },- {- "name": "get_state",- "round": 25,- "atMs": 86533.62325,- "execMs": 0.005332999979145825- },- {- "name": "add_clip",- "round": 26,- "atMs": 97622.57716599997,- "execMs": 0.0007919999770820141- },- {- "name": "get_state",- "round": 27,- "atMs": 99943.1445,- "execMs": 0.004458000010345131- },- {- "name": "get_state",- "round": 28,- "atMs": 104005.40270799998,- "execMs": 0.012790999957360327- },- {- "name": "set_view",- "round": 29,- "atMs": 114179.36791600002,- "execMs": 0.01616699999431148+ "atMs": 48000.706957999966,+ "execMs": 0.0360829999553971}],"timings": {- "gateMs": 0.36574999999720603,- "gateSkipped": true,- "ttftMs": 119466.47324999998,- "modelMs": 119705.54787300009,- "toolMs": 0.27129299991065636,- "totalMs": 119706.184916,- "judgeMs": 1471.4382079999777,+ "gateMs": 1061.0017920000246,+ "gateSkipped": false,+ "ttftMs": 50177.316207999946,+ "modelMs": 50575.90291500016,+ "toolMs": 0.8467489999602549,+ "totalMs": 50602.51908299996,+ "judgeMs": 1330.2250830000266,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6343.533332999999,+ "roundMs": 1058.8056250000373,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3378.861042000004,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.010749999986728653- },- {- "name": "add_clip",- "execMs": 0.003542000020388514+ "name": "read_skill",+ "execMs": 0.05087500001536682}]},{- "roundMs": 2280.138875000004,+ "roundMs": 2949.0402079999913,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.02733400001307018+ "name": "detect_silence",+ "execMs": 0.03141699999105185}]},{- "roundMs": 3854.272666999983,+ "roundMs": 2252.1165829999954,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.00812499999301508+ "name": "listen_audio",+ "execMs": 0.009833999967668205}]},{- "roundMs": 2006.8395000000019,+ "roundMs": 4085.306332999957,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0072499999951105565+ "name": "get_state",+ "execMs": 0.11616600002162158}]},{- "roundMs": 1907.2319580000185,+ "roundMs": 2734.2989170000073,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.011207999981706962+ "name": "detect_silence",+ "execMs": 0.027000000001862645}]},{- "roundMs": 2799.3657500000263,+ "roundMs": 11037.327625000034,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.021417000010842457+ "name": "split_at",+ "execMs": 0.18245800002478063}]},{- "roundMs": 1695.0663749999949,+ "roundMs": 2189.2580830000225,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.020541999983834103+ "name": "split_at",+ "execMs": 0.028833999997004867}]},{- "roundMs": 2710.883459000004,+ "roundMs": 2796.473667000013,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.018041000002995133+ "name": "delete_item",+ "execMs": 0.17475000000558794}]},{- "roundMs": 2836.3477910000074,+ "roundMs": 2091.561250000028,"firstDeltaMs": null,"calls": [{"name": "get_state",- "execMs": 0.011291999981040135+ "execMs": 0.0491660000407137}]},{- "roundMs": 3278.8874589999905,+ "roundMs": 3145.4002080000355,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.005541999998968095+ "name": "delete_cue",+ "execMs": 0.06154099997365847}]},{- "roundMs": 2617.1875839999993,+ "roundMs": 5307.8068329999805,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.003167000017128885+ "name": "update_cue",+ "execMs": 0.02549999998882413}]},{- "roundMs": 1907.0659159999923,+ "roundMs": 2554.7003330000443,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.009916999988490716+ "name": "update_cue",+ "execMs": 0.053124999976716936}]},{- "roundMs": 2677.630165999988,+ "roundMs": 2395.0533750000177,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.004125000006752089+ "name": "get_state",+ "execMs": 0.0360829999553971}]},{- "roundMs": 2265.273917000013,+ "roundMs": 2599.892832999991,+ "firstDeltaMs": 2175.588999999978,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "list_skills",+ "round": 1,+ "atMs": 4246.829833999975,+ "execMs": 0.09020899992901832+ },+ {+ "name": "read_skill",+ "round": 2,+ "atMs": 6334.007958999951,+ "execMs": 0.005291000008583069+ },+ {+ "name": "read_skill",+ "round": 3,+ "atMs": 8476.747208999936,+ "execMs": 0.015750000020489097+ },+ {+ "name": "detect_silence",+ "round": 4,+ "atMs": 11867.053375000018,+ "execMs": 0.04704099998343736+ },+ {+ "name": "watch_video",+ "round": 5,+ "atMs": 14466.433709000004,+ "execMs": 0.02779199997894466+ },+ {+ "name": "listen_audio",+ "round": 6,+ "atMs": 17785.82033399993,+ "execMs": 0.016125000081956387+ },+ {+ "name": "get_state",+ "round": 7,+ "atMs": 19793.918916999944,+ "execMs": 0.16170799999963492+ },+ {+ "name": "detect_silence",+ "round": 8,+ "atMs": 26398.46345899999,+ "execMs": 0.019040999934077263+ },+ {+ "name": "split_at",+ "round": 9,+ "atMs": 37913.40287500003,+ "execMs": 0.23733299993909895+ },+ {+ "name": "split_at",+ "round": 10,+ "atMs": 40390.835542000015,+ "execMs": 0.1254590000025928+ },+ {+ "name": "delete_item",+ "round": 11,+ "atMs": 44382.01787500002,+ "execMs": 0.1422499999171123+ },+ {+ "name": "get_state",+ "round": 12,+ "atMs": 46432.170334000024,+ "execMs": 0.029749999986961484+ },+ {+ "name": "delete_cue",+ "round": 13,+ "atMs": 53744.85808399995,+ "execMs": 0.0625+ },+ {+ "name": "update_cue",+ "round": 14,+ "atMs": 56224.95887500001,+ "execMs": 0.07187500002328306+ },+ {+ "name": "update_cue",+ "round": 15,+ "atMs": 58541.19062499993,+ "execMs": 0.017250000033527613+ },+ {+ "name": "update_cue",+ "round": 16,+ "atMs": 60848.77566699998,+ "execMs": 0.02083399996627122+ },+ {+ "name": "refine_speech_cuts",+ "round": 17,+ "atMs": 63129.769458999974,+ "execMs": 0.028833999997004867+ }+ ],+ "timings": {+ "gateMs": 1070.3639589999802,+ "gateSkipped": false,+ "ttftMs": 65238.285959,+ "modelMs": 65474.505917,+ "toolMs": 1.1190419998019934,+ "totalMs": 65509.375459,+ "judgeMs": 1131.8673750000307,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1068.6005839999998,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3173.6268329999875,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.006250000005820766+ "name": "list_skills",+ "execMs": 0.09020899992901832}]},{- "roundMs": 2890.6446660000074,+ "roundMs": 2085.2153330000583,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0029999999969732016+ "name": "read_skill",+ "execMs": 0.005291000008583069}]},{- "roundMs": 5257.402124999993,+ "roundMs": 2141.980082999915,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.012291999999433756+ "name": "read_skill",+ "execMs": 0.015750000020489097}]},{- "roundMs": 3338.358790999977,+ "roundMs": 3388.0405830000527,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0027089999930467457+ "name": "detect_silence",+ "execMs": 0.04704099998343736}]},{- "roundMs": 2724.468332999997,+ "roundMs": 2597.6013340000063,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.004917000012937933+ "name": "watch_video",+ "execMs": 0.02779199997894466}]},{- "roundMs": 2726.3109580000164,+ "roundMs": 3317.484000000055,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.013333999988390133+ "name": "listen_audio",+ "execMs": 0.016125000081956387}]},{- "roundMs": 5278.743542000011,+ "roundMs": 2006.9830410000868,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.004041000007418916+ "name": "get_state",+ "execMs": 0.16170799999963492}]},{- "roundMs": 2235.5099170000176,+ "roundMs": 6602.819708000054,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0020409999997355044+ "name": "detect_silence",+ "execMs": 0.019040999934077263}]},{- "roundMs": 2186.215082999988,+ "roundMs": 11512.806333999964,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.0034580000210553408+ "name": "split_at",+ "execMs": 0.23733299993909895}]},{- "roundMs": 4054.366416999983,+ "roundMs": 2474.452667000005,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0030830000177957118- },- {- "name": "add_clip",- "execMs": 0.0014999999839346856+ "name": "split_at",+ "execMs": 0.1254590000025928}]},{- "roundMs": 6737.167083000008,+ "roundMs": 3988.8979579999577,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.010332999983802438+ "name": "delete_item",+ "execMs": 0.1422499999171123}]},{- "roundMs": 7797.56700000001,+ "roundMs": 2047.7404579999857,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.002542000001994893+ "name": "get_state",+ "execMs": 0.029749999986961484}]},{- "roundMs": 2067.4492079999764,+ "roundMs": 7311.247417000006,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.005332999979145825+ "name": "delete_cue",+ "execMs": 0.0625}]},{- "roundMs": 11088.198125000024,+ "roundMs": 2477.83670799993,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.0007919999770820141+ "name": "update_cue",+ "execMs": 0.07187500002328306}]},{- "roundMs": 2319.9196249999804,+ "roundMs": 2313.7306660000468,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.004458000010345131+ "name": "update_cue",+ "execMs": 0.017250000033527613}]},{- "roundMs": 4061.178292000026,+ "roundMs": 2306.797333999886,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.012790999957360327+ "name": "update_cue",+ "execMs": 0.02083399996627122}]},{- "roundMs": 10168.111583000049,+ "roundMs": 2279.97341700003,"firstDeltaMs": null,"calls": [{- "name": "set_view",- "execMs": 0.01616699999431148+ "name": "refine_speech_cuts",+ "execMs": 0.028833999997004867}]},{- "roundMs": 5594.212375000003,- "firstDeltaMs": 5285.640000000014,+ "roundMs": 2378.6714589999756,+ "firstDeltaMs": 2107.8910840000026,"calls": []}]}+ }+ ]+ },+ {+ "name": "edit-request-still-uses-tools",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 1021,+ "p95": 1053,+ "mean": 1014,+ "n": 3+ },+ "ttftMs": {+ "p50": 3256,+ "p95": 6771,+ "mean": 4334,+ "n": 3+ },+ "totalMs": {+ "p50": 4059,+ "p95": 6999,+ "mean": 4885,+ "n": 3},+ "modelMs": {+ "p50": 4055,+ "p95": 6992,+ "mean": 4880,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": "mentions a beach video not in the prompt, creating confusion about the scope of the edit.",+ "judgeNote": null,"trace": [{- "name": "add_clip",- "round": 0,- "atMs": 6414.751958000008,- "execMs": 0.006333000026643276- },- {- "name": "add_clip",+ "name": "subtitles_generate","round": 0,- "atMs": 6415.403292000003,- "execMs": 0.012875000014901161- },- {- "name": "get_state",- "round": 1,- "atMs": 9707.97445800004,- "execMs": 0.04666599998017773- },- {- "name": "add_clip",- "round": 2,- "atMs": 14027.780625000014,- "execMs": 0.003375000029336661- },- {- "name": "add_clip",- "round": 2,- "atMs": 14027.976375000027,- "execMs": 0.00041699997382238507+ "atMs": 5359.881500000018,+ "execMs": 0.013957999995909631}],"timings": {- "gateMs": 0.2606670000241138,- "gateSkipped": true,- "ttftMs": 15992.797333000053,- "modelMs": 16356.092165999988,- "toolMs": 0.06966600002488121,- "totalMs": 16356.422499000037,- "judgeMs": 1972.9069160000072,+ "gateMs": 1052.841625000001,+ "gateSkipped": false,+ "ttftMs": 6770.96875,+ "modelMs": 6991.631916000042,+ "toolMs": 0.013957999995909631,+ "totalMs": 6998.731334000011,+ "judgeMs": 1078.5364590000827,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6411.362291000027,+ "roundMs": 5354.25225000002,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.006333000026643276- },- {- "name": "add_clip",- "execMs": 0.012875000014901161+ "name": "subtitles_generate",+ "execMs": 0.013957999995909631}]},{- "roundMs": 3290.572999999975,+ "roundMs": 1637.3796660000226,+ "firstDeltaMs": 1410.0280829999829,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": "did not state what changed in concrete terms (e.g., style, placement, or language) and ignored the approval gate for a paid task.",+ "trace": [+ {+ "name": "subtitles_generate",+ "round": 0,+ "atMs": 1798.107874999987,+ "execMs": 0.029666999937035143+ }+ ],+ "timings": {+ "gateMs": 1021.3043750000652,+ "gateSkipped": false,+ "ttftMs": 3255.7909590000054,+ "modelMs": 4055.1263330001384,+ "toolMs": 0.029666999937035143,+ "totalMs": 4059.0133749999804,+ "judgeMs": 1066.701207999955,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1795.7172500001034,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.04666599998017773+ "name": "subtitles_generate",+ "execMs": 0.029666999937035143}]},{- "roundMs": 4317.821999999986,+ "roundMs": 2259.409083000035,+ "firstDeltaMs": 1456.2615830000723,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "subtitles_generate",+ "round": 0,+ "atMs": 1513.4303340000333,+ "execMs": 0.004458000068552792+ }+ ],+ "timings": {+ "gateMs": 967.498249999946,+ "gateSkipped": false,+ "ttftMs": 2976.6096670000115,+ "modelMs": 3592.335834000027,+ "toolMs": 0.004458000068552792,+ "totalMs": 3597.6535419999855,+ "judgeMs": 1646.90295899997,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1510.32491700002,"firstDeltaMs": null,"calls": [{- "name": "add_clip",- "execMs": 0.003375000029336661- },- {- "name": "add_clip",- "execMs": 0.00041699997382238507+ "name": "subtitles_generate",+ "execMs": 0.004458000068552792}]},{- "roundMs": 2336.3348750000005,- "firstDeltaMs": 1963.407499999972,+ "roundMs": 2082.010917000007,+ "firstDeltaMs": 1461.0561669999734,"calls": []}]@@ -5473,275 +4634,186 @@]},{- "name": "filler-words-cut-with-editor",- "bucket": "multi-tool",+ "name": "cartoon-from-audio-plans-scene",+ "bucket": "single-tool","passRate": 1,"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1060,- "p95": 1258,- "mean": 1106,+ "p50": 1657,+ "p95": 1683,+ "mean": 1483,"n": 3},"ttftMs": {- "p50": 66488,- "p95": 75411,- "mean": 64297,+ "p50": 16261,+ "p95": 18283,+ "mean": 15839,"n": 3},"totalMs": {- "p50": 67010,- "p95": 75777,- "mean": 64714,+ "p50": 16492,+ "p95": 18673,+ "mean": 16164,"n": 3},"modelMs": {- "p50": 66010,- "p95": 74516,- "mean": 63607,+ "p50": 16482,+ "p95": 18660,+ "mean": 16154,"n": 3},"toolMs": {- "p50": 1,- "p95": 3,- "mean": 1,+ "p50": 0,+ "p95": 0,+ "mean": 0,"n": 3},"rounds": {- "mean": 13,- "max": 14+ "mean": 5,+ "max": 6}},"runs": [{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [- {- "name": "list_skills",- "round": 0,- "atMs": 5580.8372500000405,- "execMs": 0.8649579999619164- },{"name": "read_skill","round": 1,- "atMs": 7816.184083,- "execMs": 0.0051250000251457095+ "atMs": 5680.2327499999665,+ "execMs": 0.21808300004340708},{- "name": "detect_silence",+ "name": "listen_audio","round": 2,- "atMs": 10350.719125000003,- "execMs": 0.02345799998147413+ "atMs": 9768.099667000002,+ "execMs": 0.03637499990873039},{- "name": "listen_audio",+ "name": "get_state","round": 3,- "atMs": 14999.313291000028,- "execMs": 0.018333999963942915+ "atMs": 12000.085707999999,+ "execMs": 0.007291999994777143},{- "name": "detect_silence",+ "name": "generate_scene","round": 4,- "atMs": 19877.292790999985,- "execMs": 0.020415999984834343- },- {- "name": "watch_video",- "round": 5,- "atMs": 22609.705083000008,- "execMs": 0.018708000017795712- },- {- "name": "split_at",- "round": 6,- "atMs": 31923.090833000024,- "execMs": 0.6783330000471324- },- {- "name": "split_at",- "round": 6,- "atMs": 31924.24866600003,- "execMs": 0.04891700000735- },- {- "name": "split_at",- "round": 6,- "atMs": 31924.490583000006,- "execMs": 0.01829199999338016- },- {- "name": "delete_item",- "round": 7,- "atMs": 44025.13024999999,- "execMs": 0.2165839999797754- },- {- "name": "delete_item",- "round": 7,- "atMs": 44025.69500000001,- "execMs": 0.07304200000362471- },- {- "name": "refine_speech_cuts",- "round": 8,- "atMs": 47971.268041,- "execMs": 0.05191699997521937- },- {- "name": "get_state",- "round": 9,- "atMs": 63652.130041000026,- "execMs": 0.496458999987226- },- {- "name": "subtitles_generate",- "round": 10,- "atMs": 71592.575541,- "execMs": 0.051334000017959625+ "atMs": 14900.848832999938,+ "execMs": 0.19137499993667006}],"timings": {- "gateMs": 1258.1774580000201,+ "gateMs": 1682.6600409999955,"gateSkipped": false,- "ttftMs": 75410.774875,- "modelMs": 74516.291042,- "toolMs": 2.5858769999467768,- "totalMs": 75777.05437699996,- "judgeMs": 1622.744458000001,+ "ttftMs": 18282.558332999935,+ "modelMs": 18660.183126000105,+ "toolMs": 0.4531249998835847,+ "totalMs": 18672.514166999958,+ "judgeMs": 1354.4317499999888,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4318.188334000006,- "firstDeltaMs": null,- "calls": [- {- "name": "list_skills",- "execMs": 0.8649579999619164- }- ]- },- {- "roundMs": 2232.2167079999927,- "firstDeltaMs": null,- "calls": [- {- "name": "read_skill",- "execMs": 0.0051250000251457095- }- ]- },- {- "roundMs": 2530.8165000000154,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.02345799998147413- }- ]- },- {- "roundMs": 4645.790042000008,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.018333999963942915- }- ]- },- {- "roundMs": 4876.580292000028,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.020415999984834343- }- ]- },- {- "roundMs": 2728.2512080000015,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.018708000017795712- }- ]- },- {- "roundMs": 9310.074415999989,+ "roundMs": 1681.970083000022,"firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.6783330000471324- },- {- "name": "split_at",- "execMs": 0.04891700000735- },+ "calls": []+ },+ {+ "roundMs": 3995.4948330000043,+ "firstDeltaMs": null,+ "calls": [{- "name": "split_at",- "execMs": 0.01829199999338016+ "name": "read_skill",+ "execMs": 0.21808300004340708}]},{- "roundMs": 12095.991749999986,+ "roundMs": 4085.7516250000335,"firstDeltaMs": null,"calls": [{- "name": "delete_item",- "execMs": 0.2165839999797754- },- {- "name": "delete_item",- "execMs": 0.07304200000362471+ "name": "listen_audio",+ "execMs": 0.03637499990873039}]},{- "roundMs": 3942.437208999996,+ "roundMs": 2230.5004590000026,"firstDeltaMs": null,"calls": [{- "name": "refine_speech_cuts",- "execMs": 0.05191699997521937+ "name": "get_state",+ "execMs": 0.007291999994777143}]},{- "roundMs": 15679.742957999988,+ "roundMs": 2897.1106669999426,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.496458999987226+ "name": "generate_scene",+ "execMs": 0.19137499993667006}]},{- "roundMs": 7934.565833000001,+ "roundMs": 3769.3554590001004,+ "firstDeltaMs": 3379.739459000062,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 1,+ "atMs": 9186.846166999894,+ "execMs": 0.019582999986596406+ }+ ],+ "timings": {+ "gateMs": 1656.5625419999706,+ "gateSkipped": false,+ "ttftMs": 12974.318417000002,+ "modelMs": 13319.911041999934,+ "toolMs": 0.019582999986596406,+ "totalMs": 13326.079707999947,+ "judgeMs": 1114.840833999915,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1655.3011249999981,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 7527.293166999938,"firstDeltaMs": null,"calls": [{- "name": "subtitles_generate",- "execMs": 0.051334000017959625+ "name": "generate_scene",+ "execMs": 0.019582999986596406}]},{- "roundMs": 4221.635791999986,- "firstDeltaMs": 3816.1922919999924,+ "roundMs": 4137.316749999998,+ "firstDeltaMs": 3785.8773749999236,"calls": []}]@@ -5750,233 +4822,186 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [- {- "name": "list_skills",- "round": 0,- "atMs": 6926.977458999958,- "execMs": 0.07491700001992285- },{"name": "read_skill","round": 1,- "atMs": 12566.700124999974,- "execMs": 0.020542000012937933+ "atMs": 6067.227124999976,+ "execMs": 0.01508399995509535},{- "name": "detect_silence",+ "name": "listen_audio","round": 2,- "atMs": 15696.67966699996,- "execMs": 0.03437500004656613+ "atMs": 8535.476124999928,+ "execMs": 0.00779200007673353},{- "name": "listen_audio",+ "name": "get_state","round": 3,- "atMs": 22775.795291999995,- "execMs": 0.020040999981574714+ "atMs": 11179.066999999923,+ "execMs": 0.016416999977082014},{- "name": "watch_video",+ "name": "generate_scene","round": 4,- "atMs": 26057.220042,- "execMs": 0.02374999999301508- },- {- "name": "split_at",- "round": 5,- "atMs": 37866.295124999946,- "execMs": 0.14816699997754768- },- {- "name": "split_at",- "round": 6,- "atMs": 42695.55108399998,- "execMs": 0.04079200001433492- },- {- "name": "split_at",- "round": 7,- "atMs": 44695.987166999956,- "execMs": 0.04829100001370534- },- {- "name": "delete_item",- "round": 8,- "atMs": 47865.65625,- "execMs": 0.08283400000073016- },- {- "name": "delete_item",- "round": 9,- "atMs": 51025.99179199996,- "execMs": 0.06391599995549768- },- {- "name": "refine_speech_cuts",- "round": 10,- "atMs": 55723.74920899997,- "execMs": 0.03874999994877726- },- {- "name": "get_state",- "round": 11,- "atMs": 60025.71583399997,- "execMs": 0.09079099999507889- },- {- "name": "subtitles_generate",- "round": 12,- "atMs": 64085.775166999956,- "execMs": 0.03095799998845905+ "atMs": 13680.798207999906,+ "execMs": 0.010083000059239566}],"timings": {- "gateMs": 999.5926660000114,+ "gateMs": 1109.280207999982,"gateSkipped": false,- "ttftMs": 66488.46224999998,- "modelMs": 66009.95054300013,- "toolMs": 0.7181239999481477,- "totalMs": 67010.26133300009,- "judgeMs": 2012.1048749999609,+ "ttftMs": 16261.186666999944,+ "modelMs": 16482.418000000296,+ "toolMs": 0.04937600006815046,+ "totalMs": 16492.215916999965,+ "judgeMs": 1274.6012090001022,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5923.954209000047,+ "roundMs": 1108.0415829999838,"firstDeltaMs": null,- "calls": [- {- "name": "list_skills",- "execMs": 0.07491700001992285- }- ]+ "calls": []},{- "roundMs": 5637.810540999984,+ "roundMs": 4955.758834000095,"firstDeltaMs": null,"calls": [{"name": "read_skill",- "execMs": 0.020542000012937933- }- ]- },- {- "roundMs": 3128.153875000018,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.03437500004656613+ "execMs": 0.01508399995509535}]},{- "roundMs": 7077.547791999998,+ "roundMs": 2466.845250000013,"firstDeltaMs": null,"calls": [{"name": "listen_audio",- "execMs": 0.020040999981574714- }- ]- },- {- "roundMs": 3279.978625000047,- "firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.02374999999301508- }- ]- },- {- "roundMs": 11807.210250000004,- "firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.14816699997754768- }- ]- },- {- "roundMs": 4826.861708000011,- "firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.04079200001433492- }- ]- },- {- "roundMs": 1999.280792000005,- "firstDeltaMs": null,- "calls": [- {- "name": "split_at",- "execMs": 0.04829100001370534+ "execMs": 0.00779200007673353}]},{- "roundMs": 3168.30441599997,+ "roundMs": 2641.9408750000875,"firstDeltaMs": null,"calls": [{- "name": "delete_item",- "execMs": 0.08283400000073016+ "name": "get_state",+ "execMs": 0.016416999977082014}]},{- "roundMs": 3158.4506670000264,+ "roundMs": 2499.9526250000345,"firstDeltaMs": null,"calls": [{- "name": "delete_item",- "execMs": 0.06391599995549768+ "name": "generate_scene",+ "execMs": 0.010083000059239566}]},{- "roundMs": 4695.882917000039,- "firstDeltaMs": null,- "calls": [- {- "name": "refine_speech_cuts",- "execMs": 0.03874999994877726- }- ]- },+ "roundMs": 2809.8788330000825,+ "firstDeltaMs": 2579.092750000069,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "cartoon-korean-audio-passes-language",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 1163,+ "p95": 1763,+ "mean": 1347,+ "n": 3+ },+ "ttftMs": {+ "p50": 8846,+ "p95": 9186,+ "mean": 8728,+ "n": 3+ },+ "totalMs": {+ "p50": 9097,+ "p95": 9319,+ "mean": 8984,+ "n": 3+ },+ "modelMs": {+ "p50": 9091,+ "p95": 9311,+ "mean": 8976,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 3,+ "max": 3+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 1,+ "atMs": 6855.863832999952,+ "execMs": 0.3638749999227002+ }+ ],+ "timings": {+ "gateMs": 1163.4539999999106,+ "gateSkipped": false,+ "ttftMs": 9185.876916000037,+ "modelMs": 9311.082042000024,+ "toolMs": 0.3638749999227002,+ "totalMs": 9319.163707999978,+ "judgeMs": 2025.576250000042,+ "retryMs": 0,+ "retries": 0,+ "rounds": [{- "roundMs": 4300.739208999963,+ "roundMs": 1162.4710000000196,"firstDeltaMs": null,- "calls": [- {- "name": "get_state",- "execMs": 0.09079099999507889- }- ]+ "calls": []},{- "roundMs": 4058.1978750000126,+ "roundMs": 5687.810999999987,"firstDeltaMs": null,"calls": [{- "name": "subtitles_generate",- "execMs": 0.03095799998845905+ "name": "generate_scene",+ "execMs": 0.3638749999227002}]},{- "roundMs": 2947.577667000005,- "firstDeltaMs": 2401.0788329999777,+ "roundMs": 2460.800042000017,+ "firstDeltaMs": 2328.1809999999823,"calls": []}]@@ -5985,227 +5010,404 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",- "notes": [],- "judgeNote": null,- "trace": [- {- "name": "list_skills",- "round": 0,- "atMs": 7087.347208000021,- "execMs": 0.02074999996693805- },- {- "name": "read_skill",- "round": 1,- "atMs": 12117.584124999994,- "execMs": 0.02395900001283735- },- {- "name": "read_skill",- "round": 1,- "atMs": 12117.832915999985,- "execMs": 0.002041999949142337- },- {- "name": "detect_silence",- "round": 2,- "atMs": 15199.83812500001,- "execMs": 0.07095800002571195- },- {- "name": "watch_video",- "round": 3,- "atMs": 17819.60358300002,- "execMs": 0.026249999995343387- },- {- "name": "get_state",- "round": 4,- "atMs": 20206.557166000013,- "execMs": 0.07237499998882413- },- {- "name": "listen_audio",- "round": 5,- "atMs": 23072.90525000001,- "execMs": 0.05762499995762482- },- {- "name": "split_at",- "round": 6,- "atMs": 32408.411040999985,- "execMs": 0.12708299997029826- },- {- "name": "split_at",- "round": 7,- "atMs": 35299.161124999984,- "execMs": 0.13579099997878075- },- {- "name": "split_at",- "round": 8,- "atMs": 37978.80154100002,- "execMs": 0.1010420000529848- },- {- "name": "delete_item",- "round": 9,- "atMs": 41658.567125,- "execMs": 0.09345899999607354- },- {- "name": "delete_item",- "round": 10,- "atMs": 44294.1715,- "execMs": 0.0711250000167638- },+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [{- "name": "refine_speech_cuts",- "round": 11,- "atMs": 47768.65591600002,- "execMs": 0.08133399998769164+ "name": "generate_scene",+ "round": 1,+ "atMs": 6495.944833999965,+ "execMs": 0.015959000098519027}],"timings": {- "gateMs": 1059.9210420000018,+ "gateMs": 1762.5957090000156,"gateSkipped": false,- "ttftMs": 50990.293125000026,- "modelMs": 50293.48083499988,- "toolMs": 0.8837929998990148,- "totalMs": 51354.28566999978,- "judgeMs": 1483.9913750000414,+ "ttftMs": 8845.75774999999,+ "modelMs": 9091.322332999902,+ "toolMs": 0.015959000098519027,+ "totalMs": 9096.86487499997,+ "judgeMs": 1857.4123339999933,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6025.746207999997,+ "roundMs": 1761.25320799998,"firstDeltaMs": null,- "calls": [- {- "name": "list_skills",- "execMs": 0.02074999996693805- }- ]+ "calls": []},{- "roundMs": 5029.146708999993,+ "roundMs": 4730.534124999889,"firstDeltaMs": null,"calls": [{- "name": "read_skill",- "execMs": 0.02395900001283735- },- {- "name": "read_skill",- "execMs": 0.002041999949142337+ "name": "generate_scene",+ "execMs": 0.015959000098519027}]},{- "roundMs": 3080.0103750000126,- "firstDeltaMs": null,- "calls": [- {- "name": "detect_silence",- "execMs": 0.07095800002571195- }- ]- },+ "roundMs": 2599.5350000000326,+ "firstDeltaMs": 2348.619834000012,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 1,+ "atMs": 5688.750958000077,+ "execMs": 0.028082999982871115+ }+ ],+ "timings": {+ "gateMs": 1115.0982079999521,+ "gateSkipped": false,+ "ttftMs": 8153.627875000006,+ "modelMs": 8526.391291000065,+ "toolMs": 0.028082999982871115,+ "totalMs": 8534.664292000001,+ "judgeMs": 1319.3290840000845,+ "retryMs": 0,+ "retries": 0,+ "rounds": [{- "roundMs": 2617.637624999974,+ "roundMs": 1112.068333000061,"firstDeltaMs": null,- "calls": [- {- "name": "watch_video",- "execMs": 0.026249999995343387- }- ]+ "calls": []},{- "roundMs": 2385.1805830000085,+ "roundMs": 4569.9541250000475,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.07237499998882413+ "name": "generate_scene",+ "execMs": 0.028082999982871115}]},{- "roundMs": 2865.0903340000077,- "firstDeltaMs": null,- "calls": [- {- "name": "listen_audio",- "execMs": 0.05762499995762482- }- ]- },+ "roundMs": 2844.368832999957,+ "firstDeltaMs": 2463.598915999988,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "cartoon-style-reference-anchors-look",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "ttftMs": {+ "p50": 7881,+ "p95": 10358,+ "mean": 8196,+ "n": 3+ },+ "totalMs": {+ "p50": 8139,+ "p95": 10617,+ "mean": 8436,+ "n": 3+ },+ "modelMs": {+ "p50": 8135,+ "p95": 10611,+ "mean": 8431,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 0,+ "atMs": 6944.749375000014,+ "execMs": 0.2282090000808239+ }+ ],+ "timings": {+ "gateMs": 0.21625000005587935,+ "gateSkipped": true,+ "ttftMs": 10358.41858399997,+ "modelMs": 10611.405583999935,+ "toolMs": 0.2282090000808239,+ "totalMs": 10617.165708999964,+ "judgeMs": 1065.3071249999339,+ "retryMs": 0,+ "retries": 0,+ "rounds": [{- "roundMs": 9333.834207999986,+ "roundMs": 6941.688208999927,"firstDeltaMs": null,"calls": [{- "name": "split_at",- "execMs": 0.12708299997029826+ "name": "generate_scene",+ "execMs": 0.2282090000808239}]},{- "roundMs": 2888.891041999974,+ "roundMs": 3669.7173750000075,+ "firstDeltaMs": 3411.2746250000782,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 0,+ "atMs": 3361.6538329999894,+ "execMs": 0.014500000048428774+ }+ ],+ "timings": {+ "gateMs": 0.22966699989046901,+ "gateSkipped": true,+ "ttftMs": 6347.366207999992,+ "modelMs": 6546.729374999995,+ "toolMs": 0.014500000048428774,+ "totalMs": 6550.469832999981,+ "judgeMs": 1928.1499169999734,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 3359.4391250000335,"firstDeltaMs": null,"calls": [{- "name": "split_at",- "execMs": 0.13579099997878075+ "name": "generate_scene",+ "execMs": 0.014500000048428774}]},{- "roundMs": 2677.6118340000394,+ "roundMs": 3187.290249999962,+ "firstDeltaMs": 2984.4217499999795,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "generate_scene",+ "round": 0,+ "atMs": 4643.81708399998,+ "execMs": 0.011457999935373664+ }+ ],+ "timings": {+ "gateMs": 0.2579580000601709,+ "gateSkipped": true,+ "ttftMs": 7881.386708999984,+ "modelMs": 8134.899583999999,+ "toolMs": 0.011457999935373664,+ "totalMs": 8138.906709000003,+ "judgeMs": 1142.8243749999674,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 4641.143750000047,"firstDeltaMs": null,"calls": [{- "name": "split_at",- "execMs": 0.1010420000529848+ "name": "generate_scene",+ "execMs": 0.011457999935373664}]},{- "roundMs": 3677.743124999979,+ "roundMs": 3493.7558339999523,+ "firstDeltaMs": 3236.4434999999357,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "cartoon-question-stays-in-chat",+ "bucket": "chat",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 1531,+ "p95": 1819,+ "mean": 1447,+ "n": 3+ },+ "ttftMs": {+ "p50": 5928,+ "p95": 7291,+ "mean": 6182,+ "n": 3+ },+ "totalMs": {+ "p50": 6189,+ "p95": 7293,+ "mean": 6394,+ "n": 3+ },+ "modelMs": {+ "p50": 6186,+ "p95": 7289,+ "mean": 6389,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 990.7333750000689,+ "gateSkipped": false,+ "ttftMs": 5328.988124999916,+ "modelMs": 5692.534958999953,+ "toolMs": 0,+ "totalMs": 5698.772624999983,+ "judgeMs": 1623.411041000043,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 988.2134999999544,"firstDeltaMs": null,- "calls": [- {- "name": "delete_item",- "execMs": 0.09345899999607354- }- ]+ "calls": []},{- "roundMs": 2633.675249999971,- "firstDeltaMs": null,- "calls": [- {- "name": "delete_item",- "execMs": 0.0711250000167638- }- ]+ "roundMs": 4704.321458999999,+ "firstDeltaMs": 4334.962125000078,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 1530.7037919999566,+ "gateSkipped": false,+ "ttftMs": 7290.52887499996,+ "modelMs": 7289.229124999954,+ "toolMs": 0,+ "totalMs": 7293.026249999995,+ "judgeMs": 1473.4754170000087,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1529.5834170000162,+ "firstDeltaMs": 1522.7665839999681,+ "calls": []},{- "roundMs": 3472.2724999999627,+ "roundMs": 5759.645707999938,+ "firstDeltaMs": 5757.386667000013,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 1818.7792080000509,+ "gateSkipped": false,+ "ttftMs": 5927.923999999999,+ "modelMs": 6185.797125000041,+ "toolMs": 0,+ "totalMs": 6189.042083999957,+ "judgeMs": 1058.921457999968,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1817.5179169999901,"firstDeltaMs": null,- "calls": [- {- "name": "refine_speech_cuts",- "execMs": 0.08133399998769164- }- ]+ "calls": []},{- "roundMs": 3606.641041999974,- "firstDeltaMs": 3219.5574579999666,+ "roundMs": 4368.279208000051,+ "firstDeltaMs": 4107.400166000007,"calls": []}]@@ -6226,21 +5428,21 @@"n": 3},"ttftMs": {- "p50": 35399,- "p95": 43439,- "mean": 34699,+ "p50": 22507,+ "p95": 48811,+ "mean": 31032,"n": 3},"totalMs": {- "p50": 35668,- "p95": 43823,- "mean": 35080,+ "p50": 23021,+ "p95": 49080,+ "mean": 31383,"n": 3},"modelMs": {- "p50": 35668,- "p95": 43822,- "mean": 35079,+ "p50": 23009,+ "p95": 49069,+ "mean": 31371,"n": 3},"toolMs": {@@ -6250,127 +5452,127 @@"n": 3},"rounds": {- "mean": 5.7,- "max": 7+ "mean": 5.3,+ "max": 6}},"runs": [{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{"name": "listen_audio","round": 0,- "atMs": 8056.243207999971,- "execMs": 0.40549999999348074+ "atMs": 4187.590833000024,+ "execMs": 0.30533400003332645},{"name": "watch_video","round": 0,- "atMs": 8057.2199580000015,- "execMs": 0.012584000010974705+ "atMs": 4188.328374999925,+ "execMs": 0.009291000082157552},{"name": "watch_video","round": 0,- "atMs": 8057.368749999965,- "execMs": 0.0015830000047571957+ "atMs": 4188.498874999932,+ "execMs": 0.0019169999286532402},{"name": "get_state","round": 1,- "atMs": 11924.173040999973,- "execMs": 0.015375000017229468+ "atMs": 6315.458291999996,+ "execMs": 0.0055420000571757555},{- "name": "regenerate_shot",+ "name": "read_skill","round": 2,- "atMs": 30927.856541000016,- "execMs": 0.016124999965541065+ "atMs": 8625.43508299999,+ "execMs": 0.01454200001899153},{"name": "regenerate_shot",- "round": 2,- "atMs": 30928.10329099995,- "execMs": 0.003541999962180853+ "round": 3,+ "atMs": 19437.119749999954,+ "execMs": 0.019542000023648143},{- "name": "wait_for_renders",+ "name": "regenerate_shot","round": 3,- "atMs": 40615.82124999998,- "execMs": 0.012332999962382019+ "atMs": 19437.389291999978,+ "execMs": 0.0047920000506564975}],"timings": {- "gateMs": 0.26479099999414757,+ "gateMs": 0.23687499994412065,"gateSkipped": true,- "ttftMs": 43438.55249999993,- "modelMs": 43822.2095,- "toolMs": 0.46704199991654605,- "totalMs": 43822.94133299991,- "judgeMs": 1128.1954580000602,+ "ttftMs": 21778.184124999912,+ "modelMs": 22034.8331680001,+ "toolMs": 0.36096000019460917,+ "totalMs": 22048.31666699995,+ "judgeMs": 1057.5864169999259,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 8053.108000000007,+ "roundMs": 4184.664042000077,"firstDeltaMs": null,"calls": [{"name": "listen_audio",- "execMs": 0.40549999999348074+ "execMs": 0.30533400003332645},{"name": "watch_video",- "execMs": 0.012584000010974705+ "execMs": 0.009291000082157552},{"name": "watch_video",- "execMs": 0.0015830000047571957+ "execMs": 0.0019169999286532402}]},{- "roundMs": 3865.434874999977,+ "roundMs": 2125.6766249999637,"firstDeltaMs": null,"calls": [{"name": "get_state",- "execMs": 0.015375000017229468+ "execMs": 0.0055420000571757555}]},{- "roundMs": 18999.925749999995,+ "roundMs": 2309.200542000006,"firstDeltaMs": null,"calls": [{- "name": "regenerate_shot",- "execMs": 0.016124999965541065- },- {- "name": "regenerate_shot",- "execMs": 0.003541999962180853+ "name": "read_skill",+ "execMs": 0.01454200001899153}]},{- "roundMs": 9685.54912500002,+ "roundMs": 10807.543834000011,"firstDeltaMs": null,"calls": [{- "name": "wait_for_renders",- "execMs": 0.012332999962382019+ "name": "regenerate_shot",+ "execMs": 0.019542000023648143+ },+ {+ "name": "regenerate_shot",+ "execMs": 0.0047920000506564975}]},{- "roundMs": 3218.191749999998,- "firstDeltaMs": 2821.414166999981,+ "roundMs": 2607.748125000042,+ "firstDeltaMs": 2338.9197080000304,"calls": []}]@@ -6379,151 +5581,125 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{"name": "listen_audio","round": 0,- "atMs": 6862.350708000013,- "execMs": 0.022291000001132488+ "atMs": 4708.1722079999745,+ "execMs": 0.02904100005980581},{"name": "watch_video","round": 0,- "atMs": 6862.67975000001,- "execMs": 0.004166999948211014+ "atMs": 4708.477958000032,+ "execMs": 0.011249999981373549},{"name": "watch_video","round": 0,- "atMs": 6862.74216699996,- "execMs": 0.000750000006519258+ "atMs": 4708.603375000064,+ "execMs": 0.0028750000055879354},{"name": "get_state","round": 1,- "atMs": 12421.135541999945,- "execMs": 0.02779199997894466+ "atMs": 10059.71304199996,+ "execMs": 0.01924999989569187},{"name": "list_skills","round": 2,- "atMs": 18781.27150000003,- "execMs": 0.021167000057175756+ "atMs": 12367.492916999967,+ "execMs": 0.007165999966673553},{"name": "read_skill","round": 3,- "atMs": 20810.100083000027,- "execMs": 0.008875000057742- },- {- "name": "regenerate_shot",- "round": 4,- "atMs": 30250.42304200004,- "execMs": 0.01587499992456287+ "atMs": 14504.767749999999,+ "execMs": 0.004249999998137355},{- "name": "regenerate_shot",+ "name": "recut_scene","round": 4,- "atMs": 30250.658832999994,- "execMs": 0.002665999927558005- },- {- "name": "get_state",- "round": 5,- "atMs": 33241.44791700004,- "execMs": 0.018708000076003373+ "atMs": 20115.177917000023,+ "execMs": 0.01991699996870011}],"timings": {- "gateMs": 0.09379199997056276,+ "gateMs": 0.3236669999314472,"gateSkipped": true,- "ttftMs": 35398.84133299999,- "modelMs": 35667.60704399983,- "toolMs": 0.12229099997784942,- "totalMs": 35667.82312699978,- "judgeMs": 1233.2641660000663,+ "ttftMs": 22506.839875000063,+ "modelMs": 23009.056707000127,+ "toolMs": 0.09374899987597018,+ "totalMs": 23021.284083000035,+ "judgeMs": 1149.3532500000438,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6861.0293749999255,+ "roundMs": 4705.341708000051,"firstDeltaMs": null,"calls": [{"name": "listen_audio",- "execMs": 0.022291000001132488+ "execMs": 0.02904100005980581},{"name": "watch_video",- "execMs": 0.004166999948211014+ "execMs": 0.011249999981373549},{"name": "watch_video",- "execMs": 0.000750000006519258+ "execMs": 0.0028750000055879354}]},{- "roundMs": 5557.233833999955,+ "roundMs": 5349.469167000032,"firstDeltaMs": null,"calls": [{"name": "get_state",- "execMs": 0.02779199997894466+ "execMs": 0.01924999989569187}]},{- "roundMs": 6358.868833999964,+ "roundMs": 2306.302040999988,"firstDeltaMs": null,"calls": [{"name": "list_skills",- "execMs": 0.021167000057175756+ "execMs": 0.007165999966673553}]},{- "roundMs": 2027.348083000048,+ "roundMs": 2136.6238750000484,"firstDeltaMs": null,"calls": [{"name": "read_skill",- "execMs": 0.008875000057742- }- ]- },- {- "roundMs": 9438.901874999981,- "firstDeltaMs": null,- "calls": [- {- "name": "regenerate_shot",- "execMs": 0.01587499992456287- },- {- "name": "regenerate_shot",- "execMs": 0.002665999927558005+ "execMs": 0.004249999998137355}]},{- "roundMs": 2989.1975839999504,+ "roundMs": 5607.82358299999,"firstDeltaMs": null,"calls": [{- "name": "get_state",- "execMs": 0.018708000076003373+ "name": "recut_scene",+ "execMs": 0.01991699996870011}]},{- "roundMs": 2435.0274590000045,- "firstDeltaMs": 2155.7786250000354,+ "roundMs": 2903.4963330000173,+ "firstDeltaMs": 2389.8071660000132,"calls": []}]@@ -6532,109 +5708,119 @@{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{"name": "listen_audio","round": 0,- "atMs": 5057.851833999972,- "execMs": 0.012750000110827386+ "atMs": 29136.014417,+ "execMs": 0.12166699999943376},{"name": "watch_video",- "round": 1,- "atMs": 7860.243541999953,- "execMs": 0.013416999951004982+ "round": 0,+ "atMs": 29136.341957999975,+ "execMs": 0.006333000026643276},{"name": "watch_video",+ "round": 0,+ "atMs": 29136.451499999966,+ "execMs": 0.02612499997485429+ },+ {+ "name": "list_skills","round": 1,- "atMs": 7860.524916999973,- "execMs": 0.0032079999800771475+ "atMs": 37993.220207999926,+ "execMs": 0.014874999993480742},{- "name": "regenerate_shot",+ "name": "read_skill","round": 2,- "atMs": 17819.247250000015,- "execMs": 0.013208999997004867+ "atMs": 40291.80679199996,+ "execMs": 0.011000000056810677},{"name": "regenerate_shot",- "round": 2,- "atMs": 17819.553166999947,- "execMs": 0.005374999949708581+ "round": 3,+ "atMs": 46563.83049999992,+ "execMs": 0.02970900002401322},{- "name": "wait_for_renders",+ "name": "regenerate_shot","round": 3,- "atMs": 23448.917750000022,- "execMs": 0.011791000026278198+ "atMs": 46564.453666999936,+ "execMs": 0.011666000005789101}],"timings": {- "gateMs": 0.12062499998137355,+ "gateMs": 0.34658299991860986,"gateSkipped": true,- "ttftMs": 25259.77004199999,- "modelMs": 25748.360875999904,- "toolMs": 0.05975000001490116,- "totalMs": 25748.5412509999,- "judgeMs": 1906.3215830000117,+ "ttftMs": 48810.50724999991,+ "modelMs": 49069.40004199988,+ "toolMs": 0.22137500008102506,+ "totalMs": 49079.596499999985,+ "judgeMs": 1581.3340830000816,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 5056.326624999987,+ "roundMs": 29133.930374999996,"firstDeltaMs": null,"calls": [{"name": "listen_audio",- "execMs": 0.012750000110827386+ "execMs": 0.12166699999943376+ },+ {+ "name": "watch_video",+ "execMs": 0.006333000026643276+ },+ {+ "name": "watch_video",+ "execMs": 0.02612499997485429}]},{- "roundMs": 2800.944958999986,+ "roundMs": 8855.193916999968,"firstDeltaMs": null,"calls": [{- "name": "watch_video",- "execMs": 0.013416999951004982- },- {- "name": "watch_video",- "execMs": 0.0032079999800771475+ "name": "list_skills",+ "execMs": 0.014874999993480742}]},{- "roundMs": 9957.165624999907,+ "roundMs": 2297.297499999986,"firstDeltaMs": null,"calls": [{- "name": "regenerate_shot",- "execMs": 0.013208999997004867- },- {- "name": "regenerate_shot",- "execMs": 0.005374999949708581+ "name": "read_skill",+ "execMs": 0.011000000056810677}]},{- "roundMs": 5627.947999999975,+ "roundMs": 6270.039458999992,"firstDeltaMs": null,"calls": [{- "name": "wait_for_renders",- "execMs": 0.011791000026278198+ "name": "regenerate_shot",+ "execMs": 0.02970900002401322+ },+ {+ "name": "regenerate_shot",+ "execMs": 0.011666000005789101}]},{- "roundMs": 2305.9756670000497,- "firstDeltaMs": 1809.4944169999799,+ "roundMs": 2512.938790999935,+ "firstDeltaMs": 2244.4716659999685,"calls": []}]@@ -6649,27 +5835,27 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1816,- "p95": 1892,- "mean": 1758,+ "p50": 1081,+ "p95": 1138,+ "mean": 1090,"n": 3},"ttftMs": {- "p50": 7329,- "p95": 10454,- "mean": 7855,+ "p50": 5380,+ "p95": 8385,+ "mean": 5931,"n": 3},"totalMs": {- "p50": 8045,- "p95": 10708,- "mean": 8265,+ "p50": 5767,+ "p95": 8821,+ "mean": 6285,"n": 3},"modelMs": {- "p50": 6229,- "p95": 8815,- "mean": 6507,+ "p50": 5758,+ "p95": 8808,+ "mean": 6276,"n": 3},"toolMs": {@@ -6694,50 +5880,50 @@{"name": "set_speed","round": 0,- "atMs": 6567.871999999999,- "execMs": 0.03154199999971752+ "atMs": 5067.838542000041,+ "execMs": 0.1287089999532327},{"name": "remove_transition","round": 1,- "atMs": 9124.000666,- "execMs": 0.003584000000046217+ "atMs": 6913.768541999976,+ "execMs": 0.008832999970763922}],"timings": {- "gateMs": 1892.390709,+ "gateMs": 1050.7836670000106,"gateSkipped": false,- "ttftMs": 10454.172208,- "modelMs": 8815.177416999999,- "toolMs": 0.035125999999763735,- "totalMs": 10707.603251999997,- "judgeMs": 1091.931708,+ "ttftMs": 8385.455707999994,+ "modelMs": 8808.461208999972,+ "toolMs": 0.13754199992399663,+ "totalMs": 8820.678291999968,+ "judgeMs": 1421.2856659999816,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4665.019875,+ "roundMs": 5061.4518749999115,"firstDeltaMs": null,"calls": [{"name": "set_speed",- "execMs": 0.03154199999971752+ "execMs": 0.1287089999532327}]},{- "roundMs": 2554.0044579999994,+ "roundMs": 1842.4036250000354,"firstDeltaMs": null,"calls": [{"name": "remove_transition",- "execMs": 0.003584000000046217+ "execMs": 0.008832999970763922}]},{- "roundMs": 1596.1530839999996,- "firstDeltaMs": 1329.2802919999995,+ "roundMs": 1904.605709000025,+ "firstDeltaMs": 1470.250417000032,"calls": []}]@@ -6753,50 +5939,60 @@{"name": "set_speed","round": 0,- "atMs": 3276.7528330000005,- "execMs": 0.003582999999707681+ "atMs": 1802.389582999982,+ "execMs": 0.00958299997728318},{"name": "remove_transition","round": 1,- "atMs": 5696.155333000001,- "execMs": 0.0017079999997804407+ "atMs": 3659.1113749999786,+ "execMs": 0.007582999998703599+ },+ {+ "name": "set_transition",+ "round": 1,+ "atMs": 3659.770790999988,+ "execMs": 0.01608299999497831}],"timings": {- "gateMs": 1816.0896250000005,+ "gateMs": 1137.5719159999862,"gateSkipped": false,- "ttftMs": 7328.526917000001,- "modelMs": 6229.238291,- "toolMs": 0.005290999999488122,- "totalMs": 8045.333207,- "judgeMs": 1321.5924579999992,+ "ttftMs": 5380.138874999946,+ "modelMs": 5757.891666999902,+ "toolMs": 0.03324899997096509,+ "totalMs": 5766.940290999948,+ "judgeMs": 1134.95633299998,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1458.3694159999995,+ "roundMs": 1798.3795000000391,"firstDeltaMs": null,"calls": [{"name": "set_speed",- "execMs": 0.003582999999707681+ "execMs": 0.00958299997728318}]},{- "roundMs": 2418.0651670000007,+ "roundMs": 1854.3497499999357,"firstDeltaMs": null,"calls": [{"name": "remove_transition",- "execMs": 0.0017079999997804407+ "execMs": 0.007582999998703599+ },+ {+ "name": "set_transition",+ "execMs": 0.01608299999497831}]},{- "roundMs": 2352.8037079999995,- "firstDeltaMs": 1631.5553330000002,+ "roundMs": 2105.162416999927,+ "firstDeltaMs": 1718.4462920000078,"calls": []}]@@ -6812,50 +6008,50 @@{"name": "set_speed","round": 0,- "atMs": 2714.306375,- "execMs": 0.0660409999982221+ "atMs": 1591.6570000000065,+ "execMs": 0.008042000001296401},{"name": "remove_transition","round": 1,- "atMs": 4531.510208,- "execMs": 0.004791999999724794+ "atMs": 3000.5574169999454,+ "execMs": 0.009083000011742115}],"timings": {- "gateMs": 1564.3106249999983,+ "gateMs": 1080.5199579999316,"gateSkipped": false,- "ttftMs": 5782.315290999999,- "modelMs": 4476.862793,- "toolMs": 0.0708329999979469,- "totalMs": 6041.244250999996,- "judgeMs": 1345.8231659999983,+ "ttftMs": 4026.4974999999395,+ "modelMs": 4260.221249999944,+ "toolMs": 0.017125000013038516,+ "totalMs": 4266.780625000014,+ "judgeMs": 1232.7599159999518,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1143.7194170000002,+ "roundMs": 1588.6411249999655,"firstDeltaMs": null,"calls": [{"name": "set_speed",- "execMs": 0.0660409999982221+ "execMs": 0.008042000001296401}]},{- "roundMs": 1815.088083999999,+ "roundMs": 1407.4698749999516,"firstDeltaMs": null,"calls": [{"name": "remove_transition",- "execMs": 0.004791999999724794+ "execMs": 0.009083000011742115}]},{- "roundMs": 1518.055292000001,- "firstDeltaMs": 1249.7950420000016,+ "roundMs": 1264.110250000027,+ "firstDeltaMs": 1024.4331670000684,"calls": []}]@@ -6870,27 +6066,27 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1907,- "p95": 2092,- "mean": 1883,+ "p50": 1225,+ "p95": 1843,+ "mean": 1343,"n": 3},"ttftMs": {- "p50": 16227,- "p95": 17157,- "mean": 15269,+ "p50": 9322,+ "p95": 10322,+ "mean": 8645,"n": 3},"totalMs": {- "p50": 16540,- "p95": 17402,- "mean": 15617,+ "p50": 9424,+ "p95": 10488,+ "mean": 8746,"n": 3},"modelMs": {- "p50": 14891,- "p95": 15495,- "mean": 13735,+ "p50": 9324,+ "p95": 10482,+ "mean": 8709,"n": 3},"toolMs": {@@ -6900,59 +6096,272 @@"n": 3},"rounds": {- "mean": 2,- "max": 2+ "mean": 3.3,+ "max": 4}},"runs": [{"pass": true,"intent": "complex",- "roundModel": "gemini-3.5-flash",+ "roundModel": "gemini-3.7-flash","notes": [],"judgeNote": null,"trace": [{"name": "remove_transition",- "round": 0,- "atMs": 9053.748000000003,- "execMs": 0.2381660000028205+ "round": 2,+ "atMs": 5758.489457999938,+ "execMs": 0.2670419999631122+ },+ {+ "name": "remove_transition",+ "round": 2,+ "atMs": 5759.184832999948,+ "execMs": 0.005124999908730388+ }+ ],+ "timings": {+ "gateMs": 1842.7465000000084,+ "gateSkipped": false,+ "ttftMs": 9322.051665999927,+ "modelMs": 9323.84079200006,+ "toolMs": 0.27216699987184256,+ "totalMs": 9424.120708000031,+ "judgeMs": 1051.0892079999903,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1748.015791999991,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 0.010917000006884336,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3912.983416000032,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "remove_transition",+ "execMs": 0.2670419999631122+ },+ {+ "name": "remove_transition",+ "execMs": 0.005124999908730388+ }+ ]+ },+ {+ "roundMs": 3662.830667000031,+ "firstDeltaMs": 3561.2727920000907,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "remove_transition",+ "round": 1,+ "atMs": 5423.940833000001,+ "execMs": 0.010832999949343503+ },+ {+ "name": "remove_transition",+ "round": 1,+ "atMs": 5424.264708000002,+ "execMs": 0.0025840000016614795+ }+ ],+ "timings": {+ "gateMs": 1224.9432500000112,+ "gateSkipped": false,+ "ttftMs": 10321.68995800009,+ "modelMs": 10482.213876000023,+ "toolMs": 0.013416999951004982,+ "totalMs": 10488.190708000096,+ "judgeMs": 1043.438208000036,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1224.3218750000233,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 4195.833417000016,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "remove_transition",+ "execMs": 0.010832999949343503+ },+ {+ "name": "remove_transition",+ "execMs": 0.0025840000016614795+ }+ ]+ },+ {+ "roundMs": 5062.058583999984,+ "firstDeltaMs": 4895.948166999966,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "remove_transition",+ "round": 1,+ "atMs": 4204.064082999947,+ "execMs": 0.0058750000316649675},{- "name": "remove_transition",+ "name": "remove_transition",+ "round": 1,+ "atMs": 4204.196166999987,+ "execMs": 0.0007910000858828425+ }+ ],+ "timings": {+ "gateMs": 962.107416999992,+ "gateSkipped": false,+ "ttftMs": 6292.518541999976,+ "modelMs": 6320.321458999999,+ "toolMs": 0.00666600011754781,+ "totalMs": 6324.256499999901,+ "judgeMs": 1306.198083999916,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 961.3636669999687,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3239.7796670000535,+ "firstDeltaMs": null,+ "calls": [+ {+ "name": "remove_transition",+ "execMs": 0.0058750000316649675+ },+ {+ "name": "remove_transition",+ "execMs": 0.0007910000858828425+ }+ ]+ },+ {+ "roundMs": 2119.1781249999767,+ "firstDeltaMs": 2087.5011249999516,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "errored-call-not-claimed-done",+ "bucket": "single-tool",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 1157,+ "p95": 1249,+ "mean": 1175,+ "n": 3+ },+ "ttftMs": {+ "p50": 3291,+ "p95": 6507,+ "mean": 4346,+ "n": 3+ },+ "totalMs": {+ "p50": 4081,+ "p95": 6843,+ "mean": 4835,+ "n": 3+ },+ "modelMs": {+ "p50": 4075,+ "p95": 6838,+ "mean": 4829,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 2,+ "max": 2+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": null,+ "trace": [+ {+ "name": "set_transition","round": 0,- "atMs": 9054.308125,- "execMs": 0.0017909999951370992+ "atMs": 1531.4902920000022,+ "execMs": 0.315165999927558}],"timings": {- "gateMs": 2091.8164160000015,+ "gateMs": 1119.1461660000496,"gateSkipped": false,- "ttftMs": 12423.465999999997,- "modelMs": 10817.685292000002,- "toolMs": 0.2399569999979576,- "totalMs": 12909.741665000001,- "judgeMs": 934.2194159999999,+ "ttftMs": 3290.720250000013,+ "modelMs": 4075.1198330000043,+ "toolMs": 0.315165999927558,+ "totalMs": 4080.65149999992,+ "judgeMs": 1837.0576669999864,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 6960.687666999998,+ "roundMs": 1529.4212919999845,"firstDeltaMs": null,"calls": [{- "name": "remove_transition",- "execMs": 0.2381660000028205- },- {- "name": "remove_transition",- "execMs": 0.0017909999951370992+ "name": "set_transition",+ "execMs": 0.315165999927558}]},{- "roundMs": 3856.9976250000036,- "firstDeltaMs": 3367.9055829999998,+ "roundMs": 2545.6985410000198,+ "firstDeltaMs": 1756.126833000104,"calls": []}]@@ -6960,52 +6369,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],"judgeNote": null,"trace": [{- "name": "remove_transition",- "round": 0,- "atMs": 11446.902541000003,- "execMs": 0.009291000002122018- },- {- "name": "remove_transition",+ "name": "set_transition","round": 0,- "atMs": 11447.162540999998,- "execMs": 0.0011249999952269718+ "atMs": 1672.9543330000015,+ "execMs": 0.08108300005551428}],"timings": {- "gateMs": 1906.9025000000038,+ "gateMs": 1157.4554159999825,"gateSkipped": false,- "ttftMs": 17156.802540999997,- "modelMs": 15495.128375,- "toolMs": 0.01041599999734899,- "totalMs": 17402.041291,- "judgeMs": 1943.2617499999978,+ "ttftMs": 3239.8918330000015,+ "modelMs": 3574.3898340000305,+ "toolMs": 0.08108300005551428,+ "totalMs": 3580.2818330000155,+ "judgeMs": 1083.5274590000045,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 9538.00675,+ "roundMs": 1669.6450840000762,"firstDeltaMs": null,"calls": [{- "name": "remove_transition",- "execMs": 0.009291000002122018- },- {- "name": "remove_transition",- "execMs": 0.0011249999952269718+ "name": "set_transition",+ "execMs": 0.08108300005551428}]},{- "roundMs": 5957.121625,- "firstDeltaMs": 5708.721250000002,+ "roundMs": 1904.7447499999544,+ "firstDeltaMs": 1564.65625,"calls": []}]@@ -7013,52 +6412,42 @@},{"pass": true,- "intent": "complex",- "roundModel": "gemini-3.5-flash",+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite","notes": [],- "judgeNote": null,+ "judgeNote": "the tool did not run, but the assistant did not clearly state the failure as an editing turn.","trace": [{- "name": "remove_transition",- "round": 0,- "atMs": 13127.238042000004,- "execMs": 0.010708000001613982- },- {- "name": "remove_transition",+ "name": "set_transition","round": 0,- "atMs": 13127.710249999996,- "execMs": 0.0019580000080168247+ "atMs": 4809.502416999894,+ "execMs": 0.03645899996627122}],"timings": {- "gateMs": 1648.9195829999953,+ "gateMs": 1248.8274170000805,"gateSkipped": false,- "ttftMs": 16226.857417,- "modelMs": 14891.264333,- "toolMs": 0.012666000009630807,- "totalMs": 16540.196582000004,- "judgeMs": 1762.0136250000069,+ "ttftMs": 6506.504166999948,+ "modelMs": 6837.670207999996,+ "toolMs": 0.03645899996627122,+ "totalMs": 6842.941207999946,+ "judgeMs": 1037.1046250000363,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 11475.751208000001,+ "roundMs": 4806.579374999972,"firstDeltaMs": null,"calls": [{- "name": "remove_transition",- "execMs": 0.010708000001613982- },- {- "name": "remove_transition",- "execMs": 0.0019580000080168247+ "name": "set_transition",+ "execMs": 0.03645899996627122}]},{- "roundMs": 3415.5131249999977,- "firstDeltaMs": 3097.727332999988,+ "roundMs": 2031.0908330000238,+ "firstDeltaMs": 1694.86258299998,"calls": []}]@@ -7073,38 +6462,38 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1018,- "p95": 1250,- "mean": 1066,+ "p50": 1140,+ "p95": 1838,+ "mean": 1321,"n": 3},"ttftMs": {- "p50": 4987,- "p95": 9011,- "mean": 6012,+ "p50": 3151,+ "p95": 3584,+ "mean": 3071,"n": 3},"totalMs": {- "p50": 5172,- "p95": 9489,- "mean": 6280,+ "p50": 3391,+ "p95": 3991,+ "mean": 3367,"n": 3},"modelMs": {- "p50": 4243,- "p95": 8238,- "mean": 5214,+ "p50": 3384,+ "p95": 3982,+ "mean": 3359,"n": 3},"toolMs": {"p50": 0,- "p95": 1,+ "p95": 0,"mean": 0,"n": 3},"rounds": {- "mean": 2.3,- "max": 3+ "mean": 2,+ "max": 2}},"runs": [@@ -7118,80 +6507,64 @@{"name": "update_cue","round": 0,- "atMs": 5761.352666999999,- "execMs": 0.4585419999930309- },- {- "name": "delete_cue",- "round": 0,- "atMs": 5762.771458999996,- "execMs": 0.04679100000066683+ "atMs": 2036.0195419999072,+ "execMs": 0.28270799992606044},{"name": "update_cue",- "round": 1,- "atMs": 7945.6942500000005,- "execMs": 0.005541999998968095+ "round": 0,+ "atMs": 2036.591625000001,+ "execMs": 0.04608400003053248},{- "name": "update_cue",- "round": 1,- "atMs": 7945.741458999997,- "execMs": 0.0013750000071013346+ "name": "delete_cue",+ "round": 0,+ "atMs": 2036.888957999996,+ "execMs": 0.06337500002700835},{"name": "update_cue",- "round": 1,- "atMs": 7945.776249999995,- "execMs": 0.0011249999952269718+ "round": 0,+ "atMs": 2037.075332999928,+ "execMs": 0.023790999897755682}],"timings": {- "gateMs": 1250.0429169999989,+ "gateMs": 985.5197919999482,"gateSkipped": false,- "ttftMs": 9011.184209,- "modelMs": 8238.132668000006,- "toolMs": 0.5133749999949941,- "totalMs": 9488.68896,- "judgeMs": 953.9077499999985,+ "ttftMs": 3151.451166999992,+ "modelMs": 3384.0143329998245,+ "toolMs": 0.41595799988135695,+ "totalMs": 3390.957499999902,+ "judgeMs": 1908.927958000102,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 4504.689584000007,+ "roundMs": 2032.4608329999028,"firstDeltaMs": null,"calls": [{"name": "update_cue",- "execMs": 0.4585419999930309+ "execMs": 0.28270799992606044},- {- "name": "delete_cue",- "execMs": 0.04679100000066683- }- ]- },- {- "roundMs": 2181.597999999998,- "firstDeltaMs": null,- "calls": [{"name": "update_cue",- "execMs": 0.005541999998968095+ "execMs": 0.04608400003053248},{- "name": "update_cue",- "execMs": 0.0013750000071013346+ "name": "delete_cue",+ "execMs": 0.06337500002700835},{"name": "update_cue",- "execMs": 0.0011249999952269718+ "execMs": 0.023790999897755682}]},{- "roundMs": 1551.8450840000005,- "firstDeltaMs": 1065.0758750000095,+ "roundMs": 1351.5534999999218,+ "firstDeltaMs": 1112.3028749999357,"calls": []}]@@ -7207,64 +6580,64 @@{"name": "update_cue","round": 0,- "atMs": 2795.468417000011,- "execMs": 0.017332999996142462+ "atMs": 2435.6814160000067,+ "execMs": 0.01624999998603016},{"name": "update_cue","round": 0,- "atMs": 2795.694000000003,- "execMs": 0.008625000002211891+ "atMs": 2436.573290999979,+ "execMs": 0.00933399994391948},{"name": "delete_cue","round": 0,- "atMs": 2795.8145420000073,- "execMs": 0.019625000000814907+ "atMs": 2436.8950830000686,+ "execMs": 0.025541000068187714},{"name": "update_cue","round": 0,- "atMs": 2795.9261670000124,- "execMs": 0.005624999990686774+ "atMs": 2437.0326249999925,+ "execMs": 0.003499999991618097}],"timings": {- "gateMs": 1018.4886249999981,+ "gateMs": 1838.1636670000153,"gateSkipped": false,- "ttftMs": 4037.703125,- "modelMs": 3162.185499000014,- "toolMs": 0.051207999989856035,- "totalMs": 4180.725332000002,- "judgeMs": 1023.3754159999953,+ "ttftMs": 3584.423000000068,+ "modelMs": 3982.1040009999415,+ "toolMs": 0.05462499998975545,+ "totalMs": 3990.5751660000533,+ "judgeMs": 1192.4498330000788,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1774.6703330000018,+ "roundMs": 2431.352458999958,"firstDeltaMs": null,"calls": [{"name": "update_cue",- "execMs": 0.017332999996142462+ "execMs": 0.01624999998603016},{"name": "update_cue",- "execMs": 0.008625000002211891+ "execMs": 0.00933399994391948},{"name": "delete_cue",- "execMs": 0.019625000000814907+ "execMs": 0.025541000068187714},{"name": "update_cue",- "execMs": 0.005624999990686774+ "execMs": 0.003499999991618097}]},{- "roundMs": 1387.515166000012,- "firstDeltaMs": 1240.4643330000108,+ "roundMs": 1550.7515419999836,+ "firstDeltaMs": 1144.9505000000354,"calls": []}]@@ -7280,78 +6653,167 @@{"name": "update_cue","round": 0,- "atMs": 3607.715458999999,- "execMs": 0.014416000005439855- },- {- "name": "update_cue",- "round": 0,- "atMs": 3608.07699999999,- "execMs": 0.005875000002561137- },- {- "name": "update_cue",- "round": 0,- "atMs": 3608.2425419999927,- "execMs": 0.010708000001613982- },- {- "name": "update_cue",- "round": 0,- "atMs": 3608.3545000000013,- "execMs": 0.007499999992433004+ "atMs": 1412.2543329999316,+ "execMs": 0.13254200003575534},{"name": "update_cue","round": 0,- "atMs": 3608.466333999997,- "execMs": 0.004916999998386018+ "atMs": 1412.902374999947,+ "execMs": 0.04425000003539026}],"timings": {- "gateMs": 928.9838329999911,+ "gateMs": 1140.032750000013,"gateSkipped": false,- "ttftMs": 4987.174459000002,- "modelMs": 4242.606583999994,- "toolMs": 0.043416000000433996,- "totalMs": 5171.633832999985,- "judgeMs": 993.1812919999938,+ "ttftMs": 2478.086666999967,+ "modelMs": 2711.3581249999115,+ "toolMs": 0.1767920000711456,+ "totalMs": 2719.77566699998,+ "judgeMs": 1717.675125000067,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2676.7479170000006,+ "roundMs": 1407.245916999993,"firstDeltaMs": null,"calls": [{"name": "update_cue",- "execMs": 0.014416000005439855- },- {- "name": "update_cue",- "execMs": 0.005875000002561137- },- {- "name": "update_cue",- "execMs": 0.010708000001613982- },- {- "name": "update_cue",- "execMs": 0.007499999992433004+ "execMs": 0.13254200003575534},{"name": "update_cue",- "execMs": 0.004916999998386018+ "execMs": 0.04425000003539026}]},{- "roundMs": 1565.8586669999931,- "firstDeltaMs": 1377.2039590000059,+ "roundMs": 1304.1122079999186,+ "firstDeltaMs": 1063.1052500000224,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "question-when-ambiguous",+ "bucket": "chat",+ "passRate": 0.6666666666666666,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 1794,+ "p95": 2712,+ "mean": 2253,+ "n": 2+ },+ "ttftMs": {+ "p50": 4876,+ "p95": 9078,+ "mean": 6977,+ "n": 2+ },+ "totalMs": {+ "p50": 4929,+ "p95": 9199,+ "mean": 7064,+ "n": 2+ },+ "modelMs": {+ "p50": 4498,+ "p95": 9195,+ "mean": 6846,+ "n": 2+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 2+ },+ "rounds": {+ "mean": 2.5,+ "max": 3+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": "judge unavailable (502)",+ "trace": [],+ "timings": {+ "gateMs": 1793.5281250000698,+ "gateSkipped": false,+ "ttftMs": 4876.252792000072,+ "modelMs": 4497.608749999898,+ "toolMs": 0,+ "totalMs": 4928.664667000063,+ "judgeMs": 11277.159375000047,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1365.4640420000069,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 0.016749999951571226,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 3132.127957999939,+ "firstDeltaMs": 3080.133166000014,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "complex",+ "roundModel": "gemini-3.7-flash",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 2711.6676669999724,+ "gateSkipped": false,+ "ttftMs": 9078.016625000047,+ "modelMs": 9194.962749999948,+ "toolMs": 0,+ "totalMs": 9198.995500000077,+ "judgeMs": 2210.2406249999767,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 2711.2943749999395,+ "firstDeltaMs": null,+ "calls": []+ },+ {+ "roundMs": 6483.668375000008,+ "firstDeltaMs": 6362.950667000026,"calls": []}]}+ },+ {+ "pass": false,+ "intent": "work",+ "notes": [],+ "trace": [],+ "timings": null,+ "error": "This operation was aborted"}]},@@ -7362,27 +6824,27 @@"budgetBreaches": [],"latency": {"gateMs": {- "p50": 1721,- "p95": 1860,- "mean": 1612,+ "p50": 2014,+ "p95": 2097,+ "mean": 1699,"n": 3},"ttftMs": {- "p50": 4263,- "p95": 5319,- "mean": 4601,+ "p50": 3464,+ "p95": 7303,+ "mean": 4467,"n": 3},"totalMs": {- "p50": 4713,- "p95": 5572,- "mean": 4931,+ "p50": 3833,+ "p95": 8195,+ "mean": 5023,"n": 3},"modelMs": {- "p50": 3253,- "p95": 3713,- "mean": 3319,+ "p50": 3360,+ "p95": 8188,+ "mean": 4861,"n": 3},"toolMs": {@@ -7407,34 +6869,34 @@{"name": "trim_clip","round": 0,- "atMs": 3944.534375000003,- "execMs": 0.021833999999216758+ "atMs": 5236.01883299998,+ "execMs": 0.3058339999988675}],"timings": {- "gateMs": 1859.7827499999985,+ "gateMs": 2014.0730410000542,"gateSkipped": false,- "ttftMs": 5318.673958000014,- "modelMs": 3712.6843329999974,- "toolMs": 0.021833999999216758,- "totalMs": 5572.488916999995,- "judgeMs": 1799.7672500000044,+ "ttftMs": 7302.643708000076,+ "modelMs": 8187.526000000071,+ "toolMs": 0.3058339999988675,+ "totalMs": 8194.814708000049,+ "judgeMs": 988.7312500000698,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 2082.2217910000036,+ "roundMs": 5231.981084000086,"firstDeltaMs": null,"calls": [{"name": "trim_clip",- "execMs": 0.021833999999216758+ "execMs": 0.3058339999988675}]},{- "roundMs": 1630.4625419999938,- "firstDeltaMs": 1373.7688340000022,+ "roundMs": 2955.5449159999844,+ "firstDeltaMs": 2063.775290999911,"calls": []}]@@ -7450,34 +6912,34 @@{"name": "trim_clip","round": 0,- "atMs": 2961.0620000000054,- "execMs": 0.005124999996041879+ "atMs": 2097.1743750000605,+ "execMs": 0.0032920000376179814}],"timings": {- "gateMs": 1254.5397499999963,+ "gateMs": 2096.8562089999905,"gateSkipped": false,- "ttftMs": 4263.229042000006,- "modelMs": 3252.521374999982,- "toolMs": 0.005124999996041879,- "totalMs": 4507.066249999974,- "judgeMs": 1752.2745409999916,+ "ttftMs": 3464.152584000025,+ "modelMs": 3359.917041000095,+ "toolMs": 0.0032920000376179814,+ "totalMs": 3833.4667500000214,+ "judgeMs": 1252.2110419999808,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1704.5885829999897,+ "roundMs": 1624.392833000049,"firstDeltaMs": null,"calls": [{"name": "trim_clip",- "execMs": 0.005124999996041879+ "execMs": 0.0032920000376179814}]},{- "roundMs": 1547.9327919999923,- "firstDeltaMs": 1300.804416999992,+ "roundMs": 1735.5242080000462,+ "firstDeltaMs": 1366.4937080000527,"calls": []}]@@ -7493,34 +6955,156 @@{"name": "trim_clip","round": 0,- "atMs": 3157.790500000003,- "execMs": 0.004625000001396984+ "atMs": 1466.8616250000196,+ "execMs": 0.009624999947845936}],"timings": {- "gateMs": 1721.1325830000133,+ "gateMs": 984.6576669999631,"gateSkipped": false,- "ttftMs": 4220.116083999994,- "modelMs": 2992.297999000002,- "toolMs": 0.004625000001396984,- "totalMs": 4713.435207000017,- "judgeMs": 1583.2242079999996,+ "ttftMs": 2635.555000000051,+ "modelMs": 3036.4881670001196,+ "toolMs": 0.009624999947845936,+ "totalMs": 3040.7009170000674,+ "judgeMs": 1172.3532920000143,"retryMs": 0,"retries": 0,"rounds": [{- "roundMs": 1434.5011660000018,+ "roundMs": 1465.082250000094,"firstDeltaMs": null,"calls": [{"name": "trim_clip",- "execMs": 0.004625000001396984+ "execMs": 0.009624999947845936}]},{- "roundMs": 1557.7968330000003,- "firstDeltaMs": 1061.0222910000011,+ "roundMs": 1571.4059170000255,+ "firstDeltaMs": 1166.597833000007,+ "calls": []+ }+ ]+ }+ }+ ]+ },+ {+ "name": "legacy-thread-no-leak",+ "bucket": "chat",+ "passRate": 1,+ "budgetBreaches": [],+ "latency": {+ "gateMs": {+ "p50": 996,+ "p95": 1311,+ "mean": 1081,+ "n": 3+ },+ "ttftMs": {+ "p50": 1210,+ "p95": 1489,+ "mean": 1263,+ "n": 3+ },+ "totalMs": {+ "p50": 1368,+ "p95": 1661,+ "mean": 1442,+ "n": 3+ },+ "modelMs": {+ "p50": 1365,+ "p95": 1658,+ "mean": 1439,+ "n": 3+ },+ "toolMs": {+ "p50": 0,+ "p95": 0,+ "mean": 0,+ "n": 3+ },+ "rounds": {+ "mean": 1,+ "max": 1+ }+ },+ "runs": [+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 934.7855419999687,+ "gateSkipped": false,+ "ttftMs": 1209.547833000077,+ "modelMs": 1365.3867910000263,+ "toolMs": 0,+ "totalMs": 1368.2847499999916,+ "judgeMs": 998.8910840000026,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1365.3867910000263,+ "firstDeltaMs": 1206.719999999972,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 1310.6001249999972,+ "gateSkipped": false,+ "ttftMs": 1488.958290999988,+ "modelMs": 1658.3532499999274,+ "toolMs": 0,+ "totalMs": 1660.6062079999829,+ "judgeMs": 1379.843249999918,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1658.3532499999274,+ "firstDeltaMs": 1486.9065419998951,+ "calls": []+ }+ ]+ }+ },+ {+ "pass": true,+ "intent": "simple",+ "roundModel": "gemini-3.1-flash-lite",+ "notes": [],+ "judgeNote": null,+ "trace": [],+ "timings": {+ "gateMs": 996.2925420000684,+ "gateSkipped": false,+ "ttftMs": 1090.0226249999832,+ "modelMs": 1292.6347500000848,+ "toolMs": 0,+ "totalMs": 1295.6339999999618,+ "judgeMs": 2068.7990829999326,+ "retryMs": 0,+ "retries": 0,+ "rounds": [+ {+ "roundMs": 1292.6347500000848,+ "firstDeltaMs": 1087.285666000098,"calls": []}]diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsxindex 21566b03..9e460484 100644--- a/site/src/cut/components/AiPanel.tsx+++ b/site/src/cut/components/AiPanel.tsx@@ -328,11 +328,15 @@ export function AiPanel({const engineUp = useLocalCompute();const readOnly = useEditor((s) => s.readOnly);const signedIn = useSignedIn();- const [model, setModel] = useState<string>(() =>- typeof window === "undefined"- ? DEFAULT_MODEL- : (localStorage.getItem(MODEL_KEY) ?? DEFAULT_MODEL),- );+ // A saved id whose model left the catalog (a retired model after a bump)+ // falls back to the default; passed through, the hosted route rejects it.+ const [model, setModel] = useState<string>(() => {+ if (typeof window === "undefined") return DEFAULT_MODEL;+ const saved = localStorage.getItem(MODEL_KEY);+ return saved && AI_MODELS.some((m) => m.id === saved)+ ? saved+ : DEFAULT_MODEL;+ });// One chat is active at a time; every past chat lives in the Threads panel.// The id persists so closing and reopening the panel resumes the same chat.const [activeChat, setActiveChat] = useState<string>(() =>diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.tsindex 96763a12..3658b9bc 100644--- a/site/src/lib/credits/provider-pricing.ts+++ b/site/src/lib/credits/provider-pricing.ts@@ -247,15 +247,32 @@ const openAITextCreditRates: {{ model: "o4-mini", input: "1.1", cachedInput: "0.275", output: "4.4" },];+// gemini-3.7-flash launch pricing runs through 2026-12-31; standard pricing+// starts 2027-01-01 UTC. Audio input bills at the text rate. The record entry+// is a getter, so every lookup reads the clock and the switch happens on+// schedule without a deploy.+const geminiFlashStandardPricingStartMs = Date.UTC(2027, 0, 1);+const geminiFlashLaunchPricing = textAudioTokenPricing({+ input: "0.75",+ cachedInput: "0.075",+ output: "3.75",+ inputAudio: "0.75",+});+const geminiFlashStandardPricing = textAudioTokenPricing({+ input: "1.5",+ cachedInput: "0.15",+ output: "7.5",+ inputAudio: "1.5",+});+// Every Gemini model we run must appear here: the Record is keyed by the GeminiModel union,// so adding a model to gemini-models.ts without a price fails the type-check (and the build).const geminiModelPricing: Record<GeminiModel, ProviderCreditPricing> = {- [geminiModels.flash]: textTokenPricing({- model: "gemini-3.5-flash",- input: "1.5",- cachedInput: "0.15",- output: "9",- }),+ get [geminiModels.flash]() {+ return Date.now() >= geminiFlashStandardPricingStartMs+ ? geminiFlashStandardPricing+ : geminiFlashLaunchPricing;+ },[geminiModels.flashLite]: textAudioTokenPricing({input: "0.25",cachedInput: "0.025",diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.tsindex 321a9c14..3a33e636 100644--- a/site/src/lib/inference/gemini-models.ts+++ b/site/src/lib/inference/gemini-models.ts@@ -8,7 +8,7 @@// Canonical, dated-or-versioned model IDs. Add a new constant when adopting a// new model; do not inline raw version strings at call sites.export const geminiModels = {- flash: "gemini-3.5-flash",+ flash: "gemini-3.7-flash",flashLite: "gemini-3.1-flash-lite",// Generative image editing/generation ("nano banana"). Bump here when adopting a// newer image model.