Skip to content

Open source CapCut alternative.

TypeScript

feat(site): chat models run on gemini 3.7 flash [rebuild]

David Lec4c22aa7dcd4
diff --git a/docs/guides/backend-apis.md b/docs/guides/backend-apis.md
index 75191d6d..617ae784 100644
--- a/docs/guides/backend-apis.md
+++ b/docs/guides/backend-apis.md
@@ -109,7 +109,7 @@ Model choice is code (see Handler Rules on `process.env`):
| Call | Model |
|---|---|
| Gate-judged simple chat turns, fast structured decisions | `gemini-3.1-flash-lite` |
-| General chat, non-decision structured calls | `gemini-3.5-flash` |
+| General chat, non-decision structured calls | `gemini-3.7-flash` |
Structured requests normalize their JSON schema for Gemini and retry without a
provider-enforced schema when Vertex rejects the schema parameters.
diff --git a/evals/cut-chat.latest-report.json b/evals/cut-chat.latest-report.json
index 9e4b3267..7520b4f5 100644
--- a/evals/cut-chat.latest-report.json
+++ b/evals/cut-chat.latest-report.json
@@ -1,82 +1,82 @@
{
"schema": "cut-chat-eval/v1",
- "generatedAt": "2026-08-11T13:51:03.802Z",
+ "generatedAt": "2026-08-14T11:00:02.814Z",
"base": "http://localhost:3000",
"runsPerCase": 3,
"configs": [
{
"label": "baseline",
- "chatModel": "gemini-3.1-flash-lite | complex\u2192gemini-3.5-flash",
+ "chatModel": "gemini-3.1-flash-lite | complex\u2192gemini-3.7-flash",
"gateModel": "gemini-3.1-flash-lite",
"buckets": {
- "single-tool": {
- "cases": 16,
- "passRate": 0.92,
+ "chat": {
+ "cases": 9,
+ "passRate": 0.96,
"gateMs": {
- "p50": 1168,
- "p95": 1961,
- "mean": 1236,
- "n": 16
+ "p50": 1088,
+ "p95": 1794,
+ "mean": 1031,
+ "n": 9
},
"ttftMs": {
- "p50": 5130,
- "p95": 14917,
- "mean": 7137,
- "n": 16
+ "p50": 3796,
+ "p95": 5928,
+ "mean": 3622,
+ "n": 9
},
"totalMs": {
- "p50": 5873,
- "p95": 15188,
- "mean": 7559,
- "n": 16
+ "p50": 4139,
+ "p95": 6189,
+ "mean": 3807,
+ "n": 9
},
- "meanRounds": 2.0
+ "meanRounds": 1.7
},
"multi-tool": {
"cases": 8,
- "passRate": 0.92,
+ "passRate": 1,
"gateMs": {
- "p50": 1018,
- "p95": 1907,
- "mean": 1062,
+ "p50": 1061,
+ "p95": 2014,
+ "mean": 941,
"n": 8
},
"ttftMs": {
- "p50": 7329,
- "p95": 66488,
- "mean": 19330,
+ "p50": 5380,
+ "p95": 53553,
+ "mean": 15775,
"n": 8
},
"totalMs": {
- "p50": 8045,
- "p95": 67010,
- "mean": 19724,
+ "p50": 5767,
+ "p95": 53903,
+ "mean": 16060,
"n": 8
},
- "meanRounds": 5.9
+ "meanRounds": 5.2
},
- "chat": {
- "cases": 9,
- "passRate": 1.0,
+ "single-tool": {
+ "cases": 16,
+ "passRate": 1,
"gateMs": {
- "p50": 1266,
- "p95": 1500,
- "mean": 993,
- "n": 9
+ "p50": 1125,
+ "p95": 1881,
+ "mean": 1174,
+ "n": 16
},
"ttftMs": {
- "p50": 4385,
- "p95": 8949,
- "mean": 4740,
- "n": 9
+ "p50": 3291,
+ "p95": 16261,
+ "mean": 6018,
+ "n": 16
},
"totalMs": {
- "p50": 4390,
- "p95": 9794,
- "mean": 5030,
- "n": 9
+ "p50": 4081,
+ "p95": 16492,
+ "mean": 6354,
+ "n": 16
},
- "meanRounds": 1.0
+ "meanRounds": 2.5
}
},
"cases": [
@@ -93,21 +93,21 @@
"n": 3
},
"ttftMs": {
- "p50": 4577,
- "p95": 9515,
- "mean": 6084,
+ "p50": 3545,
+ "p95": 5295,
+ "mean": 4055,
"n": 3
},
"totalMs": {
- "p50": 4675,
- "p95": 9683,
- "mean": 6235,
+ "p50": 3661,
+ "p95": 5391,
+ "mean": 4221,
"n": 3
},
"modelMs": {
- "p50": 4675,
- "p95": 9682,
- "mean": 6235,
+ "p50": 3658,
+ "p95": 5383,
+ "mean": 4216,
"n": 3
},
"toolMs": {
@@ -125,24 +125,24 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.6382089999999607,
+ "gateMs": 0.6801250000000891,
"gateSkipped": true,
- "ttftMs": 9515.249,
- "modelMs": 9682.239625,
+ "ttftMs": 5295.283375,
+ "modelMs": 5382.8430419999995,
"toolMs": 0,
- "totalMs": 9682.877834,
- "judgeMs": 2070.2317079999993,
+ "totalMs": 5391.090625,
+ "judgeMs": 1283.3167080000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 9682.239625,
- "firstDeltaMs": 9509.159167,
+ "roundMs": 5382.8430419999995,
+ "firstDeltaMs": 5287.6332919999995,
"calls": []
}
]
@@ -151,24 +151,24 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.16216699999858974,
+ "gateMs": 0.3914169999998194,
"gateSkipped": true,
- "ttftMs": 4577.185791999998,
- "modelMs": 4674.673832999999,
+ "ttftMs": 3544.728167000001,
+ "modelMs": 3657.7012909999994,
"toolMs": 0,
- "totalMs": 4674.8359999999975,
- "judgeMs": 1124.0844169999982,
+ "totalMs": 3661.4497919999994,
+ "judgeMs": 925.4969999999994,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4674.673832999999,
- "firstDeltaMs": 4575.544040999999,
+ "roundMs": 3657.7012909999994,
+ "firstDeltaMs": 3541.2325409999994,
"calls": []
}
]
@@ -177,24 +177,24 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.21391600000060862,
+ "gateMs": 0.17783300000155577,
"gateSkipped": true,
- "ttftMs": 4160.707750000001,
- "modelMs": 4347.5510829999985,
+ "ttftMs": 3325.0255830000006,
+ "modelMs": 3607.0003749999996,
"toolMs": 0,
- "totalMs": 4347.764998999999,
- "judgeMs": 1307.9249579999996,
+ "totalMs": 3609.242291999999,
+ "judgeMs": 1483.7433330000003,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4347.5510829999985,
- "firstDeltaMs": 4158.627499999999,
+ "roundMs": 3607.0003749999996,
+ "firstDeltaMs": 3323.1571669999994,
"calls": []
}
]
@@ -209,27 +209,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1315,
- "p95": 1584,
- "mean": 1358,
+ "p50": 1246,
+ "p95": 1734,
+ "mean": 1331,
"n": 3
},
"ttftMs": {
- "p50": 2894,
- "p95": 3383,
- "mean": 3053,
+ "p50": 3572,
+ "p95": 3682,
+ "mean": 3463,
"n": 3
},
"totalMs": {
- "p50": 2910,
- "p95": 3398,
- "mean": 3063,
+ "p50": 3572,
+ "p95": 3782,
+ "mean": 3500,
"n": 3
},
"modelMs": {
- "p50": 1706,
- "p95": 1814,
- "mean": 1705,
+ "p50": 3569,
+ "p95": 3773,
+ "mean": 3495,
"n": 3
},
"toolMs": {
@@ -239,32 +239,37 @@
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 2,
+ "max": 2
}
},
"runs": [
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1315.435542000003,
+ "gateMs": 1733.6028330000008,
"gateSkipped": false,
- "ttftMs": 2893.8250000000007,
- "modelMs": 1594.2496249999967,
+ "ttftMs": 3681.6503749999974,
+ "modelMs": 3773.088208000001,
"toolMs": 0,
- "totalMs": 2909.6851669999996,
- "judgeMs": 2071.457875,
+ "totalMs": 3782.118624999999,
+ "judgeMs": 1090.0546250000007,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1594.2496249999967,
- "firstDeltaMs": 1574.7519999999968,
+ "roundMs": 1731.358916000001,
+ "firstDeltaMs": 1675.356208000001,
+ "calls": []
+ },
+ {
+ "roundMs": 2041.729292,
+ "firstDeltaMs": 1941.7217500000006,
"calls": []
}
]
@@ -273,24 +278,29 @@
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1583.569958,
+ "gateMs": 1246.2363330000007,
"gateSkipped": false,
- "ttftMs": 3382.8357499999984,
- "modelMs": 1814.494584,
+ "ttftMs": 3571.9032499999994,
+ "modelMs": 3569.397124999996,
"toolMs": 0,
- "totalMs": 3398.064542,
- "judgeMs": 1060.0287499999977,
+ "totalMs": 3572.304959000001,
+ "judgeMs": 949.5440000000017,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1814.494584,
- "firstDeltaMs": 1797.901167,
+ "roundMs": 1245.7104169999984,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2323.6867079999975,
+ "firstDeltaMs": 2323.3776249999974,
"calls": []
}
]
@@ -299,24 +309,29 @@
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1174.5241660000029,
+ "gateMs": 1013.8146249999991,
"gateSkipped": false,
- "ttftMs": 2880.972833,
- "modelMs": 1705.5139169999966,
+ "ttftMs": 3134.808499999999,
+ "modelMs": 3141.4495419999985,
"toolMs": 0,
- "totalMs": 2880.0380829999995,
- "judgeMs": 1119.4047910000008,
+ "totalMs": 3144.746292,
+ "judgeMs": 1055.9203339999985,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1705.5139169999966,
- "firstDeltaMs": 1705.1563339999993,
+ "roundMs": 1012.1462920000013,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2129.303249999997,
+ "firstDeltaMs": 2119.4951249999976,
"calls": []
}
]
@@ -331,27 +346,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 982,
- "p95": 1612,
- "mean": 1170,
+ "p50": 1625,
+ "p95": 1813,
+ "mean": 1497,
"n": 3
},
"ttftMs": {
- "p50": 2839,
- "p95": 3329,
- "mean": 2905,
+ "p50": 4437,
+ "p95": 6880,
+ "mean": 4935,
"n": 3
},
"totalMs": {
- "p50": 2838,
- "p95": 3615,
- "mean": 3006,
+ "p50": 4439,
+ "p95": 7008,
+ "mean": 4985,
"n": 3
},
"modelMs": {
- "p50": 1922,
- "p95": 2003,
- "mean": 1836,
+ "p50": 4003,
+ "p95": 6851,
+ "mean": 4786,
"n": 3
},
"toolMs": {
@@ -361,32 +376,42 @@
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 2.7,
+ "max": 3
}
},
"runs": [
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 981.7013339999976,
+ "gateMs": 1625.3346669999992,
"gateSkipped": false,
- "ttftMs": 2548.5990419999944,
- "modelMs": 1582.0289170000033,
+ "ttftMs": 6879.611375,
+ "modelMs": 6850.584874999997,
"toolMs": 0,
- "totalMs": 2563.730251000001,
- "judgeMs": 1223.5903749999998,
+ "totalMs": 7008.350666999999,
+ "judgeMs": 1155.4799579999963,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1582.0289170000033,
- "firstDeltaMs": 1563.7441250000047,
+ "roundMs": 1470.3922920000005,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 0.019791999999142718,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 5380.172790999997,
+ "firstDeltaMs": 5251.712541000001,
"calls": []
}
]
@@ -395,24 +420,29 @@
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
- "judgeNote": "overly robotic response to a social pleasantry",
+ "judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 1612.1080409999995,
+ "gateMs": 1051.4393750000017,
"gateSkipped": false,
- "ttftMs": 3329.329791999997,
- "modelMs": 2002.5194590000028,
+ "ttftMs": 3488.972542000003,
+ "modelMs": 3505.183582999991,
"toolMs": 0,
- "totalMs": 3614.6275000000023,
- "judgeMs": 2260.5377089999965,
+ "totalMs": 3508.2672500000044,
+ "judgeMs": 972.8683750000055,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2002.5194590000028,
- "firstDeltaMs": 1715.645709000004,
+ "roundMs": 1050.3509579999954,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2454.8326249999955,
+ "firstDeltaMs": 2435.768707999996,
"calls": []
}
]
@@ -421,24 +451,34 @@
{
"pass": true,
"intent": "chat",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 916.338499999998,
+ "gateMs": 1812.8156249999956,
"gateSkipped": false,
- "ttftMs": 2838.568833000005,
- "modelMs": 1922.1279589999976,
+ "ttftMs": 4436.651000000005,
+ "modelMs": 4002.911376000011,
"toolMs": 0,
- "totalMs": 2838.4664589999957,
- "judgeMs": 1691.0353329999998,
+ "totalMs": 4439.214458000002,
+ "judgeMs": 750.3692499999961,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1922.1279589999976,
- "firstDeltaMs": 1921.8951249999955,
+ "roundMs": 1380.1530420000054,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 0.03908400000364054,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 2622.719250000002,
+ "firstDeltaMs": 2620.609209000002,
"calls": []
}
]
@@ -447,44 +487,44 @@
]
},
{
- "name": "question-answers-in-chat",
- "bucket": "chat",
+ "name": "follow-up-do-it-works",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1500,
- "p95": 1584,
- "mean": 1435,
+ "p50": 1006,
+ "p95": 1111,
+ "mean": 1017,
"n": 3
},
"ttftMs": {
- "p50": 3643,
- "p95": 6706,
- "mean": 4280,
+ "p50": 3064,
+ "p95": 6813,
+ "mean": 4249,
"n": 3
},
"totalMs": {
- "p50": 4262,
- "p95": 6907,
- "mean": 4653,
+ "p50": 3341,
+ "p95": 7105,
+ "mean": 4529,
"n": 3
},
"modelMs": {
- "p50": 2678,
- "p95": 5407,
- "mean": 3217,
+ "p50": 3337,
+ "p95": 7089,
+ "mean": 4520,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 0,
+ "p95": 1,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -494,21 +534,68 @@
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
- "trace": [],
+ "trace": [
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 5375.111749999996,
+ "execMs": 0.7929579999981797
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 5377.615957999995,
+ "execMs": 0.01750000000174623
+ },
+ {
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 5378.9735,
+ "execMs": 0.05762499999400461
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 5379.2072499999995,
+ "execMs": 0.007291000001714565
+ }
+ ],
"timings": {
- "gateMs": 1499.9039579999953,
+ "gateMs": 1006.0852919999961,
"gateSkipped": false,
- "ttftMs": 6705.881500000003,
- "modelMs": 5407.145083999996,
- "toolMs": 0,
- "totalMs": 6907.0490419999915,
- "judgeMs": 2238.7736670000013,
+ "ttftMs": 6813.174874999997,
+ "modelMs": 7088.589292000004,
+ "toolMs": 0.8753739999956451,
+ "totalMs": 7104.922332999995,
+ "judgeMs": 1164.2447089999987,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5407.145083999996,
- "firstDeltaMs": 5204.016041999996,
+ "roundMs": 5365.451042000001,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.7929579999981797
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.01750000000174623
+ },
+ {
+ "name": "delete_cue",
+ "execMs": 0.05762499999400461
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.007291000001714565
+ }
+ ]
+ },
+ {
+ "roundMs": 1723.1382500000036,
+ "firstDeltaMs": 1432.1299169999984,
"calls": []
}
]
@@ -520,21 +607,48 @@
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
- "trace": [],
+ "trace": [
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 1973.907124999998,
+ "execMs": 0.011249999995925464
+ },
+ {
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 1974.1132500000022,
+ "execMs": 0.017166999998153187
+ }
+ ],
"timings": {
- "gateMs": 1221.9570000000022,
+ "gateMs": 933.0407500000001,
"gateSkipped": false,
- "ttftMs": 2490.9581670000043,
- "modelMs": 1566.588917000001,
- "toolMs": 0,
- "totalMs": 2788.545917000003,
- "judgeMs": 1711.0802499999918,
+ "ttftMs": 3063.574959000005,
+ "modelMs": 3336.548625000003,
+ "toolMs": 0.02841699999407865,
+ "totalMs": 3340.5978749999995,
+ "judgeMs": 1044.6940419999955,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1566.588917000001,
- "firstDeltaMs": 1267.5192090000055,
+ "roundMs": 1971.3986250000016,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.011249999995925464
+ },
+ {
+ "name": "delete_cue",
+ "execMs": 0.017166999998153187
+ }
+ ]
+ },
+ {
+ "roundMs": 1365.1500000000015,
+ "firstDeltaMs": 1088.4904580000002,
"calls": []
}
]
@@ -546,21 +660,68 @@
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
- "trace": [],
+ "trace": [
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 1788.0434170000008,
+ "execMs": 0.017584000001079403
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 1788.5724580000024,
+ "execMs": 0.018167000001994893
+ },
+ {
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 1788.833292000003,
+ "execMs": 0.0171249999984866
+ },
+ {
+ "name": "update_cue",
+ "round": 0,
+ "atMs": 1789.0872080000045,
+ "execMs": 0.016290999999910127
+ }
+ ],
"timings": {
- "gateMs": 1584.239333999998,
+ "gateMs": 1111.4699170000022,
"gateSkipped": false,
- "ttftMs": 3642.686916999999,
- "modelMs": 2678.1887089999946,
- "toolMs": 0,
- "totalMs": 4262.428042999993,
- "judgeMs": 1256.590584000005,
+ "ttftMs": 2871.0125830000034,
+ "modelMs": 3133.879873999991,
+ "toolMs": 0.06916700000147102,
+ "totalMs": 3140.0463750000054,
+ "judgeMs": 2061.4119169999976,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2678.1887089999946,
- "firstDeltaMs": 2056.337625,
+ "roundMs": 1784.9065409999967,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "update_cue",
+ "execMs": 0.017584000001079403
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.018167000001994893
+ },
+ {
+ "name": "delete_cue",
+ "execMs": 0.0171249999984866
+ },
+ {
+ "name": "update_cue",
+ "execMs": 0.016290999999910127
+ }
+ ]
+ },
+ {
+ "roundMs": 1348.9733329999945,
+ "firstDeltaMs": 1080.4344159999964,
"calls": []
}
]
@@ -569,33 +730,33 @@
]
},
{
- "name": "photos-question-stays-in-chat",
+ "name": "question-answers-in-chat",
"bucket": "chat",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 1,
- "mean": 0,
+ "p50": 1088,
+ "p95": 1306,
+ "mean": 1102,
"n": 3
},
"ttftMs": {
- "p50": 6838,
- "p95": 10478,
- "mean": 7730,
+ "p50": 1261,
+ "p95": 1421,
+ "mean": 1286,
"n": 3
},
"totalMs": {
- "p50": 7204,
- "p95": 11692,
- "mean": 8649,
+ "p50": 1385,
+ "p95": 1828,
+ "mean": 1514,
"n": 3
},
"modelMs": {
- "p50": 7204,
- "p95": 11692,
- "mean": 8649,
+ "p50": 1383,
+ "p95": 1827,
+ "mean": 1512,
"n": 3
},
"toolMs": {
@@ -605,32 +766,32 @@
"n": 3
},
"rounds": {
- "mean": 1.3,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "excessive filler, use of bullet points, and marketing-style suggestions instead of being concise.",
+ "judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.701375000004191,
- "gateSkipped": true,
- "ttftMs": 5874.439958999996,
- "modelMs": 7050.758457999997,
+ "gateMs": 1306.2425829999993,
+ "gateSkipped": false,
+ "ttftMs": 1420.9730419999978,
+ "modelMs": 1826.5942499999946,
"toolMs": 0,
- "totalMs": 7051.459833000001,
- "judgeMs": 1445.1213750000024,
+ "totalMs": 1827.8963749999966,
+ "judgeMs": 1049.1808750000055,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7050.758457999997,
- "firstDeltaMs": 5871.849875,
+ "roundMs": 1826.5942499999946,
+ "firstDeltaMs": 1420.0688749999972,
"calls": []
}
]
@@ -638,25 +799,25 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [],
"timings": {
- "gateMs": 0.08950000000186265,
- "gateSkipped": true,
- "ttftMs": 6838.067708000002,
- "modelMs": 7203.645833000002,
+ "gateMs": 1088.3530829999945,
+ "gateSkipped": false,
+ "ttftMs": 1176.4475419999944,
+ "modelMs": 1326.5217919999996,
"toolMs": 0,
- "totalMs": 7203.735333000004,
- "judgeMs": 1705.49229200001,
+ "totalMs": 1329.1205419999897,
+ "judgeMs": 1653.0858749999898,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7203.645833000002,
- "firstDeltaMs": 6837.30133300001,
+ "roundMs": 1326.5217919999996,
+ "firstDeltaMs": 1174.1965420000051,
"calls": []
}
]
@@ -664,52 +825,25 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "contains upsell (music) and uses filler formatting.",
- "trace": [
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 6834.3042499999865,
- "execMs": 0.030624999999417923
- },
- {
- "name": "watch_video",
- "round": 0,
- "atMs": 6834.906041999988,
- "execMs": 0.001416999992216006
- }
- ],
+ "judgeNote": null,
+ "trace": [],
"timings": {
- "gateMs": 0.30220800000824966,
- "gateSkipped": true,
- "ttftMs": 10478.143666999997,
- "modelMs": 11691.9525,
- "toolMs": 0.03204199999163393,
- "totalMs": 11692.28675,
- "judgeMs": 1305.2327920000098,
+ "gateMs": 910.9697080000042,
+ "gateSkipped": false,
+ "ttftMs": 1261.1302080000023,
+ "modelMs": 1382.7398749999993,
+ "toolMs": 0,
+ "totalMs": 1385.3621660000063,
+ "judgeMs": 2083.4081660000084,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6822.634416999994,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.030624999999417923
- },
- {
- "name": "watch_video",
- "execMs": 0.001416999992216006
- }
- ]
- },
- {
- "roundMs": 4869.318083000006,
- "firstDeltaMs": 3642.571458000006,
+ "roundMs": 1382.7398749999993,
+ "firstDeltaMs": 1258.820000000007,
"calls": []
}
]
@@ -718,33 +852,33 @@
]
},
{
- "name": "clutter-complaint-deletes-nothing",
- "bucket": "chat",
+ "name": "photos-request-places-clips",
+ "bucket": "multi-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1266,
- "p95": 1926,
- "mean": 1480,
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
"n": 3
},
"ttftMs": {
- "p50": 5197,
- "p95": 9159,
- "mean": 6068,
+ "p50": 25756,
+ "p95": 26357,
+ "mean": 25544,
"n": 3
},
"totalMs": {
- "p50": 5455,
- "p95": 9389,
- "mean": 6289,
+ "p50": 25800,
+ "p95": 26524,
+ "mean": 25615,
"n": 3
},
"modelMs": {
- "p50": 4189,
- "p95": 7463,
- "mean": 4808,
+ "p50": 25787,
+ "p95": 26507,
+ "mean": 25600,
"n": 3
},
"toolMs": {
@@ -754,966 +888,256 @@
"n": 3
},
"rounds": {
- "mean": 1,
- "max": 1
+ "mean": 6.7,
+ "max": 7
}
},
"runs": [
{
"pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1249.0194999999949,
- "gateSkipped": false,
- "ttftMs": 3847.3752499999973,
- "modelMs": 2772.835208000004,
- "toolMs": 0,
- "totalMs": 4021.854707999999,
- "judgeMs": 1100.5716249999969,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2772.835208000004,
- "firstDeltaMs": 2595.9299579999933,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1265.765665999992,
- "gateSkipped": false,
- "ttftMs": 5196.987957999998,
- "modelMs": 4188.830541000003,
- "toolMs": 0,
- "totalMs": 5454.596206999995,
- "judgeMs": 891.1059590000077,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 4188.830541000003,
- "firstDeltaMs": 3929.7065000000002,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "chat",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1925.9489170000015,
- "gateSkipped": false,
- "ttftMs": 9158.980416000006,
- "modelMs": 7463.317750000002,
- "toolMs": 0,
- "totalMs": 9389.266667000004,
- "judgeMs": 1119.002416999996,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 7463.317750000002,
- "firstDeltaMs": 7231.192625000011,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "cartoon-question-stays-in-chat",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1340,
- "p95": 1594,
- "mean": 1289,
- "n": 3
- },
- "ttftMs": {
- "p50": 8949,
- "p95": 11302,
- "mean": 9415,
- "n": 3
- },
- "totalMs": {
- "p50": 9794,
- "p95": 11667,
- "mean": 10024,
- "n": 3
- },
- "modelMs": {
- "p50": 8200,
- "p95": 10734,
- "mean": 8735,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 1,
- "max": 1
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1339.6451249999955,
- "gateSkipped": false,
- "ttftMs": 7994.283500000005,
- "modelMs": 7271.1564160000125,
- "toolMs": 0,
- "totalMs": 8610.801541000008,
- "judgeMs": 1219.9470420000143,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 7271.1564160000125,
- "firstDeltaMs": 6654.061916000006,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 933.1749160000181,
- "gateSkipped": false,
- "ttftMs": 11301.68291599999,
- "modelMs": 10733.860083000007,
- "toolMs": 0,
- "totalMs": 11667.034999000025,
- "judgeMs": 1743.929375000007,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 10733.860083000007,
- "firstDeltaMs": 10367.081417000009,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1594.2282500000147,
- "gateSkipped": false,
- "ttftMs": 8949.262208,
- "modelMs": 8199.745999999985,
- "toolMs": 0,
- "totalMs": 9793.97425,
- "judgeMs": 1565.0725410000014,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 8199.745999999985,
- "firstDeltaMs": 7353.404749999987,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "question-when-ambiguous",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1229,
- "p95": 1616,
- "mean": 1301,
- "n": 3
- },
- "ttftMs": {
- "p50": 4385,
- "p95": 4694,
- "mean": 4322,
- "n": 3
- },
- "totalMs": {
- "p50": 4390,
- "p95": 4979,
- "mean": 4497,
- "n": 3
- },
- "modelMs": {
- "p50": 3161,
- "p95": 3363,
- "mean": 3196,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 1,
- "max": 1
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1229.3556250000256,
- "gateSkipped": false,
- "ttftMs": 4384.771666999994,
- "modelMs": 3160.650291000027,
- "toolMs": 0,
- "totalMs": 4390.005916000053,
- "judgeMs": 1045.922499999986,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3160.650291000027,
- "firstDeltaMs": 3152.9439160000184,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1058.3992919999873,
- "gateSkipped": false,
- "ttftMs": 3886.625792000006,
- "modelMs": 3065.247707999981,
- "toolMs": 0,
- "totalMs": 4123.646999999968,
- "judgeMs": 2137.7861250000133,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3065.247707999981,
- "firstDeltaMs": 2826.716374999989,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1615.6097080000036,
- "gateSkipped": false,
- "ttftMs": 4694.452707999997,
- "modelMs": 3363.0867499999877,
- "toolMs": 0,
- "totalMs": 4978.696457999991,
- "judgeMs": 1115.4267080000136,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3363.0867499999877,
- "firstDeltaMs": 3076.8358339999977,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "legacy-thread-no-leak",
- "bucket": "chat",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1307,
- "p95": 1821,
- "mean": 1462,
- "n": 3
- },
- "ttftMs": {
- "p50": 3339,
- "p95": 5836,
- "mean": 3940,
- "n": 3
- },
- "totalMs": {
- "p50": 3744,
- "p95": 6461,
- "mean": 4499,
- "n": 3
- },
- "modelMs": {
- "p50": 2033,
- "p95": 5154,
- "mean": 3037,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 1,
- "max": 1
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1306.5508339999942,
- "gateSkipped": false,
- "ttftMs": 5835.870916999993,
- "modelMs": 5154.352416999987,
- "toolMs": 0,
- "totalMs": 6460.9032509999815,
- "judgeMs": 1403.1368749999965,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5154.352416999987,
- "firstDeltaMs": 4526.572374999989,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1259.9091250000056,
- "gateSkipped": false,
- "ttftMs": 2646.8406670000113,
- "modelMs": 2032.8448750000098,
- "toolMs": 0,
- "totalMs": 3292.7540000000154,
- "judgeMs": 1056.1719589999993,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2032.8448750000098,
- "firstDeltaMs": 1384.6364580000227,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [],
- "timings": {
- "gateMs": 1820.6741250000196,
- "gateSkipped": false,
- "ttftMs": 3338.5416250000126,
- "modelMs": 1923.3888330000045,
- "toolMs": 0,
- "totalMs": 3744.062958000024,
- "judgeMs": 1033.1144170000043,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1923.3888330000045,
- "firstDeltaMs": 1516.050166000001,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "video-ask-single-generate",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1693,
- "p95": 2189,
- "mean": 1692,
- "n": 3
- },
- "ttftMs": {
- "p50": 5332,
- "p95": 6072,
- "mean": 5033,
- "n": 3
- },
- "totalMs": {
- "p50": 6053,
- "p95": 6328,
- "mean": 5466,
- "n": 3
- },
- "modelMs": {
- "p50": 4139,
- "p95": 4361,
- "mean": 3774,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_video",
+ "name": "add_clip",
"round": 0,
- "atMs": 4639.7805,
- "execMs": 0.004582999999911408
- }
- ],
- "timings": {
- "gateMs": 2188.5847080000003,
- "gateSkipped": false,
- "ttftMs": 6072.40675,
- "modelMs": 4139.222292,
- "toolMs": 0.004582999999911408,
- "totalMs": 6327.811583000001,
- "judgeMs": 910.561917,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 2439.952584,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_video",
- "execMs": 0.004582999999911408
- }
- ]
- },
- {
- "roundMs": 1699.2697079999998,
- "firstDeltaMs": 1431.038458,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
+ "atMs": 5606.476874999993,
+ "execMs": 0.009625000006053597
+ },
+ {
+ "name": "add_clip",
+ "round": 0,
+ "atMs": 5606.878375,
+ "execMs": 0.0018330000020796433
+ },
+ {
+ "name": "get_state",
+ "round": 1,
+ "atMs": 8406.270332999993,
+ "execMs": 0.10975000000325963
+ },
+ {
+ "name": "add_clip",
+ "round": 2,
+ "atMs": 11743.913375000004,
+ "execMs": 0.004458000010345131
+ },
{
- "name": "generate_video",
- "round": 0,
- "atMs": 3784.281,
- "execMs": 0.0013330000001587905
+ "name": "get_state",
+ "round": 3,
+ "atMs": 15570.833249999996,
+ "execMs": 0.02425000000221189
+ },
+ {
+ "name": "add_clip",
+ "round": 4,
+ "atMs": 17930.058791999996,
+ "execMs": 0.003790999995544553
}
],
"timings": {
- "gateMs": 1692.5868330000003,
- "gateSkipped": false,
- "ttftMs": 5331.679416,
- "modelMs": 4360.817916,
- "toolMs": 0.0013330000001587905,
- "totalMs": 6053.406082,
- "judgeMs": 1652.759583000001,
+ "gateMs": 0.20495800000207964,
+ "gateSkipped": true,
+ "ttftMs": 25756.029542000004,
+ "modelMs": 25786.876457999984,
+ "toolMs": 0.15370700001949444,
+ "totalMs": 25800.268375,
+ "judgeMs": 1172.552582999997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2088.7971660000003,
+ "roundMs": 5602.695707999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.0013330000001587905
+ "name": "add_clip",
+ "execMs": 0.009625000006053597
+ },
+ {
+ "name": "add_clip",
+ "execMs": 0.0018330000020796433
}
]
},
{
- "roundMs": 2272.0207499999997,
- "firstDeltaMs": 1547.0587919999998,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": "missing approval gate for credit-consuming action",
- "trace": [
- {
- "name": "generate_video",
- "round": 0,
- "atMs": 2503.4117910000004,
- "execMs": 0.005292000001645647
- }
- ],
- "timings": {
- "gateMs": 1194.3320829999993,
- "gateSkipped": false,
- "ttftMs": 3695.109290999999,
- "modelMs": 2822.2551239999993,
- "toolMs": 0.005292000001645647,
- "totalMs": 4016.5924990000003,
- "judgeMs": 1282.6564169999983,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 1303.0563330000004,
+ "roundMs": 2796.879833999992,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.005292000001645647
+ "name": "get_state",
+ "execMs": 0.10975000000325963
}
]
},
{
- "roundMs": 1519.198790999999,
- "firstDeltaMs": 1190.5308750000004,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "iterate-keeps-existing-clip",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1404,
- "p95": 1557,
- "mean": 1347,
- "n": 3
- },
- "ttftMs": {
- "p50": 9729,
- "p95": 12070,
- "mean": 10089,
- "n": 3
- },
- "totalMs": {
- "p50": 9887,
- "p95": 12344,
- "mean": 10372,
- "n": 3
- },
- "modelMs": {
- "p50": 8330,
- "p95": 10940,
- "mean": 9025,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": "did not state the approval gate for credit expenditure.",
- "trace": [
- {
- "name": "generate_video",
- "round": 0,
- "atMs": 9959.234,
- "execMs": 0.017166999999972177
- }
- ],
- "timings": {
- "gateMs": 1404.417875,
- "gateSkipped": false,
- "ttftMs": 12069.571916,
- "modelMs": 10939.925082999998,
- "toolMs": 0.017166999999972177,
- "totalMs": 12344.360124999997,
- "judgeMs": 2059.193125,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 8541.640167,
+ "roundMs": 3335.8724999999977,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.017166999999972177
+ "name": "add_clip",
+ "execMs": 0.004458000010345131
}
]
},
{
- "roundMs": 2398.2849159999987,
- "firstDeltaMs": 2109.3704579999994,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": "missing mandatory approval gate for spending credits/rendering changes.",
- "trace": [
- {
- "name": "generate_video",
- "round": 0,
- "atMs": 7588.552333000001,
- "execMs": 0.0023330000003625173
- }
- ],
- "timings": {
- "gateMs": 1556.5739580000009,
- "gateSkipped": false,
- "ttftMs": 9729.262499999999,
- "modelMs": 8329.984250000001,
- "toolMs": 0.0023330000003625173,
- "totalMs": 9886.560541000003,
- "judgeMs": 1033.645124999999,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 6031.121999999999,
+ "roundMs": 3825.2406249999913,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.0023330000003625173
+ "name": "get_state",
+ "execMs": 0.02425000000221189
}
]
},
{
- "roundMs": 2298.862250000002,
- "firstDeltaMs": 2140.067041000002,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_video",
- "round": 0,
- "atMs": 6088.003375,
- "execMs": 0.00429200000144192
- }
- ],
- "timings": {
- "gateMs": 1078.6309169999986,
- "gateSkipped": false,
- "ttftMs": 8467.358457999999,
- "modelMs": 7805.817917,
- "toolMs": 0.00429200000144192,
- "totalMs": 8884.453126,
- "judgeMs": 1249.2672919999968,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 5007.172917,
+ "roundMs": 2357.2911659999954,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_video",
- "execMs": 0.00429200000144192
+ "name": "add_clip",
+ "execMs": 0.003790999995544553
}
]
},
{
- "roundMs": 2798.6450000000004,
- "firstDeltaMs": 2378.7929170000025,
+ "roundMs": 7868.8966250000085,
+ "firstDeltaMs": 7825.063083000001,
"calls": []
}
]
}
- }
- ]
- },
- {
- "name": "mute-all-does-not-redo-prior-turn",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1164,
- "p95": 1172,
- "mean": 1145,
- "n": 3
- },
- "ttftMs": {
- "p50": 5479,
- "p95": 7867,
- "mean": 5892,
- "n": 3
- },
- "totalMs": {
- "p50": 6275,
- "p95": 8043,
- "mean": 6334,
- "n": 3
},
- "modelMs": {
- "p50": 5111,
- "p95": 6871,
- "mean": 5189,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
- "judgeNote": "inaccuracy (the assistant claimed to mute \"city clips\" when the user requested \"all clips\")",
+ "judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
+ "name": "add_clip",
"round": 0,
- "atMs": 6596.596708999998,
- "execMs": 0.08583299999736482
+ "atMs": 5480.511957999988,
+ "execMs": 0.004040999992867
},
{
- "name": "set_clip_muted",
+ "name": "add_clip",
"round": 0,
- "atMs": 6597.089333999997,
- "execMs": 0.0033750000002328306
+ "atMs": 5480.807499999995,
+ "execMs": 0.0018330000020796433
},
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 6597.287292000001,
- "execMs": 0.0027919999993173406
+ "name": "get_state",
+ "round": 1,
+ "atMs": 9978.928874999998,
+ "execMs": 0.0342499999969732
+ },
+ {
+ "name": "add_clip",
+ "round": 2,
+ "atMs": 13573.690332999991,
+ "execMs": 0.005917000002227724
+ },
+ {
+ "name": "add_clip",
+ "round": 3,
+ "atMs": 16911.97591699999,
+ "execMs": 0.0052500000019790605
+ },
+ {
+ "name": "get_state",
+ "round": 4,
+ "atMs": 20580.118208,
+ "execMs": 0.012166999993496574
+ },
+ {
+ "name": "add_clip",
+ "round": 5,
+ "atMs": 24268.81745799999,
+ "execMs": 0.005374999993364327
}
],
"timings": {
- "gateMs": 1171.8670409999977,
- "gateSkipped": false,
- "ttftMs": 7866.677334,
- "modelMs": 6871.4575419999965,
- "toolMs": 0.091999999996915,
- "totalMs": 8043.416582999991,
- "judgeMs": 1091.9065000000046,
+ "gateMs": 0.1699159999989206,
+ "gateSkipped": true,
+ "ttftMs": 26357.05308299999,
+ "modelMs": 26507.328874000013,
+ "toolMs": 0.06883299998298753,
+ "totalMs": 26523.933332999994,
+ "judgeMs": 1062.2252500000031,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5417.422166999997,
+ "roundMs": 5478.818249999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.08583299999736482
+ "name": "add_clip",
+ "execMs": 0.004040999992867
},
{
- "name": "set_clip_muted",
- "execMs": 0.0033750000002328306
- },
+ "name": "add_clip",
+ "execMs": 0.0018330000020796433
+ }
+ ]
+ },
+ {
+ "roundMs": 4496.467917000002,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.0027919999993173406
+ "name": "get_state",
+ "execMs": 0.0342499999969732
}
]
},
{
- "roundMs": 1454.0353749999995,
- "firstDeltaMs": 1267.5579579999976,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3916.6915829999925,
- "execMs": 0.008084000000962988
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3917.0253329999978,
- "execMs": 0.0025830000085989013
- },
- {
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3917.165540999995,
- "execMs": 0.0014159999991534278
- }
- ],
- "timings": {
- "gateMs": 1163.6107920000068,
- "gateSkipped": false,
- "ttftMs": 5478.921665999995,
- "modelMs": 5111.251833999995,
- "toolMs": 0.012083000008715317,
- "totalMs": 6274.874709000011,
- "judgeMs": 1545.0772089999955,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 3592.3726250000036,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.005917000002227724
+ }
+ ]
+ },
{
- "roundMs": 2751.0811669999966,
+ "roundMs": 3335.5647080000053,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.008084000000962988
- },
+ "name": "add_clip",
+ "execMs": 0.0052500000019790605
+ }
+ ]
+ },
+ {
+ "roundMs": 3666.1145830000023,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.0025830000085989013
- },
+ "name": "get_state",
+ "execMs": 0.012166999993496574
+ }
+ ]
+ },
+ {
+ "roundMs": 3685.969916000002,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.0014159999991534278
+ "name": "add_clip",
+ "execMs": 0.005374999993364327
}
]
},
{
- "roundMs": 2360.1706669999985,
- "firstDeltaMs": 1557.9960419999989,
+ "roundMs": 2252.020875000002,
+ "firstDeltaMs": 2085.45404099999,
"calls": []
}
]
@@ -1721,235 +1145,142 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_clip_muted",
+ "name": "add_clip",
"round": 0,
- "atMs": 3206.6980840000033,
- "execMs": 0.004207999998470768
+ "atMs": 4524.402624999988,
+ "execMs": 0.004541000002063811
},
{
- "name": "set_clip_muted",
+ "name": "add_clip",
"round": 0,
- "atMs": 3206.8253750000003,
- "execMs": 0.0007079999923007563
+ "atMs": 4524.829999999987,
+ "execMs": 0.0023750000109430403
},
{
- "name": "set_clip_muted",
- "round": 0,
- "atMs": 3206.866209,
- "execMs": 0.000375000003259629
+ "name": "get_state",
+ "round": 1,
+ "atMs": 7152.849041999987,
+ "execMs": 0.014457999990554526
+ },
+ {
+ "name": "add_clip",
+ "round": 2,
+ "atMs": 11383.374249999993,
+ "execMs": 0.0038330000243149698
+ },
+ {
+ "name": "add_clip",
+ "round": 2,
+ "atMs": 11383.536499999987,
+ "execMs": 0.001416999992216006
+ },
+ {
+ "name": "get_state",
+ "round": 3,
+ "atMs": 13699.658916999993,
+ "execMs": 0.02104200000758283
+ },
+ {
+ "name": "add_clip",
+ "round": 4,
+ "atMs": 16223.84941699999,
+ "execMs": 0.008166999992681667
+ },
+ {
+ "name": "get_state",
+ "round": 5,
+ "atMs": 21366.135584000003,
+ "execMs": 0.019665999978315085
}
],
"timings": {
- "gateMs": 1099.5708330000052,
- "gateSkipped": false,
- "ttftMs": 4331.471541999999,
- "modelMs": 3583.189293000003,
- "toolMs": 0.005290999994031154,
- "totalMs": 4682.765417000002,
- "judgeMs": 1056.2835420000047,
+ "gateMs": 0.4308749999909196,
+ "gateSkipped": true,
+ "ttftMs": 24519.073459000007,
+ "modelMs": 24505.098748999953,
+ "toolMs": 0.07549899999867193,
+ "totalMs": 24521.54041700001,
+ "judgeMs": 1139.9844999999914,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2104.1892090000038,
+ "roundMs": 4519.740416000001,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_clip_muted",
- "execMs": 0.004207999998470768
- },
- {
- "name": "set_clip_muted",
- "execMs": 0.0007079999923007563
+ "name": "add_clip",
+ "execMs": 0.004541000002063811
},
{
- "name": "set_clip_muted",
- "execMs": 0.000375000003259629
+ "name": "add_clip",
+ "execMs": 0.0023750000109430403
}
]
},
{
- "roundMs": 1479.0000839999993,
- "firstDeltaMs": 1124.1863750000048,
- "calls": []
- }
- ]
- }
- }
- ]
- },
- {
- "name": "imported-photo-edit-references-photo",
- "bucket": "single-tool",
- "passRate": 1,
- "budgetBreaches": [],
- "latency": {
- "gateMs": {
- "p50": 1357,
- "p95": 1760,
- "mean": 1406,
- "n": 3
- },
- "ttftMs": {
- "p50": 9075,
- "p95": 11338,
- "mean": 9588,
- "n": 3
- },
- "totalMs": {
- "p50": 9316,
- "p95": 11646,
- "mean": 9922,
- "n": 3
- },
- "modelMs": {
- "p50": 8214,
- "p95": 10290,
- "mean": 8516,
- "n": 3
- },
- "toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 3
- },
- "rounds": {
- "mean": 2,
- "max": 2
- }
- },
- "runs": [
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_image",
- "round": 0,
- "atMs": 4581.180332999997,
- "execMs": 0.1023330000025453
- }
- ],
- "timings": {
- "gateMs": 1101.4423340000067,
- "gateSkipped": false,
- "ttftMs": 9074.625415999995,
- "modelMs": 8214.064999999988,
- "toolMs": 0.1023330000025453,
- "totalMs": 9315.609666999997,
- "judgeMs": 1006.6284580000065,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
- {
- "roundMs": 3470.6438749999943,
+ "roundMs": 2625.8867079999764,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.1023330000025453
+ "name": "get_state",
+ "execMs": 0.014457999990554526
}
]
},
{
- "roundMs": 4743.4211249999935,
- "firstDeltaMs": 4492.238375000001,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "generate_image",
- "round": 0,
- "atMs": 5732.386499999993,
- "execMs": 0.01900000000023283
- }
- ],
- "timings": {
- "gateMs": 1356.507958000002,
- "gateSkipped": false,
- "ttftMs": 11338.172582999992,
- "modelMs": 10289.911624999993,
- "toolMs": 0.01900000000023283,
- "totalMs": 11646.438582999996,
- "judgeMs": 2002.9989999999962,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 4228.620708000002,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.0038330000243149698
+ },
+ {
+ "name": "add_clip",
+ "execMs": 0.001416999992216006
+ }
+ ]
+ },
{
- "roundMs": 4373.390625,
+ "roundMs": 2315.2736670000013,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.01900000000023283
+ "name": "get_state",
+ "execMs": 0.02104200000758283
}
]
},
{
- "roundMs": 5916.520999999993,
- "firstDeltaMs": 5604.620624999996,
- "calls": []
- }
- ]
- }
- },
- {
- "pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": "does not explicitly state what changed in the project or timeline as required for an editing turn.",
- "trace": [
- {
- "name": "generate_image",
- "round": 0,
- "atMs": 6737.4905,
- "execMs": 0.014083000001846813
- }
- ],
- "timings": {
- "gateMs": 1760.3349580000067,
- "gateSkipped": false,
- "ttftMs": 8350.598333999995,
- "modelMs": 7042.557709000001,
- "toolMs": 0.014083000001846813,
- "totalMs": 8802.90675000001,
- "judgeMs": 1622.8013329999958,
- "retryMs": 0,
- "retries": 0,
- "rounds": [
+ "roundMs": 2522.8675829999847,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "add_clip",
+ "execMs": 0.008166999992681667
+ }
+ ]
+ },
{
- "roundMs": 4973.934666999994,
+ "roundMs": 5139.648833999992,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_image",
- "execMs": 0.014083000001846813
+ "name": "get_state",
+ "execMs": 0.019665999978315085
}
]
},
{
- "roundMs": 2068.6230420000065,
- "firstDeltaMs": 1611.5269580000022,
+ "roundMs": 3153.060832999996,
+ "firstDeltaMs": 3151.3847499999974,
"calls": []
}
]
@@ -1958,88 +1289,68 @@
]
},
{
- "name": "audio-ask-single-generate",
- "bucket": "single-tool",
- "passRate": 0.6666666666666666,
+ "name": "photos-question-stays-in-chat",
+ "bucket": "chat",
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1961,
- "p95": 2175,
- "mean": 2068,
- "n": 2
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
},
"ttftMs": {
- "p50": 4869,
- "p95": 7609,
- "mean": 6239,
- "n": 2
+ "p50": 3796,
+ "p95": 9432,
+ "mean": 5531,
+ "n": 3
},
"totalMs": {
- "p50": 5299,
- "p95": 8043,
- "mean": 6671,
- "n": 2
+ "p50": 4585,
+ "p95": 10114,
+ "mean": 6327,
+ "n": 3
},
"modelMs": {
- "p50": 3338,
- "p95": 5867,
- "mean": 4603,
- "n": 2
+ "p50": 4582,
+ "p95": 10112,
+ "mean": 6325,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 1,
+ "max": 1
}
},
"runs": [
{
- "pass": false,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "banned phrasing: would you like me to",
- "reply ends with a question after completed work"
- ],
- "judgeNote": "failed to state what changed in concrete terms and included an unnecessary question.",
- "trace": [
- {
- "name": "voiceover_generate",
- "round": 0,
- "atMs": 3728.9431669999904,
- "execMs": 0.008792000007815659
- }
- ],
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": "includes filler list and bolding, contrary to concise and factual requirements.",
+ "trace": [],
"timings": {
- "gateMs": 1699.8382920000004,
- "gateSkipped": false,
- "ttftMs": 5259.466499999995,
- "modelMs": 4228.192292000007,
- "toolMs": 0.008792000007815659,
- "totalMs": 5928.039376000015,
- "judgeMs": 1274.4507079999894,
+ "gateMs": 0.29920800001127645,
+ "gateSkipped": true,
+ "ttftMs": 9432.159249999997,
+ "modelMs": 10111.655667000014,
+ "toolMs": 0,
+ "totalMs": 10114.286250000005,
+ "judgeMs": 1545.4791250000126,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2024.2993750000023,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.008792000007815659
- }
- ]
- },
- {
- "roundMs": 2203.8929170000047,
- "firstDeltaMs": 1529.2497920000023,
+ "roundMs": 10111.655667000014,
+ "firstDeltaMs": 9429.893333999993,
"calls": []
}
]
@@ -2047,85 +1358,51 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
- "judgeNote": "the assistant did not perform the edit as requested and failed to state the action taken.",
- "trace": [
- {
- "name": "voiceover_generate",
- "round": 0,
- "atMs": 6426.81679099999,
- "execMs": 0.011124999989988282
- }
- ],
+ "judgeNote": "includes filler and list structure which contradicts the concise requirement.",
+ "trace": [],
"timings": {
- "gateMs": 2175.430333000011,
- "gateSkipped": false,
- "ttftMs": 7609.286749999999,
- "modelMs": 5867.479916000011,
- "toolMs": 0.011124999989988282,
- "totalMs": 8042.921374000012,
- "judgeMs": 1223.5306249999994,
+ "gateMs": 0.3065000000060536,
+ "gateSkipped": true,
+ "ttftMs": 3795.5814580000006,
+ "modelMs": 4581.852749999991,
+ "toolMs": 0,
+ "totalMs": 4584.550958000007,
+ "judgeMs": 1185.659667,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4249.87879100001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.011124999989988282
- }
- ]
- },
- {
- "roundMs": 1617.601125000001,
- "firstDeltaMs": 1180.6729170000035,
+ "roundMs": 4581.852749999991,
+ "firstDeltaMs": 3793.1336249999877,
"calls": []
}
]
}
},
{
- "pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
- "judgeNote": "editing turn failed to state what changed in concrete terms or offer placement/approval.",
- "trace": [
- {
- "name": "voiceover_generate",
- "round": 0,
- "atMs": 3922.9099590000114,
- "execMs": 0.007874999981140718
- }
- ],
+ "judgeNote": "contains self-promotional list formatting and filler instead of being concise and direct.",
+ "trace": [],
"timings": {
- "gateMs": 1960.9494589999958,
- "gateSkipped": false,
- "ttftMs": 4869.437583999999,
- "modelMs": 3338.3740000000107,
- "toolMs": 0.007874999981140718,
- "totalMs": 5299.331333999988,
- "judgeMs": 1118.1096250000119,
+ "gateMs": 0.16795900001307018,
+ "gateSkipped": true,
+ "ttftMs": 3365.5846249999886,
+ "modelMs": 4280.409333000018,
+ "toolMs": 0,
+ "totalMs": 4282.13499999998,
+ "judgeMs": 1223.6891659999965,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1958.6070419999887,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "voiceover_generate",
- "execMs": 0.007874999981140718
- }
- ]
- },
- {
- "roundMs": 1379.766958000022,
- "firstDeltaMs": 943.9545830000134,
+ "roundMs": 4280.409333000018,
+ "firstDeltaMs": 3364.0880830000096,
"calls": []
}
]
@@ -2134,33 +1411,33 @@
]
},
{
- "name": "music-ask-single-generate",
- "bucket": "single-tool",
+ "name": "clutter-complaint-deletes-nothing",
+ "bucket": "chat",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1168,
- "p95": 1922,
- "mean": 1419,
+ "p50": 999,
+ "p95": 1019,
+ "mean": 968,
"n": 3
},
"ttftMs": {
- "p50": 4174,
- "p95": 5350,
- "mean": 4484,
+ "p50": 3976,
+ "p95": 4023,
+ "mean": 3807,
"n": 3
},
"totalMs": {
- "p50": 4891,
- "p95": 5623,
- "mean": 4911,
+ "p50": 4139,
+ "p95": 4160,
+ "mean": 3975,
"n": 3
},
"modelMs": {
- "p50": 3701,
- "p95": 3724,
- "mean": 3492,
+ "p50": 4137,
+ "p95": 4157,
+ "mean": 3972,
"n": 3
},
"toolMs": {
@@ -2177,42 +1454,30 @@
"runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "chat",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
- "trace": [
- {
- "name": "generate_music",
- "round": 0,
- "atMs": 2744.4827499999956,
- "execMs": 0.05770900001516566
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 1166.6052499999932,
+ "gateMs": 1018.6645830000052,
"gateSkipped": false,
- "ttftMs": 4173.906666999974,
- "modelMs": 3724.2767920000188,
- "toolMs": 0.05770900001516566,
- "totalMs": 4890.939751000027,
- "judgeMs": 1558.3744589999842,
+ "ttftMs": 4023.348457999993,
+ "modelMs": 4156.531707999995,
+ "toolMs": 0,
+ "totalMs": 4159.913292000012,
+ "judgeMs": 1288.322165999998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1571.5885420000122,
+ "roundMs": 1017.4890829999931,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.05770900001516566
- }
- ]
+ "calls": []
},
{
- "roundMs": 2152.6882500000065,
- "firstDeltaMs": 1427.686875000014,
+ "roundMs": 3139.042625000002,
+ "firstDeltaMs": 3002.773125000007,
"calls": []
}
]
@@ -2220,42 +1485,30 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "chat",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
- "trace": [
- {
- "name": "generate_music",
- "round": 0,
- "atMs": 2664.921207999985,
- "execMs": 0.005541999998968095
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 1168.3784589999996,
+ "gateMs": 999.4806249999965,
"gateSkipped": false,
- "ttftMs": 3927.6446659999783,
- "modelMs": 3050.835167000012,
- "toolMs": 0.005541999998968095,
- "totalMs": 4219.2191680000105,
- "judgeMs": 1183.1158340000256,
+ "ttftMs": 3422.882499999978,
+ "modelMs": 3623.404459000012,
+ "toolMs": 0,
+ "totalMs": 3626.0681669999904,
+ "judgeMs": 1067.8349999999919,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1494.159249999997,
+ "roundMs": 998.0565420000057,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.005541999998968095
- }
- ]
+ "calls": []
},
{
- "roundMs": 1556.675917000015,
- "firstDeltaMs": 1261.3280000000086,
+ "roundMs": 2625.3479170000064,
+ "firstDeltaMs": 2422.437249999988,
"calls": []
}
]
@@ -2263,42 +1516,30 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "chat",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
- "trace": [
- {
- "name": "generate_music",
- "round": 0,
- "atMs": 4043.8962919999904,
- "execMs": 0.0009169999975711107
- }
- ],
+ "trace": [],
"timings": {
- "gateMs": 1922.238458000007,
+ "gateMs": 886.5164169999771,
"gateSkipped": false,
- "ttftMs": 5350.096457999985,
- "modelMs": 3701.044875999971,
- "toolMs": 0.0009169999975711107,
- "totalMs": 5623.2842509999755,
- "judgeMs": 980.3183330000029,
+ "ttftMs": 3975.787666999997,
+ "modelMs": 4136.723333000002,
+ "toolMs": 0,
+ "totalMs": 4138.84941699999,
+ "judgeMs": 995.8177499999874,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2119.050833999994,
+ "roundMs": 885.9137080000073,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "generate_music",
- "execMs": 0.0009169999975711107
- }
- ]
+ "calls": []
},
{
- "roundMs": 1581.9940419999766,
- "firstDeltaMs": 1305.8496249999735,
+ "roundMs": 3250.8096249999944,
+ "firstDeltaMs": 3087.9022500000137,
"calls": []
}
]
@@ -2307,33 +1548,33 @@
]
},
{
- "name": "trim-ask-single-tool",
+ "name": "video-ask-single-generate",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1034,
- "p95": 1941,
- "mean": 1289,
+ "p50": 1044,
+ "p95": 2039,
+ "mean": 1367,
"n": 3
},
"ttftMs": {
- "p50": 4870,
- "p95": 4978,
- "mean": 4550,
+ "p50": 2605,
+ "p95": 3613,
+ "mean": 2892,
"n": 3
},
"totalMs": {
- "p50": 5142,
- "p95": 5160,
- "mean": 4771,
+ "p50": 2937,
+ "p95": 4451,
+ "mean": 3367,
"n": 3
},
"modelMs": {
- "p50": 3200,
- "p95": 4126,
- "mean": 3482,
+ "p50": 2928,
+ "p95": 3915,
+ "mean": 3184,
"n": 3
},
"toolMs": {
@@ -2356,36 +1597,36 @@
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "generate_video",
"round": 0,
- "atMs": 2341.239499999996,
- "execMs": 0.011041999998269603
+ "atMs": 1299.2413330000127,
+ "execMs": 0.0105829999956768
}
],
"timings": {
- "gateMs": 893.2194580000069,
+ "gateMs": 1017.9502500000235,
"gateSkipped": false,
- "ttftMs": 3800.93733299998,
- "modelMs": 3118.1342920000025,
- "toolMs": 0.011041999998269603,
- "totalMs": 4011.3647920000076,
- "judgeMs": 1534.2747080000117,
+ "ttftMs": 2604.6957500000135,
+ "modelMs": 2928.2186659999716,
+ "toolMs": 0.0105829999956768,
+ "totalMs": 2936.8464169999934,
+ "judgeMs": 1152.3697089999914,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1443.4872920000053,
+ "roundMs": 1291.8809159999946,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.011041999998269603
+ "name": "generate_video",
+ "execMs": 0.0105829999956768
}
]
},
{
- "roundMs": 1674.6469999999972,
- "firstDeltaMs": 1458.2413749999832,
+ "roundMs": 1636.337749999977,
+ "firstDeltaMs": 1304.3766659999965,
"calls": []
}
]
@@ -2399,36 +1640,36 @@
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "generate_video",
"round": 0,
- "atMs": 3497.3065420000057,
- "execMs": 0.0071659999957773834
+ "atMs": 1285.855958,
+ "execMs": 0.005707999982405454
}
],
"timings": {
- "gateMs": 1941.2230410000193,
+ "gateMs": 1044.4083330000285,
"gateSkipped": false,
- "ttftMs": 4978.370083999995,
- "modelMs": 3200.454000000027,
- "toolMs": 0.0071659999957773834,
- "totalMs": 5141.684207000042,
- "judgeMs": 1043.0578340000066,
+ "ttftMs": 2457.7862920000043,
+ "modelMs": 2709.105833999987,
+ "toolMs": 0.005707999982405454,
+ "totalMs": 2712.599499999982,
+ "judgeMs": 1007.8175420000043,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1553.9718330000178,
+ "roundMs": 1283.4550419999869,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.0071659999957773834
+ "name": "generate_video",
+ "execMs": 0.005707999982405454
}
]
},
{
- "roundMs": 1646.4821670000092,
- "firstDeltaMs": 1479.5018339999951,
+ "roundMs": 1425.6507920000004,
+ "firstDeltaMs": 1171.2049999999872,
"calls": []
}
]
@@ -2442,36 +1683,36 @@
"judgeNote": null,
"trace": [
{
- "name": "trim_clip",
+ "name": "generate_video",
"round": 0,
- "atMs": 3469.6101250000065,
- "execMs": 0.009249999973690137
+ "atMs": 2039.3513749999984,
+ "execMs": 0.008792000007815659
}
],
"timings": {
- "gateMs": 1033.6389999999956,
+ "gateMs": 2039.0194999999949,
"gateSkipped": false,
- "ttftMs": 4869.926124999998,
- "modelMs": 4126.357707999967,
- "toolMs": 0.009249999973690137,
- "totalMs": 5160.005957999936,
- "judgeMs": 665.0173749999958,
+ "ttftMs": 3612.8035839999793,
+ "modelMs": 3915.338792000024,
+ "toolMs": 0.008792000007815659,
+ "totalMs": 4450.538084,
+ "judgeMs": 1063.8595839999907,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2433.7336659999855,
+ "roundMs": 1505.5185420000053,
"firstDeltaMs": null,
"calls": [
{
- "name": "trim_clip",
- "execMs": 0.009249999973690137
+ "name": "generate_video",
+ "execMs": 0.008792000007815659
}
]
},
{
- "roundMs": 1692.6240419999813,
- "firstDeltaMs": 1397.010083000001,
+ "roundMs": 2409.820250000019,
+ "firstDeltaMs": 1572.3080410000111,
"calls": []
}
]
@@ -2480,33 +1721,33 @@
]
},
{
- "name": "transition-ask-styled",
+ "name": "iterate-keeps-existing-clip",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1049,
- "p95": 1080,
- "mean": 1039,
+ "p50": 1125,
+ "p95": 1589,
+ "mean": 1274,
"n": 3
},
"ttftMs": {
- "p50": 4667,
- "p95": 6599,
- "mean": 5162,
+ "p50": 8666,
+ "p95": 12466,
+ "mean": 9679,
"n": 3
},
"totalMs": {
- "p50": 5358,
- "p95": 6721,
- "mean": 5517,
+ "p50": 8887,
+ "p95": 12692,
+ "mean": 9894,
"n": 3
},
"modelMs": {
- "p50": 4370,
- "p95": 5641,
- "mean": 4477,
+ "p50": 8880,
+ "p95": 12688,
+ "mean": 9887,
"n": 3
},
"toolMs": {
@@ -2516,49 +1757,54 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 2871.2307920000167,
- "execMs": 0.10854200000176206
+ "name": "generate_video",
+ "round": 1,
+ "atMs": 5808.098457999993,
+ "execMs": 0.4249580000177957
}
],
"timings": {
- "gateMs": 987.7601660000219,
+ "gateMs": 1106.5414170000004,
"gateSkipped": false,
- "ttftMs": 4667.087041999999,
- "modelMs": 4369.901126000012,
- "toolMs": 0.10854200000176206,
- "totalMs": 5357.769834000035,
- "judgeMs": 1705.3454590000038,
+ "ttftMs": 8666.234582999983,
+ "modelMs": 8879.914042999997,
+ "toolMs": 0.4249580000177957,
+ "totalMs": 8886.599541999982,
+ "judgeMs": 1024.8275840000133,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1878.7014170000039,
+ "roundMs": 1105.986917000002,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4698.145709000004,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.10854200000176206
+ "name": "generate_video",
+ "execMs": 0.4249580000177957
}
]
},
{
- "roundMs": 2491.1997090000077,
- "firstDeltaMs": 1794.2004590000142,
+ "roundMs": 3075.781416999991,
+ "firstDeltaMs": 2855.6023749999877,
"calls": []
}
]
@@ -2566,42 +1812,47 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 2488.6279159999976,
- "execMs": 0.015625
+ "name": "generate_video",
+ "round": 1,
+ "atMs": 9028.588958000008,
+ "execMs": 0.021500000002561137
}
],
"timings": {
- "gateMs": 1049.4243330000027,
+ "gateMs": 1589.2670409999846,
"gateSkipped": false,
- "ttftMs": 4218.372957999993,
- "modelMs": 3421.43187499998,
- "toolMs": 0.015625,
- "totalMs": 4470.871832999983,
- "judgeMs": 982.6664579999924,
+ "ttftMs": 12466.115083000012,
+ "modelMs": 12687.800541999983,
+ "toolMs": 0.021500000002561137,
+ "totalMs": 12692.396750000014,
+ "judgeMs": 1988.3183750000026,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1436.1326249999984,
+ "roundMs": 1587.6192079999892,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 7437.672583999985,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.015625
+ "name": "generate_video",
+ "execMs": 0.021500000002561137
}
]
},
{
- "roundMs": 1985.2992499999818,
- "firstDeltaMs": 1728.2534999999916,
+ "roundMs": 3662.508750000008,
+ "firstDeltaMs": 3436.438374999998,
"calls": []
}
]
@@ -2609,42 +1860,47 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 5147.19754199998,
- "execMs": 0.00979199999710545
+ "name": "generate_video",
+ "round": 1,
+ "atMs": 5275.527792000008,
+ "execMs": 0.013042000005953014
}
],
"timings": {
- "gateMs": 1080.3517920000013,
+ "gateMs": 1125.2969999999914,
"gateSkipped": false,
- "ttftMs": 6599.162707999989,
- "modelMs": 5640.641291999986,
- "toolMs": 0.00979199999710545,
- "totalMs": 6721.002875999984,
- "judgeMs": 1306.8071660000132,
+ "ttftMs": 7905.0895420000015,
+ "modelMs": 8093.165415999974,
+ "toolMs": 0.013042000005953014,
+ "totalMs": 8102.304999999993,
+ "judgeMs": 848.6339169999992,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4064.380083999975,
+ "roundMs": 1122.2649580000143,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4146.219790999981,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.00979199999710545
+ "name": "generate_video",
+ "execMs": 0.013042000005953014
}
]
},
{
- "roundMs": 1576.2612080000108,
- "firstDeltaMs": 1450.1052499999932,
+ "roundMs": 2824.6806669999787,
+ "firstDeltaMs": 2628.1128749999916,
"calls": []
}
]
@@ -2653,33 +1909,33 @@
]
},
{
- "name": "slide-in-direction-correct",
+ "name": "mute-all-does-not-redo-prior-turn",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1101,
- "p95": 2229,
- "mean": 1473,
+ "p50": 1517,
+ "p95": 1969,
+ "mean": 1514,
"n": 3
},
"ttftMs": {
- "p50": 5118,
- "p95": 11277,
- "mean": 6710,
+ "p50": 8590,
+ "p95": 10157,
+ "mean": 7317,
"n": 3
},
"totalMs": {
- "p50": 5347,
- "p95": 11920,
- "mean": 7069,
+ "p50": 8591,
+ "p95": 10179,
+ "mean": 7364,
"n": 3
},
"modelMs": {
- "p50": 3118,
- "p95": 10819,
- "mean": 5596,
+ "p50": 8581,
+ "p95": 10169,
+ "mean": 7357,
"n": 3
},
"toolMs": {
@@ -2689,49 +1945,74 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 4
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_animation",
- "round": 0,
- "atMs": 3810.6886670000004,
- "execMs": 0.04279199999928096
+ "name": "set_clip_muted",
+ "round": 1,
+ "atMs": 5964.722125,
+ "execMs": 0.08354200000758283
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 1,
+ "atMs": 5965.411999999982,
+ "execMs": 0.015583000000333413
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 1,
+ "atMs": 5965.584124999994,
+ "execMs": 0.0012089999800082296
}
],
"timings": {
- "gateMs": 2229.0063330000003,
+ "gateMs": 1056.3727910000016,
"gateSkipped": false,
- "ttftMs": 5118.017333,
- "modelMs": 3117.9767079999992,
- "toolMs": 0.04279199999928096,
- "totalMs": 5347.025832999999,
- "judgeMs": 1844.4930829999994,
+ "ttftMs": 8589.884625000006,
+ "modelMs": 8581.354957999982,
+ "toolMs": 0.10033399998792447,
+ "totalMs": 8590.777457999997,
+ "judgeMs": 896.9985420000157,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1565.4869579999995,
+ "roundMs": 1054.28125,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4903.310165999981,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.04279199999928096
+ "name": "set_clip_muted",
+ "execMs": 0.08354200000758283
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.015583000000333413
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0012089999800082296
}
]
},
{
- "roundMs": 1552.4897499999997,
- "firstDeltaMs": 1305.6425829999998,
+ "roundMs": 2623.7635420000006,
+ "firstDeltaMs": 2623.061625000002,
"calls": []
}
]
@@ -2739,42 +2020,72 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_animation",
- "round": 0,
- "atMs": 5614.405542,
- "execMs": 0.013249999999970896
+ "name": "set_clip_muted",
+ "round": 2,
+ "atMs": 7247.186042000016,
+ "execMs": 0.006792000000132248
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 2,
+ "atMs": 7247.410292000015,
+ "execMs": 0.0009589999972376972
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 2,
+ "atMs": 7247.5032920000085,
+ "execMs": 0.0005830000154674053
}
],
"timings": {
- "gateMs": 1101.1039579999997,
+ "gateMs": 1968.711707999988,
"gateSkipped": false,
- "ttftMs": 11277.413499999999,
- "modelMs": 10819.189166999999,
- "toolMs": 0.013249999999970896,
- "totalMs": 11920.306374999998,
- "judgeMs": 2039.3310840000013,
+ "ttftMs": 10157.192375000013,
+ "modelMs": 10169.06408499999,
+ "toolMs": 0.00833400001283735,
+ "totalMs": 10178.960374999995,
+ "judgeMs": 1064.9554160000116,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4509.250167,
+ "roundMs": 1963.3322089999856,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 0.012667000002693385,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 5275.5905839999905,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.013249999999970896
+ "name": "set_clip_muted",
+ "execMs": 0.006792000000132248
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0009589999972376972
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0005830000154674053
}
]
},
{
- "roundMs": 6309.9389999999985,
- "firstDeltaMs": 5660.954999999998,
+ "roundMs": 2930.128625000012,
+ "firstDeltaMs": 2908.5801670000073,
"calls": []
}
]
@@ -2788,36 +2099,56 @@
"judgeNote": null,
"trace": [
{
- "name": "set_animation",
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2015.978792000038,
+ "execMs": 0.006124999956227839
+ },
+ {
+ "name": "set_clip_muted",
+ "round": 0,
+ "atMs": 2016.2934590000077,
+ "execMs": 0.002957999997306615
+ },
+ {
+ "name": "set_clip_muted",
"round": 0,
- "atMs": 2437.9349580000016,
- "execMs": 0.019499999998515705
+ "atMs": 2016.3934590000426,
+ "execMs": 0.0012500000302679837
}
],
"timings": {
- "gateMs": 1087.6784170000028,
+ "gateMs": 1516.8079170000274,
"gateSkipped": false,
- "ttftMs": 3733.237792,
- "modelMs": 2852.1087919999954,
- "toolMs": 0.019499999998515705,
- "totalMs": 3939.8067089999968,
- "judgeMs": 1550.5267920000006,
+ "ttftMs": 3204.917375000019,
+ "modelMs": 3319.433290999965,
+ "toolMs": 0.010332999983802438,
+ "totalMs": 3323.5751670000027,
+ "judgeMs": 1101.993542000011,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1346.726083999998,
+ "roundMs": 2013.8921249999548,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_animation",
- "execMs": 0.019499999998515705
+ "name": "set_clip_muted",
+ "execMs": 0.006124999956227839
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.002957999997306615
+ },
+ {
+ "name": "set_clip_muted",
+ "execMs": 0.0012500000302679837
}
]
},
{
- "roundMs": 1505.3827079999974,
- "firstDeltaMs": 1293.4027499999975,
+ "roundMs": 1305.54116600001,
+ "firstDeltaMs": 1187.149040999997,
"calls": []
}
]
@@ -2826,33 +2157,33 @@
]
},
{
- "name": "look-ask-single-tool",
+ "name": "imported-photo-edit-references-photo",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1449,
- "p95": 1844,
- "mean": 1474,
+ "p50": 1053,
+ "p95": 1071,
+ "mean": 1049,
"n": 3
},
"ttftMs": {
- "p50": 4601,
- "p95": 5168,
- "mean": 4615,
+ "p50": 8969,
+ "p95": 10235,
+ "mean": 9316,
"n": 3
},
"totalMs": {
- "p50": 4824,
- "p95": 5416,
- "mean": 4963,
+ "p50": 9153,
+ "p95": 10456,
+ "mean": 9504,
"n": 3
},
"modelMs": {
- "p50": 3520,
- "p95": 3571,
- "mean": 3489,
+ "p50": 9146,
+ "p95": 10451,
+ "mean": 9498,
"n": 3
},
"toolMs": {
@@ -2862,49 +2193,54 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
- "round": 0,
- "atMs": 3993.3052499999976,
- "execMs": 0.03299999999944703
+ "name": "generate_image",
+ "round": 1,
+ "atMs": 5451.5324580000015,
+ "execMs": 0.07512499997392297
}
],
"timings": {
- "gateMs": 1844.3342919999996,
+ "gateMs": 1071.4392920000246,
"gateSkipped": false,
- "ttftMs": 5168.402291000002,
- "modelMs": 3571.335750000002,
- "toolMs": 0.03299999999944703,
- "totalMs": 5415.703042000001,
- "judgeMs": 1173.9096250000002,
+ "ttftMs": 8968.940958000021,
+ "modelMs": 9145.912624999997,
+ "toolMs": 0.07512499997392297,
+ "totalMs": 9153.212958000018,
+ "judgeMs": 906.2155830000411,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2142.2010420000006,
+ "roundMs": 1070.613874999981,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4374.9537080000155,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.03299999999944703
+ "name": "generate_image",
+ "execMs": 0.07512499997392297
}
]
},
{
- "roundMs": 1429.1347080000014,
- "firstDeltaMs": 1174.6121660000026,
+ "roundMs": 3700.345042000001,
+ "firstDeltaMs": 3516.6372089999495,
"calls": []
}
]
@@ -2912,42 +2248,47 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
- "round": 0,
- "atMs": 3543.826458000003,
- "execMs": 0.0020839999997406267
+ "name": "generate_image",
+ "round": 1,
+ "atMs": 5470.562291000038,
+ "execMs": 0.026165999996010214
}
],
"timings": {
- "gateMs": 1448.5215000000026,
+ "gateMs": 1053.2412089999998,
"gateSkipped": false,
- "ttftMs": 4600.839458000002,
- "modelMs": 3375.511541999993,
- "toolMs": 0.0020839999997406267,
- "totalMs": 4824.035125999995,
- "judgeMs": 1144.0252089999994,
+ "ttftMs": 10235.048708000046,
+ "modelMs": 10450.810709000041,
+ "toolMs": 0.026165999996010214,
+ "totalMs": 10456.242166000011,
+ "judgeMs": 1732.5662500000326,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2093.5255419999958,
+ "roundMs": 1051.8834170000046,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4414.898333000019,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.0020839999997406267
+ "name": "generate_image",
+ "execMs": 0.026165999996010214
}
]
},
{
- "roundMs": 1281.9859999999971,
- "firstDeltaMs": 1056.633332999998,
+ "roundMs": 4984.0289590000175,
+ "firstDeltaMs": 4763.069749999966,
"calls": []
}
]
@@ -2955,42 +2296,47 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_effect",
- "round": 0,
- "atMs": 2715.1397920000018,
- "execMs": 0.011459000001195818
+ "name": "generate_image",
+ "round": 1,
+ "atMs": 4711.853166999994,
+ "execMs": 0.014374999969732016
}
],
"timings": {
- "gateMs": 1130.4469160000008,
+ "gateMs": 1023.6219169999822,
"gateSkipped": false,
- "ttftMs": 4077.1619580000042,
- "modelMs": 3519.7674999999945,
- "toolMs": 0.011459000001195818,
- "totalMs": 4650.225874999996,
- "judgeMs": 1834.577416,
+ "ttftMs": 8743.302208999987,
+ "modelMs": 8895.867667000042,
+ "toolMs": 0.014374999969732016,
+ "totalMs": 8901.882041999954,
+ "judgeMs": 907.602125000034,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1581.8143749999945,
+ "roundMs": 1021.7506670000148,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3686.4589590000105,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_effect",
- "execMs": 0.011459000001195818
+ "name": "generate_image",
+ "execMs": 0.014374999969732016
}
]
},
{
- "roundMs": 1937.953125,
- "firstDeltaMs": 1360.5665420000005,
+ "roundMs": 4187.658041000017,
+ "firstDeltaMs": 4029.4950410000165,
"calls": []
}
]
@@ -2999,33 +2345,33 @@
]
},
{
- "name": "animation-overrides-transition",
+ "name": "audio-ask-single-generate",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1784,
- "p95": 1842,
- "mean": 1766,
+ "p50": 1533,
+ "p95": 1838,
+ "mean": 1633,
"n": 3
},
"ttftMs": {
- "p50": 8626,
- "p95": 10678,
- "mean": 9140,
+ "p50": 3057,
+ "p95": 5651,
+ "mean": 3761,
"n": 3
},
"totalMs": {
- "p50": 8788,
- "p95": 10994,
- "mean": 9387,
+ "p50": 3350,
+ "p95": 5793,
+ "mean": 3954,
"n": 3
},
"modelMs": {
- "p50": 7118,
- "p95": 9151,
- "mean": 7621,
+ "p50": 3342,
+ "p95": 5789,
+ "mean": 3947,
"n": 3
},
"toolMs": {
@@ -3042,52 +2388,42 @@
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 8791.210708999999,
- "execMs": 0.078125
- },
- {
- "name": "set_animation",
+ "name": "voiceover_generate",
"round": 0,
- "atMs": 8791.778624999999,
- "execMs": 0.003540999998222105
+ "atMs": 1700.0957499999786,
+ "execMs": 0.03291700000409037
}
],
"timings": {
- "gateMs": 1842.1546250000029,
+ "gateMs": 1532.810999999987,
"gateSkipped": false,
- "ttftMs": 10677.501459,
- "modelMs": 9151.498375000003,
- "toolMs": 0.0816659999982221,
- "totalMs": 10993.734666000004,
- "judgeMs": 955.9993340000001,
+ "ttftMs": 3057.1424579999875,
+ "modelMs": 3342.3412499999977,
+ "toolMs": 0.03291700000409037,
+ "totalMs": 3350.1117920000106,
+ "judgeMs": 1886.6240830000024,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6940.993625000003,
+ "roundMs": 1694.7835000000196,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.078125
- },
- {
- "name": "set_animation",
- "execMs": 0.003540999998222105
+ "name": "voiceover_generate",
+ "execMs": 0.03291700000409037
}
]
},
{
- "roundMs": 2210.50475,
- "firstDeltaMs": 1884.5993750000052,
+ "roundMs": 1647.5577499999781,
+ "firstDeltaMs": 1354.9294169999775,
"calls": []
}
]
@@ -3095,52 +2431,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": null,
+ "judgeNote": "failed to report the specific tool action and did not place the clip despite having the capability to do so.",
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 6783.634334000002,
- "execMs": 0.0032910000008996576
- },
- {
- "name": "set_animation",
+ "name": "voiceover_generate",
"round": 0,
- "atMs": 6783.917834,
- "execMs": 0.0020000000004074536
+ "atMs": 4378.363666000019,
+ "execMs": 0.0047500000218860805
}
],
"timings": {
- "gateMs": 1670.439624999999,
+ "gateMs": 1837.8528330000117,
"gateSkipped": false,
- "ttftMs": 8626.219708999997,
- "modelMs": 7117.609916999994,
- "toolMs": 0.005291000001307111,
- "totalMs": 8788.054832999995,
- "judgeMs": 1796.8855410000106,
+ "ttftMs": 5651.093791000021,
+ "modelMs": 5789.496667000058,
+ "toolMs": 0.0047500000218860805,
+ "totalMs": 5792.956333000038,
+ "judgeMs": 1206.63391699997,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5109.921416999998,
+ "roundMs": 4376.3344170000055,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.0032910000008996576
- },
- {
- "name": "set_animation",
- "execMs": 0.0020000000004074536
+ "name": "voiceover_generate",
+ "execMs": 0.0047500000218860805
}
]
},
{
- "roundMs": 2007.6884999999966,
- "firstDeltaMs": 1840.9062919999924,
+ "roundMs": 1413.1622500000522,
+ "firstDeltaMs": 1271.5903750000289,
"calls": []
}
]
@@ -3148,52 +2474,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
- "round": 0,
- "atMs": 6382.127334000004,
- "execMs": 0.0022499999904539436
- },
- {
- "name": "set_animation",
+ "name": "voiceover_generate",
"round": 0,
- "atMs": 6382.372000000003,
- "execMs": 0.0015839999978197739
+ "atMs": 1529.0014160000137,
+ "execMs": 0.0039579999865964055
}
],
"timings": {
- "gateMs": 1784.381583999988,
+ "gateMs": 1528.721084000019,
"gateSkipped": false,
- "ttftMs": 8117.488959000009,
- "modelMs": 6594.456666999991,
- "toolMs": 0.0038339999882737175,
- "totalMs": 8378.842084999967,
- "judgeMs": 1047.1532080000034,
+ "ttftMs": 2574.5761250000214,
+ "modelMs": 2710.5195420000236,
+ "toolMs": 0.0039579999865964055,
+ "totalMs": 2718.1632910000044,
+ "judgeMs": 1101.9649579999968,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4595.841667000001,
+ "roundMs": 1522.7087080000201,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.0022499999904539436
- },
- {
- "name": "set_animation",
- "execMs": 0.0015839999978197739
+ "name": "voiceover_generate",
+ "execMs": 0.0039579999865964055
}
]
},
{
- "roundMs": 1998.6149999999907,
- "firstDeltaMs": 1734.5484169999982,
+ "roundMs": 1187.8108340000035,
+ "firstDeltaMs": 1044.459124999994,
"calls": []
}
]
@@ -3202,40 +2518,40 @@
]
},
{
- "name": "edit-request-still-uses-tools",
+ "name": "music-ask-single-generate",
"bucket": "single-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1080,
- "p95": 1617,
- "mean": 1348,
- "n": 2
+ "p50": 1881,
+ "p95": 2644,
+ "mean": 1873,
+ "n": 3
},
"ttftMs": {
- "p50": 4367,
- "p95": 5239,
- "mean": 4803,
- "n": 2
+ "p50": 2972,
+ "p95": 3824,
+ "mean": 3137,
+ "n": 3
},
"totalMs": {
- "p50": 4546,
- "p95": 6010,
- "mean": 5278,
- "n": 2
+ "p50": 3279,
+ "p95": 4266,
+ "mean": 3505,
+ "n": 3
},
"modelMs": {
- "p50": 2929,
- "p95": 4931,
- "mean": 3930,
- "n": 2
+ "p50": 2965,
+ "p95": 3066,
+ "mean": 2986,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
"mean": 2,
@@ -3244,46 +2560,43 @@
},
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "simple",
"roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "banned phrasing: would you like me to",
- "reply ends with a question after completed work"
- ],
- "judgeNote": "includes an unnecessary question instead of being decisive.",
+ "notes": [],
+ "judgeNote": null,
"trace": [
{
- "name": "subtitles_generate",
+ "name": "generate_music",
"round": 0,
- "atMs": 6295.389165999994,
- "execMs": 0.008540999988326803
+ "atMs": 2644.8367919999873,
+ "execMs": 0.3267500000074506
}
],
"timings": {
- "gateMs": 1236.3779579999973,
+ "gateMs": 2644.0936249999795,
"gateSkipped": false,
- "ttftMs": 7565.974208,
- "modelMs": 6619.3318750000035,
- "toolMs": 0.008540999988326803,
- "totalMs": 7855.718373999989,
- "judgeMs": 1052.966916999998,
+ "ttftMs": 3823.8966670000227,
+ "modelMs": 3065.8347509999876,
+ "toolMs": 0.3267500000074506,
+ "totalMs": 4266.2813750000205,
+ "judgeMs": 993.905792000005,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5054.065875,
+ "roundMs": 1446.8140840000124,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.008540999988326803
+ "name": "generate_music",
+ "execMs": 0.3267500000074506
}
]
},
{
- "roundMs": 1565.2660000000033,
- "firstDeltaMs": 1268.640167000005,
+ "roundMs": 1619.0206669999752,
+ "firstDeltaMs": 1176.9354999999632,
"calls": []
}
]
@@ -3297,36 +2610,36 @@
"judgeNote": null,
"trace": [
{
- "name": "subtitles_generate",
+ "name": "generate_music",
"round": 0,
- "atMs": 3854.8450410000078,
- "execMs": 0.007041999997454695
+ "atMs": 1881.1422080000048,
+ "execMs": 0.011832999996840954
}
],
"timings": {
- "gateMs": 1079.525833000007,
+ "gateMs": 1880.875457999995,
"gateSkipped": false,
- "ttftMs": 5238.876791000002,
- "modelMs": 4930.8359170000185,
- "toolMs": 0.007041999997454695,
- "totalMs": 6010.368792000023,
- "judgeMs": 1846.1411249999946,
+ "ttftMs": 2972.1590419999557,
+ "modelMs": 2926.754917000013,
+ "toolMs": 0.011832999996840954,
+ "totalMs": 3279.4256249999744,
+ "judgeMs": 2055.154957999999,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2772.699375000011,
+ "roundMs": 1530.9637499999953,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.007041999997454695
+ "name": "generate_music",
+ "execMs": 0.011832999996840954
}
]
},
{
- "roundMs": 2158.1365420000075,
- "firstDeltaMs": 1382.3067500000034,
+ "roundMs": 1395.7911670000176,
+ "firstDeltaMs": 1088.8744580000057,
"calls": []
}
]
@@ -3337,39 +2650,39 @@
"intent": "simple",
"roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "failed to state what changed in concrete terms (e.g., specifying the duration or style of captions).",
+ "judgeNote": null,
"trace": [
{
- "name": "subtitles_generate",
+ "name": "generate_music",
"round": 0,
- "atMs": 3141.281541999997,
- "execMs": 0.008874999999534339
+ "atMs": 1478.7429580000462,
+ "execMs": 0.007624999969266355
}
],
"timings": {
- "gateMs": 1617.3688330000004,
+ "gateMs": 1092.6510830000043,
"gateSkipped": false,
- "ttftMs": 4366.647708999997,
- "modelMs": 2928.960375000024,
- "toolMs": 0.008874999999534339,
- "totalMs": 4546.338083000024,
- "judgeMs": 2031.9173750000045,
+ "ttftMs": 2616.122250000015,
+ "modelMs": 2965.4589169999235,
+ "toolMs": 0.007624999969266355,
+ "totalMs": 2969.112542000017,
+ "judgeMs": 916.1081659999909,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1521.2808330000116,
+ "roundMs": 1477.0544999999693,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.008874999999534339
+ "name": "generate_music",
+ "execMs": 0.007624999969266355
}
]
},
{
- "roundMs": 1407.6795420000126,
- "firstDeltaMs": 1223.2073750000127,
+ "roundMs": 1488.4044169999543,
+ "firstDeltaMs": 1135.6620419999817,
"calls": []
}
]
@@ -3378,33 +2691,33 @@
]
},
{
- "name": "cartoon-from-audio-plans-scene",
+ "name": "trim-ask-single-tool",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1185,
- "p95": 1690,
- "mean": 1303,
+ "p50": 1141,
+ "p95": 1776,
+ "mean": 1321,
"n": 3
},
"ttftMs": {
- "p50": 9898,
- "p95": 11150,
- "mean": 9587,
+ "p50": 2929,
+ "p95": 3493,
+ "mean": 2947,
"n": 3
},
"totalMs": {
- "p50": 10303,
- "p95": 11463,
- "mean": 9970,
+ "p50": 3175,
+ "p95": 3667,
+ "mean": 3154,
"n": 3
},
"modelMs": {
- "p50": 8613,
- "p95": 10279,
- "mean": 8667,
+ "p50": 2675,
+ "p95": 3664,
+ "mean": 2985,
"n": 3
},
"toolMs": {
@@ -3421,42 +2734,42 @@
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "trim_clip",
"round": 0,
- "atMs": 7860.8502919999955,
- "execMs": 0.05570799999986775
+ "atMs": 1776.7467499999912,
+ "execMs": 0.055333000025711954
}
],
"timings": {
- "gateMs": 1689.9393750000017,
+ "gateMs": 1776.2614590000012,
"gateSkipped": false,
- "ttftMs": 9897.533917,
- "modelMs": 8613.239250000006,
- "toolMs": 0.05570799999986775,
- "totalMs": 10303.234333000008,
- "judgeMs": 879.6468339999992,
+ "ttftMs": 2929.383165999956,
+ "modelMs": 2675.4219579999917,
+ "toolMs": 0.055333000025711954,
+ "totalMs": 3174.673207999964,
+ "judgeMs": 1016.290083000029,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6166.519917000005,
+ "roundMs": 1278.9317080000183,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.05570799999986775
+ "name": "trim_clip",
+ "execMs": 0.055333000025711954
}
]
},
{
- "roundMs": 2446.719333000001,
- "firstDeltaMs": 2035.7157080000034,
+ "roundMs": 1396.4902499999735,
+ "firstDeltaMs": 1151.3965409999946,
"calls": []
}
]
@@ -3464,42 +2777,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "trim_clip",
"round": 0,
- "atMs": 6130.794541000003,
- "execMs": 0.008374999997613486
+ "atMs": 2103.4463749999995,
+ "execMs": 0.0006670000148005784
}
],
"timings": {
- "gateMs": 1033.9590830000016,
+ "gateMs": 1140.7013329999754,
"gateSkipped": false,
- "ttftMs": 7712.616625000002,
- "modelMs": 7109.619416000009,
- "toolMs": 0.008374999997613486,
- "totalMs": 8143.586874000008,
- "judgeMs": 984.9002080000064,
+ "ttftMs": 3493.3332919999957,
+ "modelMs": 3664.3994999999995,
+ "toolMs": 0.0006670000148005784,
+ "totalMs": 3666.8260420000297,
+ "judgeMs": 1052.9281670000055,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5094.701291000005,
+ "roundMs": 2101.538750000007,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.008374999997613486
+ "name": "trim_clip",
+ "execMs": 0.0006670000148005784
}
]
},
{
- "roundMs": 2014.9181250000038,
- "firstDeltaMs": 1580.2008750000023,
+ "roundMs": 1562.8607499999925,
+ "firstDeltaMs": 1389.563291999977,
"calls": []
}
]
@@ -3507,42 +2820,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "trim_clip",
"round": 0,
- "atMs": 6205.330916999999,
- "execMs": 0.0034999999988940544
+ "atMs": 1310.7320419999887,
+ "execMs": 0.0037089999532327056
}
],
"timings": {
- "gateMs": 1184.7039580000055,
+ "gateMs": 1044.7667909999727,
"gateSkipped": false,
- "ttftMs": 11150.112333000005,
- "modelMs": 10278.514374999999,
- "toolMs": 0.0034999999988940544,
- "totalMs": 11463.221833000003,
- "judgeMs": 1042.4619999999995,
+ "ttftMs": 2418.112124999985,
+ "modelMs": 2616.4332499999437,
+ "toolMs": 0.0037089999532327056,
+ "totalMs": 2620.10958400002,
+ "judgeMs": 737.3471249999711,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5019.801041999999,
+ "roundMs": 1308.4917499999865,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.0034999999988940544
+ "name": "trim_clip",
+ "execMs": 0.0037089999532327056
}
]
},
{
- "roundMs": 5258.713333,
- "firstDeltaMs": 4944.258041999994,
+ "roundMs": 1307.9414999999572,
+ "firstDeltaMs": 1106.2012499999837,
"calls": []
}
]
@@ -3551,33 +2864,33 @@
]
},
{
- "name": "cartoon-korean-audio-passes-language",
+ "name": "transition-ask-styled",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1172,
- "p95": 4260,
- "mean": 2120,
+ "p50": 1052,
+ "p95": 1067,
+ "mean": 1019,
"n": 3
},
"ttftMs": {
- "p50": 14917,
- "p95": 16232,
- "mean": 13130,
+ "p50": 2449,
+ "p95": 3099,
+ "mean": 2645,
"n": 3
},
"totalMs": {
- "p50": 15188,
- "p95": 16421,
- "mean": 13432,
+ "p50": 2761,
+ "p95": 3371,
+ "mean": 2918,
"n": 3
},
"modelMs": {
- "p50": 10927,
- "p95": 15248,
- "mean": 11312,
+ "p50": 2756,
+ "p95": 3367,
+ "mean": 2913,
"n": 3
},
"toolMs": {
@@ -3594,42 +2907,42 @@
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_transition",
"round": 0,
- "atMs": 12916.456167000004,
- "execMs": 0.4334579999995185
+ "atMs": 1379.428459000017,
+ "execMs": 0.07425000000512227
}
],
"timings": {
- "gateMs": 4259.978583000004,
+ "gateMs": 1067.2609999999986,
"gateSkipped": false,
- "ttftMs": 14917.243458000004,
- "modelMs": 10927.228750000002,
- "toolMs": 0.4334579999995185,
- "totalMs": 15187.640791000005,
- "judgeMs": 905.3641669999924,
+ "ttftMs": 2448.612625000009,
+ "modelMs": 2755.9288750000414,
+ "toolMs": 0.07425000000512227,
+ "totalMs": 2761.4503339999937,
+ "judgeMs": 1047.971750000026,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8653.378042000004,
+ "roundMs": 1375.3022080000374,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.4334579999995185
+ "name": "set_transition",
+ "execMs": 0.07425000000512227
}
]
},
{
- "roundMs": 2273.850707999998,
- "firstDeltaMs": 1998.5725419999944,
+ "roundMs": 1380.626667000004,
+ "firstDeltaMs": 1068.0642080000252,
"calls": []
}
]
@@ -3637,42 +2950,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_transition",
"round": 0,
- "atMs": 6591.181500000006,
- "execMs": 0.026041999997687526
+ "atMs": 1993.977917000011,
+ "execMs": 0.01049999997485429
}
],
"timings": {
- "gateMs": 1172.1658329999918,
+ "gateMs": 938.2940839999937,
"gateSkipped": false,
- "ttftMs": 16232.217417000007,
- "modelMs": 15248.496459000002,
- "toolMs": 0.026041999997687526,
- "totalMs": 16420.68833399999,
- "judgeMs": 1196.5186669999966,
+ "ttftMs": 3098.8815420000465,
+ "modelMs": 3367.135375999962,
+ "toolMs": 0.01049999997485429,
+ "totalMs": 3371.258167000022,
+ "judgeMs": 1516.0574170000036,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5416.285833999995,
+ "roundMs": 1991.8977919999743,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.026041999997687526
+ "name": "set_transition",
+ "execMs": 0.01049999997485429
}
]
},
{
- "roundMs": 9832.210625000007,
- "firstDeltaMs": 9636.693041000006,
+ "roundMs": 1375.2375839999877,
+ "firstDeltaMs": 1103.1051669999724,
"calls": []
}
]
@@ -3680,42 +2993,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_transition",
"round": 0,
- "atMs": 6088.948000000004,
- "execMs": 0.007417000000714324
+ "atMs": 1291.093125000014,
+ "execMs": 0.010458000004291534
}
],
"timings": {
- "gateMs": 926.8268339999922,
+ "gateMs": 1051.9026669999585,
"gateSkipped": false,
- "ttftMs": 8239.541916999995,
- "modelMs": 7760.191665999999,
- "toolMs": 0.007417000000714324,
- "totalMs": 8687.025916999992,
- "judgeMs": 1205.6396669999958,
+ "ttftMs": 2386.882291999995,
+ "modelMs": 2616.712375000061,
+ "toolMs": 0.010458000004291534,
+ "totalMs": 2621.8016669999924,
+ "judgeMs": 1721.3243750000256,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5159.146458000003,
+ "roundMs": 1289.1664590000291,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.007417000000714324
+ "name": "set_transition",
+ "execMs": 0.010458000004291534
}
]
},
{
- "roundMs": 2601.045207999996,
- "firstDeltaMs": 2150.0598749999917,
+ "roundMs": 1327.5459160000319,
+ "firstDeltaMs": 1093.680457999988,
"calls": []
}
]
@@ -3724,33 +3037,33 @@
]
},
{
- "name": "cartoon-style-reference-anchors-look",
+ "name": "slide-in-direction-correct",
"bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 1,
- "mean": 0,
+ "p50": 1090,
+ "p95": 1244,
+ "mean": 1094,
"n": 3
},
"ttftMs": {
- "p50": 13338,
- "p95": 15713,
- "mean": 12894,
+ "p50": 2580,
+ "p95": 2706,
+ "mean": 2572,
"n": 3
},
"totalMs": {
- "p50": 13858,
- "p95": 16221,
- "mean": 13422,
+ "p50": 2781,
+ "p95": 2837,
+ "mean": 2767,
"n": 3
},
"modelMs": {
- "p50": 13857,
- "p95": 16221,
- "mean": 13422,
+ "p50": 2772,
+ "p95": 2834,
+ "mean": 2762,
"n": 3
},
"toolMs": {
@@ -3767,42 +3080,42 @@
"runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_animation",
"round": 0,
- "atMs": 8004.275708000001,
- "execMs": 0.07395799999358132
+ "atMs": 1557.475792000012,
+ "execMs": 0.07687499996973202
}
],
"timings": {
- "gateMs": 0.6427499999990687,
- "gateSkipped": true,
- "ttftMs": 13337.51000000001,
- "modelMs": 13856.96041700001,
- "toolMs": 0.07395799999358132,
- "totalMs": 13857.677125000002,
- "judgeMs": 1393.2539590000088,
+ "gateMs": 1090.030999999959,
+ "gateSkipped": false,
+ "ttftMs": 2580.360875000013,
+ "modelMs": 2772.343874000013,
+ "toolMs": 0.07687499996973202,
+ "totalMs": 2781.289166999981,
+ "judgeMs": 970.5906669999822,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8000.024875000003,
+ "roundMs": 1550.2604580000043,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.07395799999358132
+ "name": "set_animation",
+ "execMs": 0.07687499996973202
}
]
},
{
- "roundMs": 5856.9355420000065,
- "firstDeltaMs": 5331.7124170000025,
+ "roundMs": 1222.0834160000086,
+ "firstDeltaMs": 1021.5077910000109,
"calls": []
}
]
@@ -3810,42 +3123,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_animation",
"round": 0,
- "atMs": 8398.676292000004,
- "execMs": 0.06816599999729078
+ "atMs": 1408.6141249999637,
+ "execMs": 0.007375000044703484
}
],
"timings": {
- "gateMs": 0.1965829999971902,
- "gateSkipped": true,
- "ttftMs": 15712.965750000018,
- "modelMs": 16220.727707999991,
- "toolMs": 0.06816599999729078,
- "totalMs": 16220.992456999986,
- "judgeMs": 1314.6695829999808,
+ "gateMs": 947.7063750000088,
+ "gateSkipped": false,
+ "ttftMs": 2428.2894589999923,
+ "modelMs": 2679.0456669999985,
+ "toolMs": 0.007375000044703484,
+ "totalMs": 2683.5357919999515,
+ "judgeMs": 1084.5970409999718,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8395.28525,
+ "roundMs": 1406.1441249999916,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.06816599999729078
+ "name": "set_animation",
+ "execMs": 0.007375000044703484
}
]
},
{
- "roundMs": 7825.44245799999,
- "firstDeltaMs": 7312.30074999998,
+ "roundMs": 1272.9015420000069,
+ "firstDeltaMs": 1018.1190829999978,
"calls": []
}
]
@@ -3853,42 +3166,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "generate_scene",
+ "name": "set_animation",
"round": 0,
- "atMs": 7504.355459000013,
- "execMs": 0.017332999996142462
+ "atMs": 1452.5358750000014,
+ "execMs": 0.009458000015001744
}
],
"timings": {
- "gateMs": 0.2917500000039581,
- "gateSkipped": true,
- "ttftMs": 9631.887584000011,
- "modelMs": 10187.376000999997,
- "toolMs": 0.017332999996142462,
- "totalMs": 10187.685083999997,
- "judgeMs": 1441.2837499999732,
+ "gateMs": 1243.919166000036,
+ "gateSkipped": false,
+ "ttftMs": 2705.975124999997,
+ "modelMs": 2833.9052499999525,
+ "toolMs": 0.009458000015001744,
+ "totalMs": 2837.4290829999954,
+ "judgeMs": 1071.6321250000037,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 7500.506334000005,
+ "roundMs": 1450.3486249999842,
"firstDeltaMs": null,
"calls": [
{
- "name": "generate_scene",
- "execMs": 0.017332999996142462
+ "name": "set_animation",
+ "execMs": 0.009458000015001744
}
]
},
{
- "roundMs": 2686.8696669999918,
- "firstDeltaMs": 2125.743208,
+ "roundMs": 1383.5566249999683,
+ "firstDeltaMs": 1252.307415999996,
"calls": []
}
]
@@ -3897,40 +3210,40 @@
]
},
{
- "name": "errored-call-not-claimed-done",
+ "name": "look-ask-single-tool",
"bucket": "single-tool",
- "passRate": 0.3333333333333333,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1168,
- "p95": 1168,
- "mean": 1168,
- "n": 1
+ "p50": 1063,
+ "p95": 1855,
+ "mean": 1310,
+ "n": 3
},
"ttftMs": {
- "p50": 5130,
- "p95": 5130,
- "mean": 5130,
- "n": 1
+ "p50": 3661,
+ "p95": 5936,
+ "mean": 4186,
+ "n": 3
},
"totalMs": {
- "p50": 5873,
- "p95": 5873,
- "mean": 5873,
- "n": 1
+ "p50": 4293,
+ "p95": 6529,
+ "mean": 4638,
+ "n": 3
},
"modelMs": {
- "p50": 4704,
- "p95": 4704,
- "mean": 4704,
- "n": 1
+ "p50": 3884,
+ "p95": 6522,
+ "mean": 4498,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 1
+ "n": 3
},
"rounds": {
"mean": 2,
@@ -3939,45 +3252,43 @@
},
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "simple",
"roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "reply ends with a question after completed work"
- ],
+ "notes": [],
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "add_effect",
"round": 0,
- "atMs": 5678.251958000008,
- "execMs": 2.494957999995677
+ "atMs": 4470.417917000013,
+ "execMs": 0.1610839999630116
}
],
"timings": {
- "gateMs": 1328.9027500000084,
+ "gateMs": 1010.8807919999817,
"gateSkipped": false,
- "ttftMs": 7572.251167000009,
- "modelMs": 7079.375583999994,
- "toolMs": 2.494957999995677,
- "totalMs": 8410.773291999998,
- "judgeMs": 1852.2445000000007,
+ "ttftMs": 5936.294500000018,
+ "modelMs": 6522.251667000004,
+ "toolMs": 0.1610839999630116,
+ "totalMs": 6529.198625000019,
+ "judgeMs": 947.3155830000178,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4343.580875,
+ "roundMs": 4465.052416999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 2.494957999995677
+ "name": "add_effect",
+ "execMs": 0.1610839999630116
}
]
},
{
- "roundMs": 2735.7947089999943,
- "firstDeltaMs": 1889.5414170000004,
+ "roundMs": 2057.199250000005,
+ "firstDeltaMs": 1464.3708750000224,
"calls": []
}
]
@@ -3991,82 +3302,79 @@
"judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "add_effect",
"round": 0,
- "atMs": 3897.3800829999964,
- "execMs": 0.025584000002709217
+ "atMs": 1797.435708999983,
+ "execMs": 0.01570799999171868
}
],
"timings": {
- "gateMs": 1168.4779580000031,
+ "gateMs": 1063.0829580000136,
"gateSkipped": false,
- "ttftMs": 5129.927916999994,
- "modelMs": 4704.481707999992,
- "toolMs": 0.025584000002709217,
- "totalMs": 5872.985249999998,
- "judgeMs": 1095.860541999995,
+ "ttftMs": 2961.2141249999986,
+ "modelMs": 3087.9227079999982,
+ "toolMs": 0.01570799999171868,
+ "totalMs": 3092.5492919999524,
+ "judgeMs": 925.5277090000454,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2726.7607500000013,
+ "roundMs": 1795.6965830000117,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.025584000002709217
+ "name": "add_effect",
+ "execMs": 0.01570799999171868
}
]
},
{
- "roundMs": 1977.7209579999908,
- "firstDeltaMs": 1231.3497079999943,
+ "roundMs": 1292.2261249999865,
+ "firstDeltaMs": 1161.2092909999774,
"calls": []
}
]
}
},
{
- "pass": false,
+ "pass": true,
"intent": "simple",
"roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "banned phrasing: would you like me to",
- "reply ends with a question after completed work"
- ],
- "judgeNote": "failing to execute or explaining a skip clearly, and asking a question instead of being decisive.",
+ "notes": [],
+ "judgeNote": null,
"trace": [
{
- "name": "set_transition",
+ "name": "add_effect",
"round": 0,
- "atMs": 3271.600250000003,
- "execMs": 0.007333000001381151
+ "atMs": 1855.6511249999749,
+ "execMs": 0.03008399996906519
}
],
"timings": {
- "gateMs": 1119.0382079999981,
+ "gateMs": 1855.242166000011,
"gateSkipped": false,
- "ttftMs": 4688.491416999997,
- "modelMs": 3876.2348330000095,
- "toolMs": 0.007333000001381151,
- "totalMs": 4995.280374000009,
- "judgeMs": 1919.4953749999986,
+ "ttftMs": 3660.7609999999986,
+ "modelMs": 3884.035750000039,
+ "toolMs": 0.03008399996906519,
+ "totalMs": 4293.494957999967,
+ "judgeMs": 939.6069170000264,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2150.9803329999995,
+ "roundMs": 1448.0902080000378,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_transition",
- "execMs": 0.007333000001381151
+ "name": "add_effect",
+ "execMs": 0.03008399996906519
}
]
},
{
- "roundMs": 1725.25450000001,
- "firstDeltaMs": 1416.5740000000078,
+ "roundMs": 2435.9455420000013,
+ "firstDeltaMs": 1803.3277920000255,
"calls": []
}
]
@@ -4075,97 +3383,100 @@
]
},
{
- "name": "follow-up-do-it-works",
- "bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "name": "animation-overrides-transition",
+ "bucket": "single-tool",
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 974,
- "p95": 1565,
- "mean": 1269,
- "n": 2
+ "p50": 1284,
+ "p95": 1661,
+ "mean": 1337,
+ "n": 3
},
"ttftMs": {
- "p50": 3953,
- "p95": 4774,
- "mean": 4364,
- "n": 2
+ "p50": 10267,
+ "p95": 11152,
+ "mean": 9916,
+ "n": 3
},
"totalMs": {
- "p50": 4285,
- "p95": 5099,
- "mean": 4692,
- "n": 2
+ "p50": 10588,
+ "p95": 11325,
+ "mean": 10156,
+ "n": 3
},
"modelMs": {
- "p50": 3311,
- "p95": 3534,
- "mean": 3423,
- "n": 2
+ "p50": 10581,
+ "p95": 11317,
+ "mean": 10149,
+ "n": 3
},
"toolMs": {
"p50": 0,
"p95": 0,
"mean": 0,
- "n": 2
+ "n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3,
+ "max": 3
}
},
"runs": [
{
- "pass": false,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
- "notes": [
- "reply ends with a question after completed work"
- ],
- "judgeNote": "contains a conversational question instead of being strictly factual about the editing actions.",
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
"trace": [
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4349.902083000001,
- "execMs": 0.37966599999981554
+ "name": "set_transition",
+ "round": 1,
+ "atMs": 9012.012125000008,
+ "execMs": 0.2667500000097789
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 4351.348,
- "execMs": 0.008750000000873115
+ "name": "set_animation",
+ "round": 1,
+ "atMs": 9013.07925000001,
+ "execMs": 0.010084000008646399
}
],
"timings": {
- "gateMs": 2033.2585840000002,
+ "gateMs": 1660.529749999987,
"gateSkipped": false,
- "ttftMs": 5563.394166,
- "modelMs": 3954.6733750000003,
- "toolMs": 0.38841600000068865,
- "totalMs": 5988.320375000001,
- "judgeMs": 1984.9173330000003,
+ "ttftMs": 11152.384875000047,
+ "modelMs": 11316.946124999959,
+ "toolMs": 0.2768340000184253,
+ "totalMs": 11324.970958999998,
+ "judgeMs": 1153.9718749999884,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2296.0387500000006,
+ "roundMs": 1659.7830829999875,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 7347.547041999991,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.37966599999981554
+ "name": "set_transition",
+ "execMs": 0.2667500000097789
},
{
- "name": "update_cue",
- "execMs": 0.008750000000873115
+ "name": "set_animation",
+ "execMs": 0.010084000008646399
}
]
},
{
- "roundMs": 1658.6346249999997,
- "firstDeltaMs": 1210.815125,
+ "roundMs": 2309.61599999998,
+ "firstDeltaMs": 2137.259625000006,
"calls": []
}
]
@@ -4173,72 +3484,57 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "update_cue",
- "round": 0,
- "atMs": 2772.4438339999997,
- "execMs": 0.032540999998673215
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 2772.8794589999998,
- "execMs": 0.007665999999517226
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 2774.2670419999995,
- "execMs": 0.07520800000020245
+ "name": "set_transition",
+ "round": 1,
+ "atMs": 7483.321666999953,
+ "execMs": 0.02066699997521937
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 2774.5324170000004,
- "execMs": 0.005874999998923158
+ "name": "set_animation",
+ "round": 1,
+ "atMs": 7483.8680830000085,
+ "execMs": 0.008708000008482486
}
],
"timings": {
- "gateMs": 973.5534169999992,
+ "gateMs": 1065.7632500000182,
"gateSkipped": false,
- "ttftMs": 3952.7311250000002,
- "modelMs": 3311.4413330000007,
- "toolMs": 0.12128999999731604,
- "totalMs": 4285.116039999997,
- "judgeMs": 1936.6611670000002,
+ "ttftMs": 10266.502791999956,
+ "modelMs": 10581.017125000013,
+ "toolMs": 0.029374999983701855,
+ "totalMs": 10588.246791999962,
+ "judgeMs": 940.5445000000182,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1796.5493750000005,
+ "roundMs": 1064.5001249999623,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 6414.402708000038,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.032540999998673215
- },
- {
- "name": "update_cue",
- "execMs": 0.007665999999517226
- },
- {
- "name": "delete_cue",
- "execMs": 0.07520800000020245
+ "name": "set_transition",
+ "execMs": 0.02066699997521937
},
{
- "name": "update_cue",
- "execMs": 0.005874999998923158
+ "name": "set_animation",
+ "execMs": 0.008708000008482486
}
]
},
{
- "roundMs": 1514.8919580000002,
- "firstDeltaMs": 1176.6964169999992,
+ "roundMs": 3102.114292000013,
+ "firstDeltaMs": 2780.987167000014,
"calls": []
}
]
@@ -4246,72 +3542,57 @@
},
{
"pass": true,
- "intent": "simple",
- "roundModel": "gemini-3.1-flash-lite",
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "update_cue",
- "round": 0,
- "atMs": 3660.395124999999,
- "execMs": 0.0030829999996058177
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3660.474209,
- "execMs": 0.0029999999969732016
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 3660.513750000002,
- "execMs": 0.006833999999798834
+ "name": "set_transition",
+ "round": 1,
+ "atMs": 6226.458625000028,
+ "execMs": 0.00712500000372529
},
{
- "name": "update_cue",
- "round": 0,
- "atMs": 3660.561834,
- "execMs": 0.0017920000027515925
+ "name": "set_animation",
+ "round": 1,
+ "atMs": 6226.785499999998,
+ "execMs": 0.004167000006418675
}
],
"timings": {
- "gateMs": 1565.1637499999997,
+ "gateMs": 1284.059917000006,
"gateSkipped": false,
- "ttftMs": 4774.273000000001,
- "modelMs": 3533.5945820000015,
- "toolMs": 0.014708999999129446,
- "totalMs": 5098.773041,
- "judgeMs": 1776.3302500000027,
+ "ttftMs": 8328.875416000024,
+ "modelMs": 8549.77874999994,
+ "toolMs": 0.011292000010143965,
+ "totalMs": 8555.71754100005,
+ "judgeMs": 1086.4707079999498,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2092.6877910000003,
+ "roundMs": 1282.7284169999766,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4939.356832999969,
"firstDeltaMs": null,
"calls": [
{
- "name": "update_cue",
- "execMs": 0.0030829999996058177
- },
- {
- "name": "update_cue",
- "execMs": 0.0029999999969732016
- },
- {
- "name": "delete_cue",
- "execMs": 0.006833999999798834
+ "name": "set_transition",
+ "execMs": 0.00712500000372529
},
{
- "name": "update_cue",
- "execMs": 0.0017920000027515925
+ "name": "set_animation",
+ "execMs": 0.004167000006418675
}
]
},
{
- "roundMs": 1440.9067910000012,
- "firstDeltaMs": 1113.3020830000023,
+ "roundMs": 2327.693499999994,
+ "firstDeltaMs": 2100.9543750000303,
"calls": []
}
]
@@ -4320,529 +3601,314 @@
]
},
{
- "name": "photos-request-places-clips",
+ "name": "filler-words-cut-with-editor",
"bucket": "multi-tool",
- "passRate": 0.6666666666666666,
+ "passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
+ "p50": 1061,
+ "p95": 1070,
+ "mean": 1046,
+ "n": 3
},
"ttftMs": {
- "p50": 15993,
- "p95": 119466,
- "mean": 67730,
- "n": 2
+ "p50": 53553,
+ "p95": 65238,
+ "mean": 56323,
+ "n": 3
},
"totalMs": {
- "p50": 16356,
- "p95": 119706,
- "mean": 68031,
- "n": 2
+ "p50": 53903,
+ "p95": 65509,
+ "mean": 56672,
+ "n": 3
},
"modelMs": {
- "p50": 16356,
- "p95": 119706,
- "mean": 68031,
- "n": 2
+ "p50": 53869,
+ "p95": 65475,
+ "mean": 56640,
+ "n": 3
},
"toolMs": {
- "p50": 0,
- "p95": 0,
- "mean": 0,
- "n": 2
+ "p50": 1,
+ "p95": 2,
+ "mean": 1,
+ "n": 3
},
"rounds": {
- "mean": 17.5,
- "max": 31
+ "mean": 17,
+ "max": 19
}
},
"runs": [
{
- "pass": false,
+ "pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [
- "banned phrasing: let me know if"
- ],
- "judgeNote": "conversational tone but editing tool task only partially completed.",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
"trace": [
{
- "name": "add_clip",
- "round": 0,
- "atMs": 9365.163709,
- "execMs": 0.005541999998968095
- },
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 9365.842166999999,
- "execMs": 0.0026670000006561168
- },
- {
- "name": "get_state",
+ "name": "list_skills",
"round": 1,
- "atMs": 11869.745208999997,
- "execMs": 0.08970800000679446
+ "atMs": 4498.100333000009,
+ "execMs": 0.06716600002255291
},
{
- "name": "get_state",
+ "name": "read_skill",
"round": 2,
- "atMs": 14749.093583999998,
- "execMs": 0.026583999999274965
+ "atMs": 6580.541124999989,
+ "execMs": 0.003874999994877726
},
{
- "name": "add_clip",
+ "name": "detect_silence",
"round": 3,
- "atMs": 17343.954999999998,
- "execMs": 0.004000000000814907
+ "atMs": 8704.500292000012,
+ "execMs": 0.011499999964144081
},
{
- "name": "add_clip",
+ "name": "detect_silence",
"round": 4,
- "atMs": 19506.157834,
- "execMs": 0.0015839999978197739
+ "atMs": 13210.601000000024,
+ "execMs": 0.017832999990787357
},
{
- "name": "get_state",
+ "name": "listen_audio",
"round": 5,
- "atMs": 21685.913041999996,
- "execMs": 0.003165999994962476
+ "atMs": 15508.043082999997,
+ "execMs": 0.015167000005021691
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 6,
- "atMs": 24268.873959,
- "execMs": 0.0062499999985448085
+ "atMs": 18906.72658299998,
+ "execMs": 0.5958330000285059
},
{
- "name": "get_state",
+ "name": "watch_video",
"round": 7,
- "atMs": 26143.933291999998,
- "execMs": 0.01583299999765586
+ "atMs": 21188.934458000003,
+ "execMs": 0.00849999999627471
},
{
- "name": "add_clip",
+ "name": "split_at",
"round": 8,
- "atMs": 30329.573958999998,
- "execMs": 0.005417000000306871
+ "atMs": 30418.627625000023,
+ "execMs": 0.6585830000112765
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 9,
- "atMs": 32705.758208999996,
- "execMs": 0.027792000000772532
+ "atMs": 35752.72479200002,
+ "execMs": 0.08879100001649931
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 10,
- "atMs": 35970.680834,
- "execMs": 0.01916699999856064
+ "atMs": 38183.098624999984,
+ "execMs": 0.13295800000196323
},
{
- "name": "add_clip",
+ "name": "refine_speech_cuts",
"round": 11,
- "atMs": 38458.996084,
- "execMs": 0.007041000004392117
+ "atMs": 40802.824291999976,
+ "execMs": 0.0783750000409782
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 12,
- "atMs": 41768.666417,
- "execMs": 0.00516599999536993
+ "atMs": 43427.77137500001,
+ "execMs": 0.06870799994794652
},
{
- "name": "get_state",
+ "name": "delete_cue",
"round": 13,
- "atMs": 44489.186,
- "execMs": 0.01583299999765586
+ "atMs": 46768.41295799997,
+ "execMs": 0.021166999998968095
},
{
- "name": "add_clip",
+ "name": "update_cue",
"round": 14,
- "atMs": 48395.494167,
- "execMs": 0.004249999998137355
+ "atMs": 48999.416333,
+ "execMs": 0.009916999959386885
},
{
- "name": "get_state",
+ "name": "update_cue",
"round": 15,
- "atMs": 50938.076084,
- "execMs": 0.01245799999742303
- },
- {
- "name": "get_state",
- "round": 16,
- "atMs": 53156.87449999999,
- "execMs": 0.005374999993364327
- },
- {
- "name": "get_state",
- "round": 17,
- "atMs": 56771.60133399999,
- "execMs": 0.008916999999200925
- },
- {
- "name": "add_clip",
- "round": 18,
- "atMs": 60906.756459,
- "execMs": 0.0048749999987194315
- },
- {
- "name": "get_state",
- "round": 19,
- "atMs": 67288.815209,
- "execMs": 0.014083000001846813
- },
- {
- "name": "add_clip",
- "round": 20,
- "atMs": 69495.08629199999,
- "execMs": 0.0017920000100275502
- },
- {
- "name": "get_state",
- "round": 21,
- "atMs": 78613.232209,
- "execMs": 0.017041999992216006
- },
- {
- "name": "get_state",
- "round": 22,
- "atMs": 94290.09375,
- "execMs": 0.012583000003360212
- },
- {
- "name": "get_state",
- "round": 23,
- "atMs": 107840.301417,
- "execMs": 0.009541999999783002
- },
- {
- "name": "add_clip",
- "round": 24,
- "atMs": 116847.97025,
- "execMs": 0.008792000007815659
- },
- {
- "name": "get_state",
- "round": 25,
- "atMs": 125978.60858399999,
- "execMs": 0.01345900000887923
- },
- {
- "name": "add_clip",
- "round": 26,
- "atMs": 138560.886084,
- "execMs": 0.004833999992115423
- },
- {
- "name": "add_clip",
- "round": 27,
- "atMs": 140968.45450000002,
- "execMs": 0.007499999977881089
+ "atMs": 51278.89516700001,
+ "execMs": 0.008333000005222857
}
],
"timings": {
- "gateMs": 1.1879169999992882,
- "gateSkipped": true,
- "ttftMs": 144245.64929200002,
- "modelMs": 144549.141544,
- "toolMs": 0.36125199997331947,
- "totalMs": 144550.690713,
- "judgeMs": 1231.1373749999912,
+ "gateMs": 1006.983917000005,
+ "gateSkipped": false,
+ "ttftMs": 53553.38274999999,
+ "modelMs": 53869.068457000016,
+ "toolMs": 1.786705999984406,
+ "totalMs": 53903.30433299998,
+ "judgeMs": 1057.4297919999808,
"retryMs": 0,
"retries": 0,
- "rounds": [
- {
- "roundMs": 9354.848915999999,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.005541999998968095
- },
- {
- "name": "add_clip",
- "execMs": 0.0026670000006561168
- }
- ]
- },
- {
- "roundMs": 2500.9542089999995,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.08970800000679446
- }
- ]
- },
- {
- "roundMs": 2877.726499999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.026583999999274965
- }
- ]
- },
- {
- "roundMs": 2593.068625,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.004000000000814907
- }
- ]
- },
- {
- "roundMs": 2160.6204999999973,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0015839999978197739
- }
- ]
- },
- {
- "roundMs": 2179.226834000001,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.003165999994962476
- }
- ]
- },
- {
- "roundMs": 2582.1612499999974,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.0062499999985448085
- }
- ]
- },
- {
- "roundMs": 1873.1346660000054,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.01583299999765586
- }
- ]
- },
- {
- "roundMs": 4183.652332999998,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.005417000000306871
- }
- ]
- },
- {
- "roundMs": 2373.120541999997,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.027792000000772532
- }
- ]
- },
- {
- "roundMs": 3261.6688749999958,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.01916699999856064
- }
- ]
- },
- {
- "roundMs": 2485.8957500000033,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.007041000004392117
- }
- ]
- },
+ "rounds": [
{
- "roundMs": 3307.974583000003,
+ "roundMs": 1005.463665999996,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "add_clip",
- "execMs": 0.00516599999536993
- }
- ]
+ "calls": []
},
{
- "roundMs": 2718.184208000006,
+ "roundMs": 3487.1161250000005,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.01583299999765586
+ "name": "list_skills",
+ "execMs": 0.06716600002255291
}
]
},
{
- "roundMs": 3903.3641669999924,
+ "roundMs": 2080.839500000002,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.004249999998137355
+ "name": "read_skill",
+ "execMs": 0.003874999994877726
}
]
},
{
- "roundMs": 2539.9696669999976,
+ "roundMs": 2122.9190410000156,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.01245799999742303
+ "name": "detect_silence",
+ "execMs": 0.011499999964144081
}
]
},
{
- "roundMs": 2216.2216670000053,
+ "roundMs": 4505.259791999997,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.005374999993364327
+ "name": "detect_silence",
+ "execMs": 0.017832999990787357
}
]
},
{
- "roundMs": 3613.9260000000068,
+ "roundMs": 2294.869000000006,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.008916999999200925
+ "name": "listen_audio",
+ "execMs": 0.015167000005021691
}
]
},
{
- "roundMs": 4131.619416999994,
+ "roundMs": 3396.9686670000083,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0048749999987194315
+ "name": "get_state",
+ "execMs": 0.5958330000285059
}
]
},
{
- "roundMs": 6379.290792,
+ "roundMs": 2278.8691249999683,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.014083000001846813
+ "name": "watch_video",
+ "execMs": 0.00849999999627471
}
]
},
{
- "roundMs": 2203.8260840000003,
+ "roundMs": 9226.810332999972,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0017920000100275502
+ "name": "split_at",
+ "execMs": 0.6585830000112765
}
]
},
{
- "roundMs": 9116.870875000008,
+ "roundMs": 5330.670750000048,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.017041999992216006
+ "name": "split_at",
+ "execMs": 0.08879100001649931
}
]
},
{
- "roundMs": 15674.640417000002,
+ "roundMs": 2428.1704579999787,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.012583000003360212
+ "name": "delete_item",
+ "execMs": 0.13295800000196323
}
]
},
{
- "roundMs": 13546.471165999988,
+ "roundMs": 2617.4469590000226,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.009541999999783002
+ "name": "refine_speech_cuts",
+ "execMs": 0.0783750000409782
}
]
},
{
- "roundMs": 9004.178334000011,
+ "roundMs": 2623.427250000008,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.008792000007815659
+ "name": "get_state",
+ "execMs": 0.06870799994794652
}
]
},
{
- "roundMs": 9126.41066600001,
+ "roundMs": 3338.361499999999,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.01345900000887923
+ "name": "delete_cue",
+ "execMs": 0.021166999998968095
}
]
},
{
- "roundMs": 12579.425334,
+ "roundMs": 2229.974125000008,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.004833999992115423
+ "name": "update_cue",
+ "execMs": 0.009916999959386885
}
]
},
{
- "roundMs": 2403.5741670000134,
+ "roundMs": 2278.8755409999867,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.007499999977881089
+ "name": "update_cue",
+ "execMs": 0.008333000005222857
}
]
},
{
- "roundMs": 3657.1149999999907,
- "firstDeltaMs": 3274.4027500000084,
+ "roundMs": 2623.0266249999986,
+ "firstDeltaMs": 2273.851040999987,
"calls": []
}
]
@@ -4851,620 +3917,715 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "add_clip",
- "round": 0,
- "atMs": 6347.024291000009,
- "execMs": 0.010749999986728653
- },
- {
- "name": "add_clip",
- "round": 0,
- "atMs": 6347.542916000006,
- "execMs": 0.003542000020388514
- },
- {
- "name": "get_state",
+ "name": "read_skill",
"round": 1,
- "atMs": 8630.411083000014,
- "execMs": 0.02733400001307018
+ "atMs": 4444.065416999976,
+ "execMs": 0.05087500001536682
},
{
- "name": "add_clip",
+ "name": "detect_silence",
"round": 2,
- "atMs": 12486.632083000004,
- "execMs": 0.00812499999301508
+ "atMs": 7394.844541999977,
+ "execMs": 0.03141699999105185
},
{
- "name": "add_clip",
+ "name": "listen_audio",
"round": 3,
- "atMs": 14495.920332999987,
- "execMs": 0.0072499999951105565
+ "atMs": 9648.370166999986,
+ "execMs": 0.009833999967668205
},
{
"name": "get_state",
"round": 4,
- "atMs": 16405.490208000003,
- "execMs": 0.011207999981706962
+ "atMs": 13734.405291999981,
+ "execMs": 0.11616600002162158
},
{
- "name": "add_clip",
+ "name": "detect_silence",
"round": 5,
- "atMs": 19207.112708,
- "execMs": 0.021417000010842457
+ "atMs": 16470.391999999993,
+ "execMs": 0.027000000001862645
},
{
- "name": "get_state",
+ "name": "split_at",
"round": 6,
- "atMs": 20904.51225,
- "execMs": 0.020541999983834103
+ "atMs": 27509.322291999997,
+ "execMs": 0.18245800002478063
},
{
- "name": "add_clip",
+ "name": "split_at",
"round": 7,
- "atMs": 23616.670625,
- "execMs": 0.018041000002995133
+ "atMs": 29699.89966699999,
+ "execMs": 0.028833999997004867
},
{
- "name": "get_state",
+ "name": "delete_item",
"round": 8,
- "atMs": 26455.49966599999,
- "execMs": 0.011291999981040135
+ "atMs": 32497.72758299997,
+ "execMs": 0.17475000000558794
},
{
- "name": "add_clip",
+ "name": "get_state",
"round": 9,
- "atMs": 29737.228457999998,
- "execMs": 0.005541999998968095
+ "atMs": 34591.16470799997,
+ "execMs": 0.0491660000407137
},
{
- "name": "add_clip",
+ "name": "delete_cue",
"round": 10,
- "atMs": 32356.517083000013,
- "execMs": 0.003167000017128885
+ "atMs": 37737.48195799999,
+ "execMs": 0.06154099997365847
},
{
- "name": "get_state",
+ "name": "update_cue",
"round": 11,
- "atMs": 34265.730583,
- "execMs": 0.009916999988490716
+ "atMs": 43047.10120799998,
+ "execMs": 0.02549999998882413
},
{
- "name": "add_clip",
+ "name": "update_cue",
"round": 12,
- "atMs": 36945.80374999999,
- "execMs": 0.004125000006752089
+ "atMs": 45603.681667,
+ "execMs": 0.053124999976716936
},
{
"name": "get_state",
"round": 13,
- "atMs": 39213.31774999999,
- "execMs": 0.006250000005820766
- },
- {
- "name": "add_clip",
- "round": 14,
- "atMs": 42105.20529099999,
- "execMs": 0.0029999999969732016
- },
- {
- "name": "get_state",
- "round": 15,
- "atMs": 47365.30050000001,
- "execMs": 0.012291999999433756
- },
- {
- "name": "add_clip",
- "round": 16,
- "atMs": 50709.069208,
- "execMs": 0.0027089999930467457
- },
- {
- "name": "add_clip",
- "round": 17,
- "atMs": 53436.82837500001,
- "execMs": 0.004917000012937933
- },
- {
- "name": "get_state",
- "round": 18,
- "atMs": 56165.44862499999,
- "execMs": 0.013333999988390133
- },
- {
- "name": "add_clip",
- "round": 19,
- "atMs": 61446.066374999995,
- "execMs": 0.004041000007418916
- },
- {
- "name": "add_clip",
- "round": 20,
- "atMs": 63683.539166,
- "execMs": 0.0020409999997355044
- },
- {
- "name": "get_state",
- "round": 21,
- "atMs": 65870.680208,
- "execMs": 0.0034580000210553408
- },
- {
- "name": "add_clip",
- "round": 22,
- "atMs": 69925.819583,
- "execMs": 0.0030830000177957118
- },
- {
- "name": "add_clip",
- "round": 22,
- "atMs": 69926.04358299999,
- "execMs": 0.0014999999839346856
- },
- {
- "name": "get_state",
- "round": 23,
- "atMs": 76665.128333,
- "execMs": 0.010332999983802438
- },
- {
- "name": "add_clip",
- "round": 24,
- "atMs": 84464.85920800001,
- "execMs": 0.002542000001994893
- },
- {
- "name": "get_state",
- "round": 25,
- "atMs": 86533.62325,
- "execMs": 0.005332999979145825
- },
- {
- "name": "add_clip",
- "round": 26,
- "atMs": 97622.57716599997,
- "execMs": 0.0007919999770820141
- },
- {
- "name": "get_state",
- "round": 27,
- "atMs": 99943.1445,
- "execMs": 0.004458000010345131
- },
- {
- "name": "get_state",
- "round": 28,
- "atMs": 104005.40270799998,
- "execMs": 0.012790999957360327
- },
- {
- "name": "set_view",
- "round": 29,
- "atMs": 114179.36791600002,
- "execMs": 0.01616699999431148
+ "atMs": 48000.706957999966,
+ "execMs": 0.0360829999553971
}
],
"timings": {
- "gateMs": 0.36574999999720603,
- "gateSkipped": true,
- "ttftMs": 119466.47324999998,
- "modelMs": 119705.54787300009,
- "toolMs": 0.27129299991065636,
- "totalMs": 119706.184916,
- "judgeMs": 1471.4382079999777,
+ "gateMs": 1061.0017920000246,
+ "gateSkipped": false,
+ "ttftMs": 50177.316207999946,
+ "modelMs": 50575.90291500016,
+ "toolMs": 0.8467489999602549,
+ "totalMs": 50602.51908299996,
+ "judgeMs": 1330.2250830000266,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6343.533332999999,
+ "roundMs": 1058.8056250000373,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3378.861042000004,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.010749999986728653
- },
- {
- "name": "add_clip",
- "execMs": 0.003542000020388514
+ "name": "read_skill",
+ "execMs": 0.05087500001536682
}
]
},
{
- "roundMs": 2280.138875000004,
+ "roundMs": 2949.0402079999913,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.02733400001307018
+ "name": "detect_silence",
+ "execMs": 0.03141699999105185
}
]
},
{
- "roundMs": 3854.272666999983,
+ "roundMs": 2252.1165829999954,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.00812499999301508
+ "name": "listen_audio",
+ "execMs": 0.009833999967668205
}
]
},
{
- "roundMs": 2006.8395000000019,
+ "roundMs": 4085.306332999957,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0072499999951105565
+ "name": "get_state",
+ "execMs": 0.11616600002162158
}
]
},
{
- "roundMs": 1907.2319580000185,
+ "roundMs": 2734.2989170000073,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.011207999981706962
+ "name": "detect_silence",
+ "execMs": 0.027000000001862645
}
]
},
{
- "roundMs": 2799.3657500000263,
+ "roundMs": 11037.327625000034,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.021417000010842457
+ "name": "split_at",
+ "execMs": 0.18245800002478063
}
]
},
{
- "roundMs": 1695.0663749999949,
+ "roundMs": 2189.2580830000225,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.020541999983834103
+ "name": "split_at",
+ "execMs": 0.028833999997004867
}
]
},
{
- "roundMs": 2710.883459000004,
+ "roundMs": 2796.473667000013,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.018041000002995133
+ "name": "delete_item",
+ "execMs": 0.17475000000558794
}
]
},
{
- "roundMs": 2836.3477910000074,
+ "roundMs": 2091.561250000028,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.011291999981040135
+ "execMs": 0.0491660000407137
}
]
},
{
- "roundMs": 3278.8874589999905,
+ "roundMs": 3145.4002080000355,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.005541999998968095
+ "name": "delete_cue",
+ "execMs": 0.06154099997365847
}
]
},
{
- "roundMs": 2617.1875839999993,
+ "roundMs": 5307.8068329999805,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.003167000017128885
+ "name": "update_cue",
+ "execMs": 0.02549999998882413
}
]
},
{
- "roundMs": 1907.0659159999923,
+ "roundMs": 2554.7003330000443,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.009916999988490716
+ "name": "update_cue",
+ "execMs": 0.053124999976716936
}
]
},
{
- "roundMs": 2677.630165999988,
+ "roundMs": 2395.0533750000177,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.004125000006752089
+ "name": "get_state",
+ "execMs": 0.0360829999553971
}
]
},
{
- "roundMs": 2265.273917000013,
+ "roundMs": 2599.892832999991,
+ "firstDeltaMs": 2175.588999999978,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "list_skills",
+ "round": 1,
+ "atMs": 4246.829833999975,
+ "execMs": 0.09020899992901832
+ },
+ {
+ "name": "read_skill",
+ "round": 2,
+ "atMs": 6334.007958999951,
+ "execMs": 0.005291000008583069
+ },
+ {
+ "name": "read_skill",
+ "round": 3,
+ "atMs": 8476.747208999936,
+ "execMs": 0.015750000020489097
+ },
+ {
+ "name": "detect_silence",
+ "round": 4,
+ "atMs": 11867.053375000018,
+ "execMs": 0.04704099998343736
+ },
+ {
+ "name": "watch_video",
+ "round": 5,
+ "atMs": 14466.433709000004,
+ "execMs": 0.02779199997894466
+ },
+ {
+ "name": "listen_audio",
+ "round": 6,
+ "atMs": 17785.82033399993,
+ "execMs": 0.016125000081956387
+ },
+ {
+ "name": "get_state",
+ "round": 7,
+ "atMs": 19793.918916999944,
+ "execMs": 0.16170799999963492
+ },
+ {
+ "name": "detect_silence",
+ "round": 8,
+ "atMs": 26398.46345899999,
+ "execMs": 0.019040999934077263
+ },
+ {
+ "name": "split_at",
+ "round": 9,
+ "atMs": 37913.40287500003,
+ "execMs": 0.23733299993909895
+ },
+ {
+ "name": "split_at",
+ "round": 10,
+ "atMs": 40390.835542000015,
+ "execMs": 0.1254590000025928
+ },
+ {
+ "name": "delete_item",
+ "round": 11,
+ "atMs": 44382.01787500002,
+ "execMs": 0.1422499999171123
+ },
+ {
+ "name": "get_state",
+ "round": 12,
+ "atMs": 46432.170334000024,
+ "execMs": 0.029749999986961484
+ },
+ {
+ "name": "delete_cue",
+ "round": 13,
+ "atMs": 53744.85808399995,
+ "execMs": 0.0625
+ },
+ {
+ "name": "update_cue",
+ "round": 14,
+ "atMs": 56224.95887500001,
+ "execMs": 0.07187500002328306
+ },
+ {
+ "name": "update_cue",
+ "round": 15,
+ "atMs": 58541.19062499993,
+ "execMs": 0.017250000033527613
+ },
+ {
+ "name": "update_cue",
+ "round": 16,
+ "atMs": 60848.77566699998,
+ "execMs": 0.02083399996627122
+ },
+ {
+ "name": "refine_speech_cuts",
+ "round": 17,
+ "atMs": 63129.769458999974,
+ "execMs": 0.028833999997004867
+ }
+ ],
+ "timings": {
+ "gateMs": 1070.3639589999802,
+ "gateSkipped": false,
+ "ttftMs": 65238.285959,
+ "modelMs": 65474.505917,
+ "toolMs": 1.1190419998019934,
+ "totalMs": 65509.375459,
+ "judgeMs": 1131.8673750000307,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1068.6005839999998,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3173.6268329999875,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.006250000005820766
+ "name": "list_skills",
+ "execMs": 0.09020899992901832
}
]
},
{
- "roundMs": 2890.6446660000074,
+ "roundMs": 2085.2153330000583,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0029999999969732016
+ "name": "read_skill",
+ "execMs": 0.005291000008583069
}
]
},
{
- "roundMs": 5257.402124999993,
+ "roundMs": 2141.980082999915,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.012291999999433756
+ "name": "read_skill",
+ "execMs": 0.015750000020489097
}
]
},
{
- "roundMs": 3338.358790999977,
+ "roundMs": 3388.0405830000527,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0027089999930467457
+ "name": "detect_silence",
+ "execMs": 0.04704099998343736
}
]
},
{
- "roundMs": 2724.468332999997,
+ "roundMs": 2597.6013340000063,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.004917000012937933
+ "name": "watch_video",
+ "execMs": 0.02779199997894466
}
]
},
{
- "roundMs": 2726.3109580000164,
+ "roundMs": 3317.484000000055,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.013333999988390133
+ "name": "listen_audio",
+ "execMs": 0.016125000081956387
}
]
},
{
- "roundMs": 5278.743542000011,
+ "roundMs": 2006.9830410000868,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.004041000007418916
+ "name": "get_state",
+ "execMs": 0.16170799999963492
}
]
},
{
- "roundMs": 2235.5099170000176,
+ "roundMs": 6602.819708000054,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0020409999997355044
+ "name": "detect_silence",
+ "execMs": 0.019040999934077263
}
]
},
{
- "roundMs": 2186.215082999988,
+ "roundMs": 11512.806333999964,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.0034580000210553408
+ "name": "split_at",
+ "execMs": 0.23733299993909895
}
]
},
{
- "roundMs": 4054.366416999983,
+ "roundMs": 2474.452667000005,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0030830000177957118
- },
- {
- "name": "add_clip",
- "execMs": 0.0014999999839346856
+ "name": "split_at",
+ "execMs": 0.1254590000025928
}
]
},
{
- "roundMs": 6737.167083000008,
+ "roundMs": 3988.8979579999577,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.010332999983802438
+ "name": "delete_item",
+ "execMs": 0.1422499999171123
}
]
},
{
- "roundMs": 7797.56700000001,
+ "roundMs": 2047.7404579999857,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.002542000001994893
+ "name": "get_state",
+ "execMs": 0.029749999986961484
}
]
},
{
- "roundMs": 2067.4492079999764,
+ "roundMs": 7311.247417000006,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.005332999979145825
+ "name": "delete_cue",
+ "execMs": 0.0625
}
]
},
{
- "roundMs": 11088.198125000024,
+ "roundMs": 2477.83670799993,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.0007919999770820141
+ "name": "update_cue",
+ "execMs": 0.07187500002328306
}
]
},
{
- "roundMs": 2319.9196249999804,
+ "roundMs": 2313.7306660000468,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.004458000010345131
+ "name": "update_cue",
+ "execMs": 0.017250000033527613
}
]
},
{
- "roundMs": 4061.178292000026,
+ "roundMs": 2306.797333999886,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.012790999957360327
+ "name": "update_cue",
+ "execMs": 0.02083399996627122
}
]
},
{
- "roundMs": 10168.111583000049,
+ "roundMs": 2279.97341700003,
"firstDeltaMs": null,
"calls": [
{
- "name": "set_view",
- "execMs": 0.01616699999431148
+ "name": "refine_speech_cuts",
+ "execMs": 0.028833999997004867
}
]
},
{
- "roundMs": 5594.212375000003,
- "firstDeltaMs": 5285.640000000014,
+ "roundMs": 2378.6714589999756,
+ "firstDeltaMs": 2107.8910840000026,
"calls": []
}
]
}
+ }
+ ]
+ },
+ {
+ "name": "edit-request-still-uses-tools",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1021,
+ "p95": 1053,
+ "mean": 1014,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 3256,
+ "p95": 6771,
+ "mean": 4334,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 4059,
+ "p95": 6999,
+ "mean": 4885,
+ "n": 3
},
+ "modelMs": {
+ "p50": 4055,
+ "p95": 6992,
+ "mean": 4880,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": "mentions a beach video not in the prompt, creating confusion about the scope of the edit.",
+ "judgeNote": null,
"trace": [
{
- "name": "add_clip",
- "round": 0,
- "atMs": 6414.751958000008,
- "execMs": 0.006333000026643276
- },
- {
- "name": "add_clip",
+ "name": "subtitles_generate",
"round": 0,
- "atMs": 6415.403292000003,
- "execMs": 0.012875000014901161
- },
- {
- "name": "get_state",
- "round": 1,
- "atMs": 9707.97445800004,
- "execMs": 0.04666599998017773
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 14027.780625000014,
- "execMs": 0.003375000029336661
- },
- {
- "name": "add_clip",
- "round": 2,
- "atMs": 14027.976375000027,
- "execMs": 0.00041699997382238507
+ "atMs": 5359.881500000018,
+ "execMs": 0.013957999995909631
}
],
"timings": {
- "gateMs": 0.2606670000241138,
- "gateSkipped": true,
- "ttftMs": 15992.797333000053,
- "modelMs": 16356.092165999988,
- "toolMs": 0.06966600002488121,
- "totalMs": 16356.422499000037,
- "judgeMs": 1972.9069160000072,
+ "gateMs": 1052.841625000001,
+ "gateSkipped": false,
+ "ttftMs": 6770.96875,
+ "modelMs": 6991.631916000042,
+ "toolMs": 0.013957999995909631,
+ "totalMs": 6998.731334000011,
+ "judgeMs": 1078.5364590000827,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6411.362291000027,
+ "roundMs": 5354.25225000002,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.006333000026643276
- },
- {
- "name": "add_clip",
- "execMs": 0.012875000014901161
+ "name": "subtitles_generate",
+ "execMs": 0.013957999995909631
}
]
},
{
- "roundMs": 3290.572999999975,
+ "roundMs": 1637.3796660000226,
+ "firstDeltaMs": 1410.0280829999829,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": "did not state what changed in concrete terms (e.g., style, placement, or language) and ignored the approval gate for a paid task.",
+ "trace": [
+ {
+ "name": "subtitles_generate",
+ "round": 0,
+ "atMs": 1798.107874999987,
+ "execMs": 0.029666999937035143
+ }
+ ],
+ "timings": {
+ "gateMs": 1021.3043750000652,
+ "gateSkipped": false,
+ "ttftMs": 3255.7909590000054,
+ "modelMs": 4055.1263330001384,
+ "toolMs": 0.029666999937035143,
+ "totalMs": 4059.0133749999804,
+ "judgeMs": 1066.701207999955,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1795.7172500001034,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.04666599998017773
+ "name": "subtitles_generate",
+ "execMs": 0.029666999937035143
}
]
},
{
- "roundMs": 4317.821999999986,
+ "roundMs": 2259.409083000035,
+ "firstDeltaMs": 1456.2615830000723,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "subtitles_generate",
+ "round": 0,
+ "atMs": 1513.4303340000333,
+ "execMs": 0.004458000068552792
+ }
+ ],
+ "timings": {
+ "gateMs": 967.498249999946,
+ "gateSkipped": false,
+ "ttftMs": 2976.6096670000115,
+ "modelMs": 3592.335834000027,
+ "toolMs": 0.004458000068552792,
+ "totalMs": 3597.6535419999855,
+ "judgeMs": 1646.90295899997,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1510.32491700002,
"firstDeltaMs": null,
"calls": [
{
- "name": "add_clip",
- "execMs": 0.003375000029336661
- },
- {
- "name": "add_clip",
- "execMs": 0.00041699997382238507
+ "name": "subtitles_generate",
+ "execMs": 0.004458000068552792
}
]
},
{
- "roundMs": 2336.3348750000005,
- "firstDeltaMs": 1963.407499999972,
+ "roundMs": 2082.010917000007,
+ "firstDeltaMs": 1461.0561669999734,
"calls": []
}
]
@@ -5473,275 +4634,186 @@
]
},
{
- "name": "filler-words-cut-with-editor",
- "bucket": "multi-tool",
+ "name": "cartoon-from-audio-plans-scene",
+ "bucket": "single-tool",
"passRate": 1,
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1060,
- "p95": 1258,
- "mean": 1106,
+ "p50": 1657,
+ "p95": 1683,
+ "mean": 1483,
"n": 3
},
"ttftMs": {
- "p50": 66488,
- "p95": 75411,
- "mean": 64297,
+ "p50": 16261,
+ "p95": 18283,
+ "mean": 15839,
"n": 3
},
"totalMs": {
- "p50": 67010,
- "p95": 75777,
- "mean": 64714,
+ "p50": 16492,
+ "p95": 18673,
+ "mean": 16164,
"n": 3
},
"modelMs": {
- "p50": 66010,
- "p95": 74516,
- "mean": 63607,
+ "p50": 16482,
+ "p95": 18660,
+ "mean": 16154,
"n": 3
},
"toolMs": {
- "p50": 1,
- "p95": 3,
- "mean": 1,
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
"n": 3
},
"rounds": {
- "mean": 13,
- "max": 14
+ "mean": 5,
+ "max": 6
}
},
"runs": [
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 5580.8372500000405,
- "execMs": 0.8649579999619164
- },
{
"name": "read_skill",
"round": 1,
- "atMs": 7816.184083,
- "execMs": 0.0051250000251457095
+ "atMs": 5680.2327499999665,
+ "execMs": 0.21808300004340708
},
{
- "name": "detect_silence",
+ "name": "listen_audio",
"round": 2,
- "atMs": 10350.719125000003,
- "execMs": 0.02345799998147413
+ "atMs": 9768.099667000002,
+ "execMs": 0.03637499990873039
},
{
- "name": "listen_audio",
+ "name": "get_state",
"round": 3,
- "atMs": 14999.313291000028,
- "execMs": 0.018333999963942915
+ "atMs": 12000.085707999999,
+ "execMs": 0.007291999994777143
},
{
- "name": "detect_silence",
+ "name": "generate_scene",
"round": 4,
- "atMs": 19877.292790999985,
- "execMs": 0.020415999984834343
- },
- {
- "name": "watch_video",
- "round": 5,
- "atMs": 22609.705083000008,
- "execMs": 0.018708000017795712
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 31923.090833000024,
- "execMs": 0.6783330000471324
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 31924.24866600003,
- "execMs": 0.04891700000735
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 31924.490583000006,
- "execMs": 0.01829199999338016
- },
- {
- "name": "delete_item",
- "round": 7,
- "atMs": 44025.13024999999,
- "execMs": 0.2165839999797754
- },
- {
- "name": "delete_item",
- "round": 7,
- "atMs": 44025.69500000001,
- "execMs": 0.07304200000362471
- },
- {
- "name": "refine_speech_cuts",
- "round": 8,
- "atMs": 47971.268041,
- "execMs": 0.05191699997521937
- },
- {
- "name": "get_state",
- "round": 9,
- "atMs": 63652.130041000026,
- "execMs": 0.496458999987226
- },
- {
- "name": "subtitles_generate",
- "round": 10,
- "atMs": 71592.575541,
- "execMs": 0.051334000017959625
+ "atMs": 14900.848832999938,
+ "execMs": 0.19137499993667006
}
],
"timings": {
- "gateMs": 1258.1774580000201,
+ "gateMs": 1682.6600409999955,
"gateSkipped": false,
- "ttftMs": 75410.774875,
- "modelMs": 74516.291042,
- "toolMs": 2.5858769999467768,
- "totalMs": 75777.05437699996,
- "judgeMs": 1622.744458000001,
+ "ttftMs": 18282.558332999935,
+ "modelMs": 18660.183126000105,
+ "toolMs": 0.4531249998835847,
+ "totalMs": 18672.514166999958,
+ "judgeMs": 1354.4317499999888,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4318.188334000006,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.8649579999619164
- }
- ]
- },
- {
- "roundMs": 2232.2167079999927,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "read_skill",
- "execMs": 0.0051250000251457095
- }
- ]
- },
- {
- "roundMs": 2530.8165000000154,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.02345799998147413
- }
- ]
- },
- {
- "roundMs": 4645.790042000008,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.018333999963942915
- }
- ]
- },
- {
- "roundMs": 4876.580292000028,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.020415999984834343
- }
- ]
- },
- {
- "roundMs": 2728.2512080000015,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.018708000017795712
- }
- ]
- },
- {
- "roundMs": 9310.074415999989,
+ "roundMs": 1681.970083000022,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.6783330000471324
- },
- {
- "name": "split_at",
- "execMs": 0.04891700000735
- },
+ "calls": []
+ },
+ {
+ "roundMs": 3995.4948330000043,
+ "firstDeltaMs": null,
+ "calls": [
{
- "name": "split_at",
- "execMs": 0.01829199999338016
+ "name": "read_skill",
+ "execMs": 0.21808300004340708
}
]
},
{
- "roundMs": 12095.991749999986,
+ "roundMs": 4085.7516250000335,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.2165839999797754
- },
- {
- "name": "delete_item",
- "execMs": 0.07304200000362471
+ "name": "listen_audio",
+ "execMs": 0.03637499990873039
}
]
},
{
- "roundMs": 3942.437208999996,
+ "roundMs": 2230.5004590000026,
"firstDeltaMs": null,
"calls": [
{
- "name": "refine_speech_cuts",
- "execMs": 0.05191699997521937
+ "name": "get_state",
+ "execMs": 0.007291999994777143
}
]
},
{
- "roundMs": 15679.742957999988,
+ "roundMs": 2897.1106669999426,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.496458999987226
+ "name": "generate_scene",
+ "execMs": 0.19137499993667006
}
]
},
{
- "roundMs": 7934.565833000001,
+ "roundMs": 3769.3554590001004,
+ "firstDeltaMs": 3379.739459000062,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 9186.846166999894,
+ "execMs": 0.019582999986596406
+ }
+ ],
+ "timings": {
+ "gateMs": 1656.5625419999706,
+ "gateSkipped": false,
+ "ttftMs": 12974.318417000002,
+ "modelMs": 13319.911041999934,
+ "toolMs": 0.019582999986596406,
+ "totalMs": 13326.079707999947,
+ "judgeMs": 1114.840833999915,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1655.3011249999981,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 7527.293166999938,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.051334000017959625
+ "name": "generate_scene",
+ "execMs": 0.019582999986596406
}
]
},
{
- "roundMs": 4221.635791999986,
- "firstDeltaMs": 3816.1922919999924,
+ "roundMs": 4137.316749999998,
+ "firstDeltaMs": 3785.8773749999236,
"calls": []
}
]
@@ -5750,233 +4822,186 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 6926.977458999958,
- "execMs": 0.07491700001992285
- },
{
"name": "read_skill",
"round": 1,
- "atMs": 12566.700124999974,
- "execMs": 0.020542000012937933
+ "atMs": 6067.227124999976,
+ "execMs": 0.01508399995509535
},
{
- "name": "detect_silence",
+ "name": "listen_audio",
"round": 2,
- "atMs": 15696.67966699996,
- "execMs": 0.03437500004656613
+ "atMs": 8535.476124999928,
+ "execMs": 0.00779200007673353
},
{
- "name": "listen_audio",
+ "name": "get_state",
"round": 3,
- "atMs": 22775.795291999995,
- "execMs": 0.020040999981574714
+ "atMs": 11179.066999999923,
+ "execMs": 0.016416999977082014
},
{
- "name": "watch_video",
+ "name": "generate_scene",
"round": 4,
- "atMs": 26057.220042,
- "execMs": 0.02374999999301508
- },
- {
- "name": "split_at",
- "round": 5,
- "atMs": 37866.295124999946,
- "execMs": 0.14816699997754768
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 42695.55108399998,
- "execMs": 0.04079200001433492
- },
- {
- "name": "split_at",
- "round": 7,
- "atMs": 44695.987166999956,
- "execMs": 0.04829100001370534
- },
- {
- "name": "delete_item",
- "round": 8,
- "atMs": 47865.65625,
- "execMs": 0.08283400000073016
- },
- {
- "name": "delete_item",
- "round": 9,
- "atMs": 51025.99179199996,
- "execMs": 0.06391599995549768
- },
- {
- "name": "refine_speech_cuts",
- "round": 10,
- "atMs": 55723.74920899997,
- "execMs": 0.03874999994877726
- },
- {
- "name": "get_state",
- "round": 11,
- "atMs": 60025.71583399997,
- "execMs": 0.09079099999507889
- },
- {
- "name": "subtitles_generate",
- "round": 12,
- "atMs": 64085.775166999956,
- "execMs": 0.03095799998845905
+ "atMs": 13680.798207999906,
+ "execMs": 0.010083000059239566
}
],
"timings": {
- "gateMs": 999.5926660000114,
+ "gateMs": 1109.280207999982,
"gateSkipped": false,
- "ttftMs": 66488.46224999998,
- "modelMs": 66009.95054300013,
- "toolMs": 0.7181239999481477,
- "totalMs": 67010.26133300009,
- "judgeMs": 2012.1048749999609,
+ "ttftMs": 16261.186666999944,
+ "modelMs": 16482.418000000296,
+ "toolMs": 0.04937600006815046,
+ "totalMs": 16492.215916999965,
+ "judgeMs": 1274.6012090001022,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5923.954209000047,
+ "roundMs": 1108.0415829999838,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.07491700001992285
- }
- ]
+ "calls": []
},
{
- "roundMs": 5637.810540999984,
+ "roundMs": 4955.758834000095,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.020542000012937933
- }
- ]
- },
- {
- "roundMs": 3128.153875000018,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.03437500004656613
+ "execMs": 0.01508399995509535
}
]
},
{
- "roundMs": 7077.547791999998,
+ "roundMs": 2466.845250000013,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.020040999981574714
- }
- ]
- },
- {
- "roundMs": 3279.978625000047,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.02374999999301508
- }
- ]
- },
- {
- "roundMs": 11807.210250000004,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.14816699997754768
- }
- ]
- },
- {
- "roundMs": 4826.861708000011,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.04079200001433492
- }
- ]
- },
- {
- "roundMs": 1999.280792000005,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "split_at",
- "execMs": 0.04829100001370534
+ "execMs": 0.00779200007673353
}
]
},
{
- "roundMs": 3168.30441599997,
+ "roundMs": 2641.9408750000875,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.08283400000073016
+ "name": "get_state",
+ "execMs": 0.016416999977082014
}
]
},
{
- "roundMs": 3158.4506670000264,
+ "roundMs": 2499.9526250000345,
"firstDeltaMs": null,
"calls": [
{
- "name": "delete_item",
- "execMs": 0.06391599995549768
+ "name": "generate_scene",
+ "execMs": 0.010083000059239566
}
]
},
{
- "roundMs": 4695.882917000039,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.03874999994877726
- }
- ]
- },
+ "roundMs": 2809.8788330000825,
+ "firstDeltaMs": 2579.092750000069,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "cartoon-korean-audio-passes-language",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1163,
+ "p95": 1763,
+ "mean": 1347,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 8846,
+ "p95": 9186,
+ "mean": 8728,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 9097,
+ "p95": 9319,
+ "mean": 8984,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 9091,
+ "p95": 9311,
+ "mean": 8976,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 3,
+ "max": 3
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 6855.863832999952,
+ "execMs": 0.3638749999227002
+ }
+ ],
+ "timings": {
+ "gateMs": 1163.4539999999106,
+ "gateSkipped": false,
+ "ttftMs": 9185.876916000037,
+ "modelMs": 9311.082042000024,
+ "toolMs": 0.3638749999227002,
+ "totalMs": 9319.163707999978,
+ "judgeMs": 2025.576250000042,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
{
- "roundMs": 4300.739208999963,
+ "roundMs": 1162.4710000000196,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "get_state",
- "execMs": 0.09079099999507889
- }
- ]
+ "calls": []
},
{
- "roundMs": 4058.1978750000126,
+ "roundMs": 5687.810999999987,
"firstDeltaMs": null,
"calls": [
{
- "name": "subtitles_generate",
- "execMs": 0.03095799998845905
+ "name": "generate_scene",
+ "execMs": 0.3638749999227002
}
]
},
{
- "roundMs": 2947.577667000005,
- "firstDeltaMs": 2401.0788329999777,
+ "roundMs": 2460.800042000017,
+ "firstDeltaMs": 2328.1809999999823,
"calls": []
}
]
@@ -5985,227 +5010,404 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
- "notes": [],
- "judgeNote": null,
- "trace": [
- {
- "name": "list_skills",
- "round": 0,
- "atMs": 7087.347208000021,
- "execMs": 0.02074999996693805
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 12117.584124999994,
- "execMs": 0.02395900001283735
- },
- {
- "name": "read_skill",
- "round": 1,
- "atMs": 12117.832915999985,
- "execMs": 0.002041999949142337
- },
- {
- "name": "detect_silence",
- "round": 2,
- "atMs": 15199.83812500001,
- "execMs": 0.07095800002571195
- },
- {
- "name": "watch_video",
- "round": 3,
- "atMs": 17819.60358300002,
- "execMs": 0.026249999995343387
- },
- {
- "name": "get_state",
- "round": 4,
- "atMs": 20206.557166000013,
- "execMs": 0.07237499998882413
- },
- {
- "name": "listen_audio",
- "round": 5,
- "atMs": 23072.90525000001,
- "execMs": 0.05762499995762482
- },
- {
- "name": "split_at",
- "round": 6,
- "atMs": 32408.411040999985,
- "execMs": 0.12708299997029826
- },
- {
- "name": "split_at",
- "round": 7,
- "atMs": 35299.161124999984,
- "execMs": 0.13579099997878075
- },
- {
- "name": "split_at",
- "round": 8,
- "atMs": 37978.80154100002,
- "execMs": 0.1010420000529848
- },
- {
- "name": "delete_item",
- "round": 9,
- "atMs": 41658.567125,
- "execMs": 0.09345899999607354
- },
- {
- "name": "delete_item",
- "round": 10,
- "atMs": 44294.1715,
- "execMs": 0.0711250000167638
- },
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
{
- "name": "refine_speech_cuts",
- "round": 11,
- "atMs": 47768.65591600002,
- "execMs": 0.08133399998769164
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 6495.944833999965,
+ "execMs": 0.015959000098519027
}
],
"timings": {
- "gateMs": 1059.9210420000018,
+ "gateMs": 1762.5957090000156,
"gateSkipped": false,
- "ttftMs": 50990.293125000026,
- "modelMs": 50293.48083499988,
- "toolMs": 0.8837929998990148,
- "totalMs": 51354.28566999978,
- "judgeMs": 1483.9913750000414,
+ "ttftMs": 8845.75774999999,
+ "modelMs": 9091.322332999902,
+ "toolMs": 0.015959000098519027,
+ "totalMs": 9096.86487499997,
+ "judgeMs": 1857.4123339999933,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6025.746207999997,
+ "roundMs": 1761.25320799998,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "list_skills",
- "execMs": 0.02074999996693805
- }
- ]
+ "calls": []
},
{
- "roundMs": 5029.146708999993,
+ "roundMs": 4730.534124999889,
"firstDeltaMs": null,
"calls": [
{
- "name": "read_skill",
- "execMs": 0.02395900001283735
- },
- {
- "name": "read_skill",
- "execMs": 0.002041999949142337
+ "name": "generate_scene",
+ "execMs": 0.015959000098519027
}
]
},
{
- "roundMs": 3080.0103750000126,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "detect_silence",
- "execMs": 0.07095800002571195
- }
- ]
- },
+ "roundMs": 2599.5350000000326,
+ "firstDeltaMs": 2348.619834000012,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 1,
+ "atMs": 5688.750958000077,
+ "execMs": 0.028082999982871115
+ }
+ ],
+ "timings": {
+ "gateMs": 1115.0982079999521,
+ "gateSkipped": false,
+ "ttftMs": 8153.627875000006,
+ "modelMs": 8526.391291000065,
+ "toolMs": 0.028082999982871115,
+ "totalMs": 8534.664292000001,
+ "judgeMs": 1319.3290840000845,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
{
- "roundMs": 2617.637624999974,
+ "roundMs": 1112.068333000061,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "watch_video",
- "execMs": 0.026249999995343387
- }
- ]
+ "calls": []
},
{
- "roundMs": 2385.1805830000085,
+ "roundMs": 4569.9541250000475,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.07237499998882413
+ "name": "generate_scene",
+ "execMs": 0.028082999982871115
}
]
},
{
- "roundMs": 2865.0903340000077,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "listen_audio",
- "execMs": 0.05762499995762482
- }
- ]
- },
+ "roundMs": 2844.368832999957,
+ "firstDeltaMs": 2463.598915999988,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "cartoon-style-reference-anchors-look",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 7881,
+ "p95": 10358,
+ "mean": 8196,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 8139,
+ "p95": 10617,
+ "mean": 8436,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 8135,
+ "p95": 10611,
+ "mean": 8431,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 0,
+ "atMs": 6944.749375000014,
+ "execMs": 0.2282090000808239
+ }
+ ],
+ "timings": {
+ "gateMs": 0.21625000005587935,
+ "gateSkipped": true,
+ "ttftMs": 10358.41858399997,
+ "modelMs": 10611.405583999935,
+ "toolMs": 0.2282090000808239,
+ "totalMs": 10617.165708999964,
+ "judgeMs": 1065.3071249999339,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
{
- "roundMs": 9333.834207999986,
+ "roundMs": 6941.688208999927,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.12708299997029826
+ "name": "generate_scene",
+ "execMs": 0.2282090000808239
}
]
},
{
- "roundMs": 2888.891041999974,
+ "roundMs": 3669.7173750000075,
+ "firstDeltaMs": 3411.2746250000782,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 0,
+ "atMs": 3361.6538329999894,
+ "execMs": 0.014500000048428774
+ }
+ ],
+ "timings": {
+ "gateMs": 0.22966699989046901,
+ "gateSkipped": true,
+ "ttftMs": 6347.366207999992,
+ "modelMs": 6546.729374999995,
+ "toolMs": 0.014500000048428774,
+ "totalMs": 6550.469832999981,
+ "judgeMs": 1928.1499169999734,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 3359.4391250000335,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.13579099997878075
+ "name": "generate_scene",
+ "execMs": 0.014500000048428774
}
]
},
{
- "roundMs": 2677.6118340000394,
+ "roundMs": 3187.290249999962,
+ "firstDeltaMs": 2984.4217499999795,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "generate_scene",
+ "round": 0,
+ "atMs": 4643.81708399998,
+ "execMs": 0.011457999935373664
+ }
+ ],
+ "timings": {
+ "gateMs": 0.2579580000601709,
+ "gateSkipped": true,
+ "ttftMs": 7881.386708999984,
+ "modelMs": 8134.899583999999,
+ "toolMs": 0.011457999935373664,
+ "totalMs": 8138.906709000003,
+ "judgeMs": 1142.8243749999674,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 4641.143750000047,
"firstDeltaMs": null,
"calls": [
{
- "name": "split_at",
- "execMs": 0.1010420000529848
+ "name": "generate_scene",
+ "execMs": 0.011457999935373664
}
]
},
{
- "roundMs": 3677.743124999979,
+ "roundMs": 3493.7558339999523,
+ "firstDeltaMs": 3236.4434999999357,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "cartoon-question-stays-in-chat",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1531,
+ "p95": 1819,
+ "mean": 1447,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 5928,
+ "p95": 7291,
+ "mean": 6182,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 6189,
+ "p95": 7293,
+ "mean": 6394,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 6186,
+ "p95": 7289,
+ "mean": 6389,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 990.7333750000689,
+ "gateSkipped": false,
+ "ttftMs": 5328.988124999916,
+ "modelMs": 5692.534958999953,
+ "toolMs": 0,
+ "totalMs": 5698.772624999983,
+ "judgeMs": 1623.411041000043,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 988.2134999999544,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.09345899999607354
- }
- ]
+ "calls": []
},
{
- "roundMs": 2633.675249999971,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "delete_item",
- "execMs": 0.0711250000167638
- }
- ]
+ "roundMs": 4704.321458999999,
+ "firstDeltaMs": 4334.962125000078,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 1530.7037919999566,
+ "gateSkipped": false,
+ "ttftMs": 7290.52887499996,
+ "modelMs": 7289.229124999954,
+ "toolMs": 0,
+ "totalMs": 7293.026249999995,
+ "judgeMs": 1473.4754170000087,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1529.5834170000162,
+ "firstDeltaMs": 1522.7665839999681,
+ "calls": []
},
{
- "roundMs": 3472.2724999999627,
+ "roundMs": 5759.645707999938,
+ "firstDeltaMs": 5757.386667000013,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 1818.7792080000509,
+ "gateSkipped": false,
+ "ttftMs": 5927.923999999999,
+ "modelMs": 6185.797125000041,
+ "toolMs": 0,
+ "totalMs": 6189.042083999957,
+ "judgeMs": 1058.921457999968,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1817.5179169999901,
"firstDeltaMs": null,
- "calls": [
- {
- "name": "refine_speech_cuts",
- "execMs": 0.08133399998769164
- }
- ]
+ "calls": []
},
{
- "roundMs": 3606.641041999974,
- "firstDeltaMs": 3219.5574579999666,
+ "roundMs": 4368.279208000051,
+ "firstDeltaMs": 4107.400166000007,
"calls": []
}
]
@@ -6226,21 +5428,21 @@
"n": 3
},
"ttftMs": {
- "p50": 35399,
- "p95": 43439,
- "mean": 34699,
+ "p50": 22507,
+ "p95": 48811,
+ "mean": 31032,
"n": 3
},
"totalMs": {
- "p50": 35668,
- "p95": 43823,
- "mean": 35080,
+ "p50": 23021,
+ "p95": 49080,
+ "mean": 31383,
"n": 3
},
"modelMs": {
- "p50": 35668,
- "p95": 43822,
- "mean": 35079,
+ "p50": 23009,
+ "p95": 49069,
+ "mean": 31371,
"n": 3
},
"toolMs": {
@@ -6250,127 +5452,127 @@
"n": 3
},
"rounds": {
- "mean": 5.7,
- "max": 7
+ "mean": 5.3,
+ "max": 6
}
},
"runs": [
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 8056.243207999971,
- "execMs": 0.40549999999348074
+ "atMs": 4187.590833000024,
+ "execMs": 0.30533400003332645
},
{
"name": "watch_video",
"round": 0,
- "atMs": 8057.2199580000015,
- "execMs": 0.012584000010974705
+ "atMs": 4188.328374999925,
+ "execMs": 0.009291000082157552
},
{
"name": "watch_video",
"round": 0,
- "atMs": 8057.368749999965,
- "execMs": 0.0015830000047571957
+ "atMs": 4188.498874999932,
+ "execMs": 0.0019169999286532402
},
{
"name": "get_state",
"round": 1,
- "atMs": 11924.173040999973,
- "execMs": 0.015375000017229468
+ "atMs": 6315.458291999996,
+ "execMs": 0.0055420000571757555
},
{
- "name": "regenerate_shot",
+ "name": "read_skill",
"round": 2,
- "atMs": 30927.856541000016,
- "execMs": 0.016124999965541065
+ "atMs": 8625.43508299999,
+ "execMs": 0.01454200001899153
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 30928.10329099995,
- "execMs": 0.003541999962180853
+ "round": 3,
+ "atMs": 19437.119749999954,
+ "execMs": 0.019542000023648143
},
{
- "name": "wait_for_renders",
+ "name": "regenerate_shot",
"round": 3,
- "atMs": 40615.82124999998,
- "execMs": 0.012332999962382019
+ "atMs": 19437.389291999978,
+ "execMs": 0.0047920000506564975
}
],
"timings": {
- "gateMs": 0.26479099999414757,
+ "gateMs": 0.23687499994412065,
"gateSkipped": true,
- "ttftMs": 43438.55249999993,
- "modelMs": 43822.2095,
- "toolMs": 0.46704199991654605,
- "totalMs": 43822.94133299991,
- "judgeMs": 1128.1954580000602,
+ "ttftMs": 21778.184124999912,
+ "modelMs": 22034.8331680001,
+ "toolMs": 0.36096000019460917,
+ "totalMs": 22048.31666699995,
+ "judgeMs": 1057.5864169999259,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 8053.108000000007,
+ "roundMs": 4184.664042000077,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.40549999999348074
+ "execMs": 0.30533400003332645
},
{
"name": "watch_video",
- "execMs": 0.012584000010974705
+ "execMs": 0.009291000082157552
},
{
"name": "watch_video",
- "execMs": 0.0015830000047571957
+ "execMs": 0.0019169999286532402
}
]
},
{
- "roundMs": 3865.434874999977,
+ "roundMs": 2125.6766249999637,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.015375000017229468
+ "execMs": 0.0055420000571757555
}
]
},
{
- "roundMs": 18999.925749999995,
+ "roundMs": 2309.200542000006,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.016124999965541065
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.003541999962180853
+ "name": "read_skill",
+ "execMs": 0.01454200001899153
}
]
},
{
- "roundMs": 9685.54912500002,
+ "roundMs": 10807.543834000011,
"firstDeltaMs": null,
"calls": [
{
- "name": "wait_for_renders",
- "execMs": 0.012332999962382019
+ "name": "regenerate_shot",
+ "execMs": 0.019542000023648143
+ },
+ {
+ "name": "regenerate_shot",
+ "execMs": 0.0047920000506564975
}
]
},
{
- "roundMs": 3218.191749999998,
- "firstDeltaMs": 2821.414166999981,
+ "roundMs": 2607.748125000042,
+ "firstDeltaMs": 2338.9197080000304,
"calls": []
}
]
@@ -6379,151 +5581,125 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 6862.350708000013,
- "execMs": 0.022291000001132488
+ "atMs": 4708.1722079999745,
+ "execMs": 0.02904100005980581
},
{
"name": "watch_video",
"round": 0,
- "atMs": 6862.67975000001,
- "execMs": 0.004166999948211014
+ "atMs": 4708.477958000032,
+ "execMs": 0.011249999981373549
},
{
"name": "watch_video",
"round": 0,
- "atMs": 6862.74216699996,
- "execMs": 0.000750000006519258
+ "atMs": 4708.603375000064,
+ "execMs": 0.0028750000055879354
},
{
"name": "get_state",
"round": 1,
- "atMs": 12421.135541999945,
- "execMs": 0.02779199997894466
+ "atMs": 10059.71304199996,
+ "execMs": 0.01924999989569187
},
{
"name": "list_skills",
"round": 2,
- "atMs": 18781.27150000003,
- "execMs": 0.021167000057175756
+ "atMs": 12367.492916999967,
+ "execMs": 0.007165999966673553
},
{
"name": "read_skill",
"round": 3,
- "atMs": 20810.100083000027,
- "execMs": 0.008875000057742
- },
- {
- "name": "regenerate_shot",
- "round": 4,
- "atMs": 30250.42304200004,
- "execMs": 0.01587499992456287
+ "atMs": 14504.767749999999,
+ "execMs": 0.004249999998137355
},
{
- "name": "regenerate_shot",
+ "name": "recut_scene",
"round": 4,
- "atMs": 30250.658832999994,
- "execMs": 0.002665999927558005
- },
- {
- "name": "get_state",
- "round": 5,
- "atMs": 33241.44791700004,
- "execMs": 0.018708000076003373
+ "atMs": 20115.177917000023,
+ "execMs": 0.01991699996870011
}
],
"timings": {
- "gateMs": 0.09379199997056276,
+ "gateMs": 0.3236669999314472,
"gateSkipped": true,
- "ttftMs": 35398.84133299999,
- "modelMs": 35667.60704399983,
- "toolMs": 0.12229099997784942,
- "totalMs": 35667.82312699978,
- "judgeMs": 1233.2641660000663,
+ "ttftMs": 22506.839875000063,
+ "modelMs": 23009.056707000127,
+ "toolMs": 0.09374899987597018,
+ "totalMs": 23021.284083000035,
+ "judgeMs": 1149.3532500000438,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6861.0293749999255,
+ "roundMs": 4705.341708000051,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.022291000001132488
+ "execMs": 0.02904100005980581
},
{
"name": "watch_video",
- "execMs": 0.004166999948211014
+ "execMs": 0.011249999981373549
},
{
"name": "watch_video",
- "execMs": 0.000750000006519258
+ "execMs": 0.0028750000055879354
}
]
},
{
- "roundMs": 5557.233833999955,
+ "roundMs": 5349.469167000032,
"firstDeltaMs": null,
"calls": [
{
"name": "get_state",
- "execMs": 0.02779199997894466
+ "execMs": 0.01924999989569187
}
]
},
{
- "roundMs": 6358.868833999964,
+ "roundMs": 2306.302040999988,
"firstDeltaMs": null,
"calls": [
{
"name": "list_skills",
- "execMs": 0.021167000057175756
+ "execMs": 0.007165999966673553
}
]
},
{
- "roundMs": 2027.348083000048,
+ "roundMs": 2136.6238750000484,
"firstDeltaMs": null,
"calls": [
{
"name": "read_skill",
- "execMs": 0.008875000057742
- }
- ]
- },
- {
- "roundMs": 9438.901874999981,
- "firstDeltaMs": null,
- "calls": [
- {
- "name": "regenerate_shot",
- "execMs": 0.01587499992456287
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.002665999927558005
+ "execMs": 0.004249999998137355
}
]
},
{
- "roundMs": 2989.1975839999504,
+ "roundMs": 5607.82358299999,
"firstDeltaMs": null,
"calls": [
{
- "name": "get_state",
- "execMs": 0.018708000076003373
+ "name": "recut_scene",
+ "execMs": 0.01991699996870011
}
]
},
{
- "roundMs": 2435.0274590000045,
- "firstDeltaMs": 2155.7786250000354,
+ "roundMs": 2903.4963330000173,
+ "firstDeltaMs": 2389.8071660000132,
"calls": []
}
]
@@ -6532,109 +5708,119 @@
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "listen_audio",
"round": 0,
- "atMs": 5057.851833999972,
- "execMs": 0.012750000110827386
+ "atMs": 29136.014417,
+ "execMs": 0.12166699999943376
},
{
"name": "watch_video",
- "round": 1,
- "atMs": 7860.243541999953,
- "execMs": 0.013416999951004982
+ "round": 0,
+ "atMs": 29136.341957999975,
+ "execMs": 0.006333000026643276
},
{
"name": "watch_video",
+ "round": 0,
+ "atMs": 29136.451499999966,
+ "execMs": 0.02612499997485429
+ },
+ {
+ "name": "list_skills",
"round": 1,
- "atMs": 7860.524916999973,
- "execMs": 0.0032079999800771475
+ "atMs": 37993.220207999926,
+ "execMs": 0.014874999993480742
},
{
- "name": "regenerate_shot",
+ "name": "read_skill",
"round": 2,
- "atMs": 17819.247250000015,
- "execMs": 0.013208999997004867
+ "atMs": 40291.80679199996,
+ "execMs": 0.011000000056810677
},
{
"name": "regenerate_shot",
- "round": 2,
- "atMs": 17819.553166999947,
- "execMs": 0.005374999949708581
+ "round": 3,
+ "atMs": 46563.83049999992,
+ "execMs": 0.02970900002401322
},
{
- "name": "wait_for_renders",
+ "name": "regenerate_shot",
"round": 3,
- "atMs": 23448.917750000022,
- "execMs": 0.011791000026278198
+ "atMs": 46564.453666999936,
+ "execMs": 0.011666000005789101
}
],
"timings": {
- "gateMs": 0.12062499998137355,
+ "gateMs": 0.34658299991860986,
"gateSkipped": true,
- "ttftMs": 25259.77004199999,
- "modelMs": 25748.360875999904,
- "toolMs": 0.05975000001490116,
- "totalMs": 25748.5412509999,
- "judgeMs": 1906.3215830000117,
+ "ttftMs": 48810.50724999991,
+ "modelMs": 49069.40004199988,
+ "toolMs": 0.22137500008102506,
+ "totalMs": 49079.596499999985,
+ "judgeMs": 1581.3340830000816,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 5056.326624999987,
+ "roundMs": 29133.930374999996,
"firstDeltaMs": null,
"calls": [
{
"name": "listen_audio",
- "execMs": 0.012750000110827386
+ "execMs": 0.12166699999943376
+ },
+ {
+ "name": "watch_video",
+ "execMs": 0.006333000026643276
+ },
+ {
+ "name": "watch_video",
+ "execMs": 0.02612499997485429
}
]
},
{
- "roundMs": 2800.944958999986,
+ "roundMs": 8855.193916999968,
"firstDeltaMs": null,
"calls": [
{
- "name": "watch_video",
- "execMs": 0.013416999951004982
- },
- {
- "name": "watch_video",
- "execMs": 0.0032079999800771475
+ "name": "list_skills",
+ "execMs": 0.014874999993480742
}
]
},
{
- "roundMs": 9957.165624999907,
+ "roundMs": 2297.297499999986,
"firstDeltaMs": null,
"calls": [
{
- "name": "regenerate_shot",
- "execMs": 0.013208999997004867
- },
- {
- "name": "regenerate_shot",
- "execMs": 0.005374999949708581
+ "name": "read_skill",
+ "execMs": 0.011000000056810677
}
]
},
{
- "roundMs": 5627.947999999975,
+ "roundMs": 6270.039458999992,
"firstDeltaMs": null,
"calls": [
{
- "name": "wait_for_renders",
- "execMs": 0.011791000026278198
+ "name": "regenerate_shot",
+ "execMs": 0.02970900002401322
+ },
+ {
+ "name": "regenerate_shot",
+ "execMs": 0.011666000005789101
}
]
},
{
- "roundMs": 2305.9756670000497,
- "firstDeltaMs": 1809.4944169999799,
+ "roundMs": 2512.938790999935,
+ "firstDeltaMs": 2244.4716659999685,
"calls": []
}
]
@@ -6649,27 +5835,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1816,
- "p95": 1892,
- "mean": 1758,
+ "p50": 1081,
+ "p95": 1138,
+ "mean": 1090,
"n": 3
},
"ttftMs": {
- "p50": 7329,
- "p95": 10454,
- "mean": 7855,
+ "p50": 5380,
+ "p95": 8385,
+ "mean": 5931,
"n": 3
},
"totalMs": {
- "p50": 8045,
- "p95": 10708,
- "mean": 8265,
+ "p50": 5767,
+ "p95": 8821,
+ "mean": 6285,
"n": 3
},
"modelMs": {
- "p50": 6229,
- "p95": 8815,
- "mean": 6507,
+ "p50": 5758,
+ "p95": 8808,
+ "mean": 6276,
"n": 3
},
"toolMs": {
@@ -6694,50 +5880,50 @@
{
"name": "set_speed",
"round": 0,
- "atMs": 6567.871999999999,
- "execMs": 0.03154199999971752
+ "atMs": 5067.838542000041,
+ "execMs": 0.1287089999532327
},
{
"name": "remove_transition",
"round": 1,
- "atMs": 9124.000666,
- "execMs": 0.003584000000046217
+ "atMs": 6913.768541999976,
+ "execMs": 0.008832999970763922
}
],
"timings": {
- "gateMs": 1892.390709,
+ "gateMs": 1050.7836670000106,
"gateSkipped": false,
- "ttftMs": 10454.172208,
- "modelMs": 8815.177416999999,
- "toolMs": 0.035125999999763735,
- "totalMs": 10707.603251999997,
- "judgeMs": 1091.931708,
+ "ttftMs": 8385.455707999994,
+ "modelMs": 8808.461208999972,
+ "toolMs": 0.13754199992399663,
+ "totalMs": 8820.678291999968,
+ "judgeMs": 1421.2856659999816,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4665.019875,
+ "roundMs": 5061.4518749999115,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed",
- "execMs": 0.03154199999971752
+ "execMs": 0.1287089999532327
}
]
},
{
- "roundMs": 2554.0044579999994,
+ "roundMs": 1842.4036250000354,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.003584000000046217
+ "execMs": 0.008832999970763922
}
]
},
{
- "roundMs": 1596.1530839999996,
- "firstDeltaMs": 1329.2802919999995,
+ "roundMs": 1904.605709000025,
+ "firstDeltaMs": 1470.250417000032,
"calls": []
}
]
@@ -6753,50 +5939,60 @@
{
"name": "set_speed",
"round": 0,
- "atMs": 3276.7528330000005,
- "execMs": 0.003582999999707681
+ "atMs": 1802.389582999982,
+ "execMs": 0.00958299997728318
},
{
"name": "remove_transition",
"round": 1,
- "atMs": 5696.155333000001,
- "execMs": 0.0017079999997804407
+ "atMs": 3659.1113749999786,
+ "execMs": 0.007582999998703599
+ },
+ {
+ "name": "set_transition",
+ "round": 1,
+ "atMs": 3659.770790999988,
+ "execMs": 0.01608299999497831
}
],
"timings": {
- "gateMs": 1816.0896250000005,
+ "gateMs": 1137.5719159999862,
"gateSkipped": false,
- "ttftMs": 7328.526917000001,
- "modelMs": 6229.238291,
- "toolMs": 0.005290999999488122,
- "totalMs": 8045.333207,
- "judgeMs": 1321.5924579999992,
+ "ttftMs": 5380.138874999946,
+ "modelMs": 5757.891666999902,
+ "toolMs": 0.03324899997096509,
+ "totalMs": 5766.940290999948,
+ "judgeMs": 1134.95633299998,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1458.3694159999995,
+ "roundMs": 1798.3795000000391,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed",
- "execMs": 0.003582999999707681
+ "execMs": 0.00958299997728318
}
]
},
{
- "roundMs": 2418.0651670000007,
+ "roundMs": 1854.3497499999357,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.0017079999997804407
+ "execMs": 0.007582999998703599
+ },
+ {
+ "name": "set_transition",
+ "execMs": 0.01608299999497831
}
]
},
{
- "roundMs": 2352.8037079999995,
- "firstDeltaMs": 1631.5553330000002,
+ "roundMs": 2105.162416999927,
+ "firstDeltaMs": 1718.4462920000078,
"calls": []
}
]
@@ -6812,50 +6008,50 @@
{
"name": "set_speed",
"round": 0,
- "atMs": 2714.306375,
- "execMs": 0.0660409999982221
+ "atMs": 1591.6570000000065,
+ "execMs": 0.008042000001296401
},
{
"name": "remove_transition",
"round": 1,
- "atMs": 4531.510208,
- "execMs": 0.004791999999724794
+ "atMs": 3000.5574169999454,
+ "execMs": 0.009083000011742115
}
],
"timings": {
- "gateMs": 1564.3106249999983,
+ "gateMs": 1080.5199579999316,
"gateSkipped": false,
- "ttftMs": 5782.315290999999,
- "modelMs": 4476.862793,
- "toolMs": 0.0708329999979469,
- "totalMs": 6041.244250999996,
- "judgeMs": 1345.8231659999983,
+ "ttftMs": 4026.4974999999395,
+ "modelMs": 4260.221249999944,
+ "toolMs": 0.017125000013038516,
+ "totalMs": 4266.780625000014,
+ "judgeMs": 1232.7599159999518,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1143.7194170000002,
+ "roundMs": 1588.6411249999655,
"firstDeltaMs": null,
"calls": [
{
"name": "set_speed",
- "execMs": 0.0660409999982221
+ "execMs": 0.008042000001296401
}
]
},
{
- "roundMs": 1815.088083999999,
+ "roundMs": 1407.4698749999516,
"firstDeltaMs": null,
"calls": [
{
"name": "remove_transition",
- "execMs": 0.004791999999724794
+ "execMs": 0.009083000011742115
}
]
},
{
- "roundMs": 1518.055292000001,
- "firstDeltaMs": 1249.7950420000016,
+ "roundMs": 1264.110250000027,
+ "firstDeltaMs": 1024.4331670000684,
"calls": []
}
]
@@ -6870,27 +6066,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1907,
- "p95": 2092,
- "mean": 1883,
+ "p50": 1225,
+ "p95": 1843,
+ "mean": 1343,
"n": 3
},
"ttftMs": {
- "p50": 16227,
- "p95": 17157,
- "mean": 15269,
+ "p50": 9322,
+ "p95": 10322,
+ "mean": 8645,
"n": 3
},
"totalMs": {
- "p50": 16540,
- "p95": 17402,
- "mean": 15617,
+ "p50": 9424,
+ "p95": 10488,
+ "mean": 8746,
"n": 3
},
"modelMs": {
- "p50": 14891,
- "p95": 15495,
- "mean": 13735,
+ "p50": 9324,
+ "p95": 10482,
+ "mean": 8709,
"n": 3
},
"toolMs": {
@@ -6900,59 +6096,272 @@
"n": 3
},
"rounds": {
- "mean": 2,
- "max": 2
+ "mean": 3.3,
+ "max": 4
}
},
"runs": [
{
"pass": true,
"intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "roundModel": "gemini-3.7-flash",
"notes": [],
"judgeNote": null,
"trace": [
{
"name": "remove_transition",
- "round": 0,
- "atMs": 9053.748000000003,
- "execMs": 0.2381660000028205
+ "round": 2,
+ "atMs": 5758.489457999938,
+ "execMs": 0.2670419999631122
+ },
+ {
+ "name": "remove_transition",
+ "round": 2,
+ "atMs": 5759.184832999948,
+ "execMs": 0.005124999908730388
+ }
+ ],
+ "timings": {
+ "gateMs": 1842.7465000000084,
+ "gateSkipped": false,
+ "ttftMs": 9322.051665999927,
+ "modelMs": 9323.84079200006,
+ "toolMs": 0.27216699987184256,
+ "totalMs": 9424.120708000031,
+ "judgeMs": 1051.0892079999903,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1748.015791999991,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 0.010917000006884336,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3912.983416000032,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "remove_transition",
+ "execMs": 0.2670419999631122
+ },
+ {
+ "name": "remove_transition",
+ "execMs": 0.005124999908730388
+ }
+ ]
+ },
+ {
+ "roundMs": 3662.830667000031,
+ "firstDeltaMs": 3561.2727920000907,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "remove_transition",
+ "round": 1,
+ "atMs": 5423.940833000001,
+ "execMs": 0.010832999949343503
+ },
+ {
+ "name": "remove_transition",
+ "round": 1,
+ "atMs": 5424.264708000002,
+ "execMs": 0.0025840000016614795
+ }
+ ],
+ "timings": {
+ "gateMs": 1224.9432500000112,
+ "gateSkipped": false,
+ "ttftMs": 10321.68995800009,
+ "modelMs": 10482.213876000023,
+ "toolMs": 0.013416999951004982,
+ "totalMs": 10488.190708000096,
+ "judgeMs": 1043.438208000036,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1224.3218750000233,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 4195.833417000016,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "remove_transition",
+ "execMs": 0.010832999949343503
+ },
+ {
+ "name": "remove_transition",
+ "execMs": 0.0025840000016614795
+ }
+ ]
+ },
+ {
+ "roundMs": 5062.058583999984,
+ "firstDeltaMs": 4895.948166999966,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "remove_transition",
+ "round": 1,
+ "atMs": 4204.064082999947,
+ "execMs": 0.0058750000316649675
},
{
- "name": "remove_transition",
+ "name": "remove_transition",
+ "round": 1,
+ "atMs": 4204.196166999987,
+ "execMs": 0.0007910000858828425
+ }
+ ],
+ "timings": {
+ "gateMs": 962.107416999992,
+ "gateSkipped": false,
+ "ttftMs": 6292.518541999976,
+ "modelMs": 6320.321458999999,
+ "toolMs": 0.00666600011754781,
+ "totalMs": 6324.256499999901,
+ "judgeMs": 1306.198083999916,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 961.3636669999687,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3239.7796670000535,
+ "firstDeltaMs": null,
+ "calls": [
+ {
+ "name": "remove_transition",
+ "execMs": 0.0058750000316649675
+ },
+ {
+ "name": "remove_transition",
+ "execMs": 0.0007910000858828425
+ }
+ ]
+ },
+ {
+ "roundMs": 2119.1781249999767,
+ "firstDeltaMs": 2087.5011249999516,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "errored-call-not-claimed-done",
+ "bucket": "single-tool",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1157,
+ "p95": 1249,
+ "mean": 1175,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 3291,
+ "p95": 6507,
+ "mean": 4346,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 4081,
+ "p95": 6843,
+ "mean": 4835,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 4075,
+ "p95": 6838,
+ "mean": 4829,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 2,
+ "max": 2
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [
+ {
+ "name": "set_transition",
"round": 0,
- "atMs": 9054.308125,
- "execMs": 0.0017909999951370992
+ "atMs": 1531.4902920000022,
+ "execMs": 0.315165999927558
}
],
"timings": {
- "gateMs": 2091.8164160000015,
+ "gateMs": 1119.1461660000496,
"gateSkipped": false,
- "ttftMs": 12423.465999999997,
- "modelMs": 10817.685292000002,
- "toolMs": 0.2399569999979576,
- "totalMs": 12909.741665000001,
- "judgeMs": 934.2194159999999,
+ "ttftMs": 3290.720250000013,
+ "modelMs": 4075.1198330000043,
+ "toolMs": 0.315165999927558,
+ "totalMs": 4080.65149999992,
+ "judgeMs": 1837.0576669999864,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 6960.687666999998,
+ "roundMs": 1529.4212919999845,
"firstDeltaMs": null,
"calls": [
{
- "name": "remove_transition",
- "execMs": 0.2381660000028205
- },
- {
- "name": "remove_transition",
- "execMs": 0.0017909999951370992
+ "name": "set_transition",
+ "execMs": 0.315165999927558
}
]
},
{
- "roundMs": 3856.9976250000036,
- "firstDeltaMs": 3367.9055829999998,
+ "roundMs": 2545.6985410000198,
+ "firstDeltaMs": 1756.126833000104,
"calls": []
}
]
@@ -6960,52 +6369,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
"judgeNote": null,
"trace": [
{
- "name": "remove_transition",
- "round": 0,
- "atMs": 11446.902541000003,
- "execMs": 0.009291000002122018
- },
- {
- "name": "remove_transition",
+ "name": "set_transition",
"round": 0,
- "atMs": 11447.162540999998,
- "execMs": 0.0011249999952269718
+ "atMs": 1672.9543330000015,
+ "execMs": 0.08108300005551428
}
],
"timings": {
- "gateMs": 1906.9025000000038,
+ "gateMs": 1157.4554159999825,
"gateSkipped": false,
- "ttftMs": 17156.802540999997,
- "modelMs": 15495.128375,
- "toolMs": 0.01041599999734899,
- "totalMs": 17402.041291,
- "judgeMs": 1943.2617499999978,
+ "ttftMs": 3239.8918330000015,
+ "modelMs": 3574.3898340000305,
+ "toolMs": 0.08108300005551428,
+ "totalMs": 3580.2818330000155,
+ "judgeMs": 1083.5274590000045,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 9538.00675,
+ "roundMs": 1669.6450840000762,
"firstDeltaMs": null,
"calls": [
{
- "name": "remove_transition",
- "execMs": 0.009291000002122018
- },
- {
- "name": "remove_transition",
- "execMs": 0.0011249999952269718
+ "name": "set_transition",
+ "execMs": 0.08108300005551428
}
]
},
{
- "roundMs": 5957.121625,
- "firstDeltaMs": 5708.721250000002,
+ "roundMs": 1904.7447499999544,
+ "firstDeltaMs": 1564.65625,
"calls": []
}
]
@@ -7013,52 +6412,42 @@
},
{
"pass": true,
- "intent": "complex",
- "roundModel": "gemini-3.5-flash",
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
"notes": [],
- "judgeNote": null,
+ "judgeNote": "the tool did not run, but the assistant did not clearly state the failure as an editing turn.",
"trace": [
{
- "name": "remove_transition",
- "round": 0,
- "atMs": 13127.238042000004,
- "execMs": 0.010708000001613982
- },
- {
- "name": "remove_transition",
+ "name": "set_transition",
"round": 0,
- "atMs": 13127.710249999996,
- "execMs": 0.0019580000080168247
+ "atMs": 4809.502416999894,
+ "execMs": 0.03645899996627122
}
],
"timings": {
- "gateMs": 1648.9195829999953,
+ "gateMs": 1248.8274170000805,
"gateSkipped": false,
- "ttftMs": 16226.857417,
- "modelMs": 14891.264333,
- "toolMs": 0.012666000009630807,
- "totalMs": 16540.196582000004,
- "judgeMs": 1762.0136250000069,
+ "ttftMs": 6506.504166999948,
+ "modelMs": 6837.670207999996,
+ "toolMs": 0.03645899996627122,
+ "totalMs": 6842.941207999946,
+ "judgeMs": 1037.1046250000363,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 11475.751208000001,
+ "roundMs": 4806.579374999972,
"firstDeltaMs": null,
"calls": [
{
- "name": "remove_transition",
- "execMs": 0.010708000001613982
- },
- {
- "name": "remove_transition",
- "execMs": 0.0019580000080168247
+ "name": "set_transition",
+ "execMs": 0.03645899996627122
}
]
},
{
- "roundMs": 3415.5131249999977,
- "firstDeltaMs": 3097.727332999988,
+ "roundMs": 2031.0908330000238,
+ "firstDeltaMs": 1694.86258299998,
"calls": []
}
]
@@ -7073,38 +6462,38 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1018,
- "p95": 1250,
- "mean": 1066,
+ "p50": 1140,
+ "p95": 1838,
+ "mean": 1321,
"n": 3
},
"ttftMs": {
- "p50": 4987,
- "p95": 9011,
- "mean": 6012,
+ "p50": 3151,
+ "p95": 3584,
+ "mean": 3071,
"n": 3
},
"totalMs": {
- "p50": 5172,
- "p95": 9489,
- "mean": 6280,
+ "p50": 3391,
+ "p95": 3991,
+ "mean": 3367,
"n": 3
},
"modelMs": {
- "p50": 4243,
- "p95": 8238,
- "mean": 5214,
+ "p50": 3384,
+ "p95": 3982,
+ "mean": 3359,
"n": 3
},
"toolMs": {
"p50": 0,
- "p95": 1,
+ "p95": 0,
"mean": 0,
"n": 3
},
"rounds": {
- "mean": 2.3,
- "max": 3
+ "mean": 2,
+ "max": 2
}
},
"runs": [
@@ -7118,80 +6507,64 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 5761.352666999999,
- "execMs": 0.4585419999930309
- },
- {
- "name": "delete_cue",
- "round": 0,
- "atMs": 5762.771458999996,
- "execMs": 0.04679100000066683
+ "atMs": 2036.0195419999072,
+ "execMs": 0.28270799992606044
},
{
"name": "update_cue",
- "round": 1,
- "atMs": 7945.6942500000005,
- "execMs": 0.005541999998968095
+ "round": 0,
+ "atMs": 2036.591625000001,
+ "execMs": 0.04608400003053248
},
{
- "name": "update_cue",
- "round": 1,
- "atMs": 7945.741458999997,
- "execMs": 0.0013750000071013346
+ "name": "delete_cue",
+ "round": 0,
+ "atMs": 2036.888957999996,
+ "execMs": 0.06337500002700835
},
{
"name": "update_cue",
- "round": 1,
- "atMs": 7945.776249999995,
- "execMs": 0.0011249999952269718
+ "round": 0,
+ "atMs": 2037.075332999928,
+ "execMs": 0.023790999897755682
}
],
"timings": {
- "gateMs": 1250.0429169999989,
+ "gateMs": 985.5197919999482,
"gateSkipped": false,
- "ttftMs": 9011.184209,
- "modelMs": 8238.132668000006,
- "toolMs": 0.5133749999949941,
- "totalMs": 9488.68896,
- "judgeMs": 953.9077499999985,
+ "ttftMs": 3151.451166999992,
+ "modelMs": 3384.0143329998245,
+ "toolMs": 0.41595799988135695,
+ "totalMs": 3390.957499999902,
+ "judgeMs": 1908.927958000102,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 4504.689584000007,
+ "roundMs": 2032.4608329999028,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.4585419999930309
+ "execMs": 0.28270799992606044
},
- {
- "name": "delete_cue",
- "execMs": 0.04679100000066683
- }
- ]
- },
- {
- "roundMs": 2181.597999999998,
- "firstDeltaMs": null,
- "calls": [
{
"name": "update_cue",
- "execMs": 0.005541999998968095
+ "execMs": 0.04608400003053248
},
{
- "name": "update_cue",
- "execMs": 0.0013750000071013346
+ "name": "delete_cue",
+ "execMs": 0.06337500002700835
},
{
"name": "update_cue",
- "execMs": 0.0011249999952269718
+ "execMs": 0.023790999897755682
}
]
},
{
- "roundMs": 1551.8450840000005,
- "firstDeltaMs": 1065.0758750000095,
+ "roundMs": 1351.5534999999218,
+ "firstDeltaMs": 1112.3028749999357,
"calls": []
}
]
@@ -7207,64 +6580,64 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 2795.468417000011,
- "execMs": 0.017332999996142462
+ "atMs": 2435.6814160000067,
+ "execMs": 0.01624999998603016
},
{
"name": "update_cue",
"round": 0,
- "atMs": 2795.694000000003,
- "execMs": 0.008625000002211891
+ "atMs": 2436.573290999979,
+ "execMs": 0.00933399994391948
},
{
"name": "delete_cue",
"round": 0,
- "atMs": 2795.8145420000073,
- "execMs": 0.019625000000814907
+ "atMs": 2436.8950830000686,
+ "execMs": 0.025541000068187714
},
{
"name": "update_cue",
"round": 0,
- "atMs": 2795.9261670000124,
- "execMs": 0.005624999990686774
+ "atMs": 2437.0326249999925,
+ "execMs": 0.003499999991618097
}
],
"timings": {
- "gateMs": 1018.4886249999981,
+ "gateMs": 1838.1636670000153,
"gateSkipped": false,
- "ttftMs": 4037.703125,
- "modelMs": 3162.185499000014,
- "toolMs": 0.051207999989856035,
- "totalMs": 4180.725332000002,
- "judgeMs": 1023.3754159999953,
+ "ttftMs": 3584.423000000068,
+ "modelMs": 3982.1040009999415,
+ "toolMs": 0.05462499998975545,
+ "totalMs": 3990.5751660000533,
+ "judgeMs": 1192.4498330000788,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1774.6703330000018,
+ "roundMs": 2431.352458999958,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.017332999996142462
+ "execMs": 0.01624999998603016
},
{
"name": "update_cue",
- "execMs": 0.008625000002211891
+ "execMs": 0.00933399994391948
},
{
"name": "delete_cue",
- "execMs": 0.019625000000814907
+ "execMs": 0.025541000068187714
},
{
"name": "update_cue",
- "execMs": 0.005624999990686774
+ "execMs": 0.003499999991618097
}
]
},
{
- "roundMs": 1387.515166000012,
- "firstDeltaMs": 1240.4643330000108,
+ "roundMs": 1550.7515419999836,
+ "firstDeltaMs": 1144.9505000000354,
"calls": []
}
]
@@ -7280,78 +6653,167 @@
{
"name": "update_cue",
"round": 0,
- "atMs": 3607.715458999999,
- "execMs": 0.014416000005439855
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3608.07699999999,
- "execMs": 0.005875000002561137
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3608.2425419999927,
- "execMs": 0.010708000001613982
- },
- {
- "name": "update_cue",
- "round": 0,
- "atMs": 3608.3545000000013,
- "execMs": 0.007499999992433004
+ "atMs": 1412.2543329999316,
+ "execMs": 0.13254200003575534
},
{
"name": "update_cue",
"round": 0,
- "atMs": 3608.466333999997,
- "execMs": 0.004916999998386018
+ "atMs": 1412.902374999947,
+ "execMs": 0.04425000003539026
}
],
"timings": {
- "gateMs": 928.9838329999911,
+ "gateMs": 1140.032750000013,
"gateSkipped": false,
- "ttftMs": 4987.174459000002,
- "modelMs": 4242.606583999994,
- "toolMs": 0.043416000000433996,
- "totalMs": 5171.633832999985,
- "judgeMs": 993.1812919999938,
+ "ttftMs": 2478.086666999967,
+ "modelMs": 2711.3581249999115,
+ "toolMs": 0.1767920000711456,
+ "totalMs": 2719.77566699998,
+ "judgeMs": 1717.675125000067,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2676.7479170000006,
+ "roundMs": 1407.245916999993,
"firstDeltaMs": null,
"calls": [
{
"name": "update_cue",
- "execMs": 0.014416000005439855
- },
- {
- "name": "update_cue",
- "execMs": 0.005875000002561137
- },
- {
- "name": "update_cue",
- "execMs": 0.010708000001613982
- },
- {
- "name": "update_cue",
- "execMs": 0.007499999992433004
+ "execMs": 0.13254200003575534
},
{
"name": "update_cue",
- "execMs": 0.004916999998386018
+ "execMs": 0.04425000003539026
}
]
},
{
- "roundMs": 1565.8586669999931,
- "firstDeltaMs": 1377.2039590000059,
+ "roundMs": 1304.1122079999186,
+ "firstDeltaMs": 1063.1052500000224,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "question-when-ambiguous",
+ "bucket": "chat",
+ "passRate": 0.6666666666666666,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 1794,
+ "p95": 2712,
+ "mean": 2253,
+ "n": 2
+ },
+ "ttftMs": {
+ "p50": 4876,
+ "p95": 9078,
+ "mean": 6977,
+ "n": 2
+ },
+ "totalMs": {
+ "p50": 4929,
+ "p95": 9199,
+ "mean": 7064,
+ "n": 2
+ },
+ "modelMs": {
+ "p50": 4498,
+ "p95": 9195,
+ "mean": 6846,
+ "n": 2
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 2
+ },
+ "rounds": {
+ "mean": 2.5,
+ "max": 3
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": "judge unavailable (502)",
+ "trace": [],
+ "timings": {
+ "gateMs": 1793.5281250000698,
+ "gateSkipped": false,
+ "ttftMs": 4876.252792000072,
+ "modelMs": 4497.608749999898,
+ "toolMs": 0,
+ "totalMs": 4928.664667000063,
+ "judgeMs": 11277.159375000047,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1365.4640420000069,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 0.016749999951571226,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 3132.127957999939,
+ "firstDeltaMs": 3080.133166000014,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "complex",
+ "roundModel": "gemini-3.7-flash",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 2711.6676669999724,
+ "gateSkipped": false,
+ "ttftMs": 9078.016625000047,
+ "modelMs": 9194.962749999948,
+ "toolMs": 0,
+ "totalMs": 9198.995500000077,
+ "judgeMs": 2210.2406249999767,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 2711.2943749999395,
+ "firstDeltaMs": null,
+ "calls": []
+ },
+ {
+ "roundMs": 6483.668375000008,
+ "firstDeltaMs": 6362.950667000026,
"calls": []
}
]
}
+ },
+ {
+ "pass": false,
+ "intent": "work",
+ "notes": [],
+ "trace": [],
+ "timings": null,
+ "error": "This operation was aborted"
}
]
},
@@ -7362,27 +6824,27 @@
"budgetBreaches": [],
"latency": {
"gateMs": {
- "p50": 1721,
- "p95": 1860,
- "mean": 1612,
+ "p50": 2014,
+ "p95": 2097,
+ "mean": 1699,
"n": 3
},
"ttftMs": {
- "p50": 4263,
- "p95": 5319,
- "mean": 4601,
+ "p50": 3464,
+ "p95": 7303,
+ "mean": 4467,
"n": 3
},
"totalMs": {
- "p50": 4713,
- "p95": 5572,
- "mean": 4931,
+ "p50": 3833,
+ "p95": 8195,
+ "mean": 5023,
"n": 3
},
"modelMs": {
- "p50": 3253,
- "p95": 3713,
- "mean": 3319,
+ "p50": 3360,
+ "p95": 8188,
+ "mean": 4861,
"n": 3
},
"toolMs": {
@@ -7407,34 +6869,34 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 3944.534375000003,
- "execMs": 0.021833999999216758
+ "atMs": 5236.01883299998,
+ "execMs": 0.3058339999988675
}
],
"timings": {
- "gateMs": 1859.7827499999985,
+ "gateMs": 2014.0730410000542,
"gateSkipped": false,
- "ttftMs": 5318.673958000014,
- "modelMs": 3712.6843329999974,
- "toolMs": 0.021833999999216758,
- "totalMs": 5572.488916999995,
- "judgeMs": 1799.7672500000044,
+ "ttftMs": 7302.643708000076,
+ "modelMs": 8187.526000000071,
+ "toolMs": 0.3058339999988675,
+ "totalMs": 8194.814708000049,
+ "judgeMs": 988.7312500000698,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 2082.2217910000036,
+ "roundMs": 5231.981084000086,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.021833999999216758
+ "execMs": 0.3058339999988675
}
]
},
{
- "roundMs": 1630.4625419999938,
- "firstDeltaMs": 1373.7688340000022,
+ "roundMs": 2955.5449159999844,
+ "firstDeltaMs": 2063.775290999911,
"calls": []
}
]
@@ -7450,34 +6912,34 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 2961.0620000000054,
- "execMs": 0.005124999996041879
+ "atMs": 2097.1743750000605,
+ "execMs": 0.0032920000376179814
}
],
"timings": {
- "gateMs": 1254.5397499999963,
+ "gateMs": 2096.8562089999905,
"gateSkipped": false,
- "ttftMs": 4263.229042000006,
- "modelMs": 3252.521374999982,
- "toolMs": 0.005124999996041879,
- "totalMs": 4507.066249999974,
- "judgeMs": 1752.2745409999916,
+ "ttftMs": 3464.152584000025,
+ "modelMs": 3359.917041000095,
+ "toolMs": 0.0032920000376179814,
+ "totalMs": 3833.4667500000214,
+ "judgeMs": 1252.2110419999808,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1704.5885829999897,
+ "roundMs": 1624.392833000049,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.005124999996041879
+ "execMs": 0.0032920000376179814
}
]
},
{
- "roundMs": 1547.9327919999923,
- "firstDeltaMs": 1300.804416999992,
+ "roundMs": 1735.5242080000462,
+ "firstDeltaMs": 1366.4937080000527,
"calls": []
}
]
@@ -7493,34 +6955,156 @@
{
"name": "trim_clip",
"round": 0,
- "atMs": 3157.790500000003,
- "execMs": 0.004625000001396984
+ "atMs": 1466.8616250000196,
+ "execMs": 0.009624999947845936
}
],
"timings": {
- "gateMs": 1721.1325830000133,
+ "gateMs": 984.6576669999631,
"gateSkipped": false,
- "ttftMs": 4220.116083999994,
- "modelMs": 2992.297999000002,
- "toolMs": 0.004625000001396984,
- "totalMs": 4713.435207000017,
- "judgeMs": 1583.2242079999996,
+ "ttftMs": 2635.555000000051,
+ "modelMs": 3036.4881670001196,
+ "toolMs": 0.009624999947845936,
+ "totalMs": 3040.7009170000674,
+ "judgeMs": 1172.3532920000143,
"retryMs": 0,
"retries": 0,
"rounds": [
{
- "roundMs": 1434.5011660000018,
+ "roundMs": 1465.082250000094,
"firstDeltaMs": null,
"calls": [
{
"name": "trim_clip",
- "execMs": 0.004625000001396984
+ "execMs": 0.009624999947845936
}
]
},
{
- "roundMs": 1557.7968330000003,
- "firstDeltaMs": 1061.0222910000011,
+ "roundMs": 1571.4059170000255,
+ "firstDeltaMs": 1166.597833000007,
+ "calls": []
+ }
+ ]
+ }
+ }
+ ]
+ },
+ {
+ "name": "legacy-thread-no-leak",
+ "bucket": "chat",
+ "passRate": 1,
+ "budgetBreaches": [],
+ "latency": {
+ "gateMs": {
+ "p50": 996,
+ "p95": 1311,
+ "mean": 1081,
+ "n": 3
+ },
+ "ttftMs": {
+ "p50": 1210,
+ "p95": 1489,
+ "mean": 1263,
+ "n": 3
+ },
+ "totalMs": {
+ "p50": 1368,
+ "p95": 1661,
+ "mean": 1442,
+ "n": 3
+ },
+ "modelMs": {
+ "p50": 1365,
+ "p95": 1658,
+ "mean": 1439,
+ "n": 3
+ },
+ "toolMs": {
+ "p50": 0,
+ "p95": 0,
+ "mean": 0,
+ "n": 3
+ },
+ "rounds": {
+ "mean": 1,
+ "max": 1
+ }
+ },
+ "runs": [
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 934.7855419999687,
+ "gateSkipped": false,
+ "ttftMs": 1209.547833000077,
+ "modelMs": 1365.3867910000263,
+ "toolMs": 0,
+ "totalMs": 1368.2847499999916,
+ "judgeMs": 998.8910840000026,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1365.3867910000263,
+ "firstDeltaMs": 1206.719999999972,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 1310.6001249999972,
+ "gateSkipped": false,
+ "ttftMs": 1488.958290999988,
+ "modelMs": 1658.3532499999274,
+ "toolMs": 0,
+ "totalMs": 1660.6062079999829,
+ "judgeMs": 1379.843249999918,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1658.3532499999274,
+ "firstDeltaMs": 1486.9065419998951,
+ "calls": []
+ }
+ ]
+ }
+ },
+ {
+ "pass": true,
+ "intent": "simple",
+ "roundModel": "gemini-3.1-flash-lite",
+ "notes": [],
+ "judgeNote": null,
+ "trace": [],
+ "timings": {
+ "gateMs": 996.2925420000684,
+ "gateSkipped": false,
+ "ttftMs": 1090.0226249999832,
+ "modelMs": 1292.6347500000848,
+ "toolMs": 0,
+ "totalMs": 1295.6339999999618,
+ "judgeMs": 2068.7990829999326,
+ "retryMs": 0,
+ "retries": 0,
+ "rounds": [
+ {
+ "roundMs": 1292.6347500000848,
+ "firstDeltaMs": 1087.285666000098,
"calls": []
}
]
diff --git a/site/src/cut/components/AiPanel.tsx b/site/src/cut/components/AiPanel.tsx
index 21566b03..9e460484 100644
--- a/site/src/cut/components/AiPanel.tsx
+++ b/site/src/cut/components/AiPanel.tsx
@@ -328,11 +328,15 @@ export function AiPanel({
const engineUp = useLocalCompute();
const readOnly = useEditor((s) => s.readOnly);
const signedIn = useSignedIn();
- const [model, setModel] = useState<string>(() =>
- typeof window === "undefined"
- ? DEFAULT_MODEL
- : (localStorage.getItem(MODEL_KEY) ?? DEFAULT_MODEL),
- );
+ // A saved id whose model left the catalog (a retired model after a bump)
+ // falls back to the default; passed through, the hosted route rejects it.
+ const [model, setModel] = useState<string>(() => {
+ if (typeof window === "undefined") return DEFAULT_MODEL;
+ const saved = localStorage.getItem(MODEL_KEY);
+ return saved && AI_MODELS.some((m) => m.id === saved)
+ ? saved
+ : DEFAULT_MODEL;
+ });
// One chat is active at a time; every past chat lives in the Threads panel.
// The id persists so closing and reopening the panel resumes the same chat.
const [activeChat, setActiveChat] = useState<string>(() =>
diff --git a/site/src/lib/credits/provider-pricing.ts b/site/src/lib/credits/provider-pricing.ts
index 96763a12..3658b9bc 100644
--- a/site/src/lib/credits/provider-pricing.ts
+++ b/site/src/lib/credits/provider-pricing.ts
@@ -247,15 +247,32 @@ const openAITextCreditRates: {
{ model: "o4-mini", input: "1.1", cachedInput: "0.275", output: "4.4" },
];
+// gemini-3.7-flash launch pricing runs through 2026-12-31; standard pricing
+// starts 2027-01-01 UTC. Audio input bills at the text rate. The record entry
+// is a getter, so every lookup reads the clock and the switch happens on
+// schedule without a deploy.
+const geminiFlashStandardPricingStartMs = Date.UTC(2027, 0, 1);
+const geminiFlashLaunchPricing = textAudioTokenPricing({
+ input: "0.75",
+ cachedInput: "0.075",
+ output: "3.75",
+ inputAudio: "0.75",
+});
+const geminiFlashStandardPricing = textAudioTokenPricing({
+ input: "1.5",
+ cachedInput: "0.15",
+ output: "7.5",
+ inputAudio: "1.5",
+});
+
// Every Gemini model we run must appear here: the Record is keyed by the GeminiModel union,
// so adding a model to gemini-models.ts without a price fails the type-check (and the build).
const geminiModelPricing: Record<GeminiModel, ProviderCreditPricing> = {
- [geminiModels.flash]: textTokenPricing({
- model: "gemini-3.5-flash",
- input: "1.5",
- cachedInput: "0.15",
- output: "9",
- }),
+ get [geminiModels.flash]() {
+ return Date.now() >= geminiFlashStandardPricingStartMs
+ ? geminiFlashStandardPricing
+ : geminiFlashLaunchPricing;
+ },
[geminiModels.flashLite]: textAudioTokenPricing({
input: "0.25",
cachedInput: "0.025",
diff --git a/site/src/lib/inference/gemini-models.ts b/site/src/lib/inference/gemini-models.ts
index 321a9c14..3a33e636 100644
--- a/site/src/lib/inference/gemini-models.ts
+++ b/site/src/lib/inference/gemini-models.ts
@@ -8,7 +8,7 @@
// Canonical, dated-or-versioned model IDs. Add a new constant when adopting a
// new model; do not inline raw version strings at call sites.
export const geminiModels = {
- flash: "gemini-3.5-flash",
+ flash: "gemini-3.7-flash",
flashLite: "gemini-3.1-flash-lite",
// Generative image editing/generation ("nano banana"). Bump here when adopting a
// newer image model.